Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
chore(release): v3.12.0
Apify Release Botca5109157f46
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex cb11c6828..e10f87d34 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -3,6 +3,25 @@All notable changes to this project will be documented in this file.See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.+# [3.12.0](https://github.com/apify/crawlee/compare/v3.11.5...v3.12.0) (2024-11-04)+++### Bug Fixes++* `.trim()` urls from pretty-printed sitemap.xml files ([#2709](https://github.com/apify/crawlee/issues/2709)) ([802a6fe](https://github.com/apify/crawlee/commit/802a6fea7b2125e2b36d740fc2d5d131de5d53ed)), closes [#2698](https://github.com/apify/crawlee/issues/2698)+* **core:** ensure correct column order in CSV export ([#2734](https://github.com/apify/crawlee/issues/2734)) ([b66784f](https://github.com/apify/crawlee/commit/b66784f89f011c2f972d73ec9cd47235a0411d1c)), closes [#2718](https://github.com/apify/crawlee/issues/2718)+* ignore errors from iframe content extraction ([#2714](https://github.com/apify/crawlee/issues/2714)) ([627e5c2](https://github.com/apify/crawlee/commit/627e5c2fbadce63c7e631217cd0e735597c0ce08)), closes [#2708](https://github.com/apify/crawlee/issues/2708)+* update `fingerprintGeneratorOptions` types ([#2705](https://github.com/apify/crawlee/issues/2705)) ([fcb098d](https://github.com/apify/crawlee/commit/fcb098d6357b69e6d1790765076e4fe4146c8143)), closes [/github.com/apify/fingerprint-suite/blob/c61814e6ba8822543deb0ce6c03e0a0249933629/packages/fingerprint-generator/src/fingerprint-generator.ts#L73](https://github.com//github.com/apify/fingerprint-suite/blob/c61814e6ba8822543deb0ce6c03e0a0249933629/packages/fingerprint-generator/src/fingerprint-generator.ts/issues/L73) [/github.com/apify/fingerprint-suite/blob/c61814e6ba8822543deb0ce6c03e0a0249933629/packages/header-generator/src/header-generator.ts#L87](https://github.com//github.com/apify/fingerprint-suite/blob/c61814e6ba8822543deb0ce6c03e0a0249933629/packages/header-generator/src/header-generator.ts/issues/L87) [#2703](https://github.com/apify/crawlee/issues/2703)+++### Features++* allow using other HTTP clients ([#2661](https://github.com/apify/crawlee/issues/2661)) ([568c655](https://github.com/apify/crawlee/commit/568c6556d79ce91654c8a715d1d1729d7d6ed8ef)), closes [#2659](https://github.com/apify/crawlee/issues/2659)+++++## [3.11.5](https://github.com/apify/crawlee/compare/v3.11.4...v3.11.5) (2024-10-04)diff --git a/lerna.json b/lerna.jsonindex 77c035457..dd362bada 100644--- a/lerna.json+++ b/lerna.json@@ -2,7 +2,7 @@"packages": ["packages/*"],- "version": "3.11.5",+ "version": "3.12.0","command": {"version": {"conventionalCommits": true,diff --git a/packages/basic-crawler/CHANGELOG.md b/packages/basic-crawler/CHANGELOG.mdindex f522b8da4..86f33d61b 100644--- a/packages/basic-crawler/CHANGELOG.md+++ b/packages/basic-crawler/CHANGELOG.md@@ -3,6 +3,17 @@All notable changes to this project will be documented in this file.See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.+# [3.12.0](https://github.com/apify/crawlee/compare/v3.11.5...v3.12.0) (2024-11-04)+++### Features++* allow using other HTTP clients ([#2661](https://github.com/apify/crawlee/issues/2661)) ([568c655](https://github.com/apify/crawlee/commit/568c6556d79ce91654c8a715d1d1729d7d6ed8ef)), closes [#2659](https://github.com/apify/crawlee/issues/2659)+++++## [3.11.5](https://github.com/apify/crawlee/compare/v3.11.4...v3.11.5) (2024-10-04)diff --git a/packages/basic-crawler/package.json b/packages/basic-crawler/package.jsonindex 59fae8fac..521c6118b 100644--- a/packages/basic-crawler/package.json+++ b/packages/basic-crawler/package.json@@ -1,6 +1,6 @@{"name": "@crawlee/basic",- "version": "3.11.5",+ "version": "3.12.0","description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.","engines": {"node": ">=16.0.0"@@ -48,9 +48,9 @@"@apify/log": "^2.4.0","@apify/timeout": "^0.3.0","@apify/utilities": "^2.7.10",- "@crawlee/core": "3.11.5",- "@crawlee/types": "3.11.5",- "@crawlee/utils": "3.11.5",+ "@crawlee/core": "^3.12.0",+ "@crawlee/types": "^3.12.0",+ "@crawlee/utils": "^3.12.0","csv-stringify": "^6.2.0","fs-extra": "^11.0.0","got-scraping": "^4.0.0",diff --git a/packages/browser-crawler/CHANGELOG.md b/packages/browser-crawler/CHANGELOG.mdindex ac3a0a940..9a612b739 100644--- a/packages/browser-crawler/CHANGELOG.md+++ b/packages/browser-crawler/CHANGELOG.md@@ -3,6 +3,14 @@All notable changes to this project will be documented in this file.See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.+# [3.12.0](https://github.com/apify/crawlee/compare/v3.11.5...v3.12.0) (2024-11-04)++**Note:** Version bump only for package @crawlee/browser+++++## [3.11.5](https://github.com/apify/crawlee/compare/v3.11.4...v3.11.5) (2024-10-04)diff --git a/packages/browser-crawler/package.json b/packages/browser-crawler/package.jsonindex e490102b8..57819a832 100644--- a/packages/browser-crawler/package.json+++ b/packages/browser-crawler/package.json@@ -1,6 +1,6 @@{"name": "@crawlee/browser",- "version": "3.11.5",+ "version": "3.12.0","description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.","engines": {"node": ">=16.0.0"@@ -54,10 +54,10 @@},"dependencies": {"@apify/timeout": "^0.3.0",- "@crawlee/basic": "3.11.5",- "@crawlee/browser-pool": "3.11.5",- "@crawlee/types": "3.11.5",- "@crawlee/utils": "3.11.5",+ "@crawlee/basic": "^3.12.0",+ "@crawlee/browser-pool": "^3.12.0",+ "@crawlee/types": "^3.12.0",+ "@crawlee/utils": "^3.12.0","ow": "^0.28.1","tslib": "^2.4.0","type-fest": "^4.0.0"diff --git a/packages/browser-pool/CHANGELOG.md b/packages/browser-pool/CHANGELOG.mdindex 6aad22cd6..5f0abbdfa 100644--- a/packages/browser-pool/CHANGELOG.md+++ b/packages/browser-pool/CHANGELOG.md@@ -3,6 +3,22 @@All notable changes to this project will be documented in this file.See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.+# [3.12.0](https://github.com/apify/crawlee/compare/v3.11.5...v3.12.0) (2024-11-04)+++### Bug Fixes++* update `fingerprintGeneratorOptions` types ([#2705](https://github.com/apify/crawlee/issues/2705)) ([fcb098d](https://github.com/apify/crawlee/commit/fcb098d6357b69e6d1790765076e4fe4146c8143)), closes [/github.com/apify/fingerprint-suite/blob/c61814e6ba8822543deb0ce6c03e0a0249933629/packages/fingerprint-generator/src/fingerprint-generator.ts#L73](https://github.com//github.com/apify/fingerprint-suite/blob/c61814e6ba8822543deb0ce6c03e0a0249933629/packages/fingerprint-generator/src/fingerprint-generator.ts/issues/L73) [/github.com/apify/fingerprint-suite/blob/c61814e6ba8822543deb0ce6c03e0a0249933629/packages/header-generator/src/header-generator.ts#L87](https://github.com//github.com/apify/fingerprint-suite/blob/c61814e6ba8822543deb0ce6c03e0a0249933629/packages/header-generator/src/header-generator.ts/issues/L87) [#2703](https://github.com/apify/crawlee/issues/2703)+++### Features++* allow using other HTTP clients ([#2661](https://github.com/apify/crawlee/issues/2661)) ([568c655](https://github.com/apify/crawlee/commit/568c6556d79ce91654c8a715d1d1729d7d6ed8ef)), closes [#2659](https://github.com/apify/crawlee/issues/2659)+++++## [3.11.5](https://github.com/apify/crawlee/compare/v3.11.4...v3.11.5) (2024-10-04)**Note:** Version bump only for package @crawlee/browser-pooldiff --git a/packages/browser-pool/package.json b/packages/browser-pool/package.jsonindex 91f3f4871..9f05f3ef5 100644--- a/packages/browser-pool/package.json+++ b/packages/browser-pool/package.json@@ -1,6 +1,6 @@{"name": "@crawlee/browser-pool",- "version": "3.11.5",+ "version": "3.12.0","description": "Rotate multiple browsers using popular automation libraries such as Playwright or Puppeteer.","engines": {"node": ">=16.0.0"@@ -38,8 +38,8 @@"dependencies": {"@apify/log": "^2.4.0","@apify/timeout": "^0.3.0",- "@crawlee/core": "3.11.5",- "@crawlee/types": "3.11.5",+ "@crawlee/core": "^3.12.0",+ "@crawlee/types": "^3.12.0","fingerprint-generator": "^2.0.6","fingerprint-injector": "^2.0.5","lodash.merge": "^4.6.2",diff --git a/packages/cheerio-crawler/CHANGELOG.md b/packages/cheerio-crawler/CHANGELOG.mdindex 4889a031d..66472598d 100644--- a/packages/cheerio-crawler/CHANGELOG.md+++ b/packages/cheerio-crawler/CHANGELOG.md@@ -3,6 +3,14 @@All notable changes to this project will be documented in this file.See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.+# [3.12.0](https://github.com/apify/crawlee/compare/v3.11.5...v3.12.0) (2024-11-04)++**Note:** Version bump only for package @crawlee/cheerio+++++## [3.11.5](https://github.com/apify/crawlee/compare/v3.11.4...v3.11.5) (2024-10-04)**Note:** Version bump only for package @crawlee/cheeriodiff --git a/packages/cheerio-crawler/package.json b/packages/cheerio-crawler/package.jsonindex 62a2d34f5..b6b522d22 100644--- a/packages/cheerio-crawler/package.json+++ b/packages/cheerio-crawler/package.json@@ -1,6 +1,6 @@{"name": "@crawlee/cheerio",- "version": "3.11.5",+ "version": "3.12.0","description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.","engines": {"node": ">=16.0.0"@@ -53,9 +53,9 @@"access": "public"},"dependencies": {- "@crawlee/http": "3.11.5",- "@crawlee/types": "3.11.5",- "@crawlee/utils": "3.11.5",+ "@crawlee/http": "^3.12.0",+ "@crawlee/types": "^3.12.0",+ "@crawlee/utils": "^3.12.0","cheerio": "1.0.0-rc.12","htmlparser2": "^9.0.0","tslib": "^2.4.0"diff --git a/packages/cli/CHANGELOG.md b/packages/cli/CHANGELOG.mdindex 33fc7b3bb..58c384442 100644--- a/packages/cli/CHANGELOG.md+++ b/packages/cli/CHANGELOG.md@@ -3,6 +3,14 @@All notable changes to this project will be documented in this file.See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.+# [3.12.0](https://github.com/apify/crawlee/compare/v3.11.5...v3.12.0) (2024-11-04)++**Note:** Version bump only for package @crawlee/cli+++++## [3.11.5](https://github.com/apify/crawlee/compare/v3.11.4...v3.11.5) (2024-10-04)**Note:** Version bump only for package @crawlee/clidiff --git a/packages/cli/package.json b/packages/cli/package.jsonindex 28134ed97..02555a194 100644--- a/packages/cli/package.json+++ b/packages/cli/package.json@@ -1,6 +1,6 @@{"name": "@crawlee/cli",- "version": "3.11.5",+ "version": "3.12.0","description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.","engines": {"node": ">=16.0.0"@@ -51,7 +51,7 @@"access": "public"},"dependencies": {- "@crawlee/templates": "3.11.5",+ "@crawlee/templates": "^3.12.0","ansi-colors": "^4.1.3","fs-extra": "^11.0.0","inquirer": "^8.2.4",diff --git a/packages/core/CHANGELOG.md b/packages/core/CHANGELOG.mdindex 18e2ccf45..b46c03e25 100644--- a/packages/core/CHANGELOG.md+++ b/packages/core/CHANGELOG.md@@ -3,6 +3,22 @@All notable changes to this project will be documented in this file.See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.+# [3.12.0](https://github.com/apify/crawlee/compare/v3.11.5...v3.12.0) (2024-11-04)+++### Bug Fixes++* **core:** ensure correct column order in CSV export ([#2734](https://github.com/apify/crawlee/issues/2734)) ([b66784f](https://github.com/apify/crawlee/commit/b66784f89f011c2f972d73ec9cd47235a0411d1c)), closes [#2718](https://github.com/apify/crawlee/issues/2718)+++### Features++* allow using other HTTP clients ([#2661](https://github.com/apify/crawlee/issues/2661)) ([568c655](https://github.com/apify/crawlee/commit/568c6556d79ce91654c8a715d1d1729d7d6ed8ef)), closes [#2659](https://github.com/apify/crawlee/issues/2659)+++++## [3.11.5](https://github.com/apify/crawlee/compare/v3.11.4...v3.11.5) (2024-10-04)diff --git a/packages/core/package.json b/packages/core/package.jsonindex 17295492c..ff9b0b32c 100644--- a/packages/core/package.json+++ b/packages/core/package.json@@ -1,6 +1,6 @@{"name": "@crawlee/core",- "version": "3.11.5",+ "version": "3.12.0","description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.","engines": {"node": ">=16.0.0"@@ -59,9 +59,9 @@"@apify/pseudo_url": "^2.0.30","@apify/timeout": "^0.3.0","@apify/utilities": "^2.7.10",- "@crawlee/memory-storage": "3.11.5",- "@crawlee/types": "3.11.5",- "@crawlee/utils": "3.11.5",+ "@crawlee/memory-storage": "^3.12.0",+ "@crawlee/types": "^3.12.0",+ "@crawlee/utils": "^3.12.0","@sapphire/async-queue": "^1.5.1","@vladfrangu/async_event_emitter": "^2.2.2","csv-stringify": "^6.2.0",diff --git a/packages/crawlee/CHANGELOG.md b/packages/crawlee/CHANGELOG.mdindex 46052be83..aa70731d2 100644--- a/packages/crawlee/CHANGELOG.md+++ b/packages/crawlee/CHANGELOG.md@@ -3,6 +3,14 @@All notable changes to this project will be documented in this file.See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.+# [3.12.0](https://github.com/apify/crawlee/compare/v3.11.5...v3.12.0) (2024-11-04)++**Note:** Version bump only for package crawlee+++++## [3.11.5](https://github.com/apify/crawlee/compare/v3.11.4...v3.11.5) (2024-10-04)**Note:** Version bump only for package crawleediff --git a/packages/crawlee/package.json b/packages/crawlee/package.jsonindex bf6a6a13a..44a41ad89 100644--- a/packages/crawlee/package.json+++ b/packages/crawlee/package.json@@ -1,6 +1,6 @@{"name": "crawlee",- "version": "3.11.5",+ "version": "3.12.0","description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.","engines": {"node": ">=16.0.0"@@ -54,18 +54,18 @@"access": "public"},"dependencies": {- "@crawlee/basic": "3.11.5",- "@crawlee/browser": "3.11.5",- "@crawlee/browser-pool": "3.11.5",- "@crawlee/cheerio": "3.11.5",- "@crawlee/cli": "3.11.5",- "@crawlee/core": "3.11.5",- "@crawlee/http": "3.11.5",- "@crawlee/jsdom": "3.11.5",- "@crawlee/linkedom": "3.11.5",- "@crawlee/playwright": "3.11.5",- "@crawlee/puppeteer": "3.11.5",- "@crawlee/utils": "3.11.5",+ "@crawlee/basic": "^3.12.0",+ "@crawlee/browser": "^3.12.0",+ "@crawlee/browser-pool": "^3.12.0",+ "@crawlee/cheerio": "^3.12.0",+ "@crawlee/cli": "^3.12.0",+ "@crawlee/core": "^3.12.0",+ "@crawlee/http": "^3.12.0",+ "@crawlee/jsdom": "^3.12.0",+ "@crawlee/linkedom": "^3.12.0",+ "@crawlee/playwright": "^3.12.0",+ "@crawlee/puppeteer": "^3.12.0",+ "@crawlee/utils": "^3.12.0","import-local": "^3.1.0","tslib": "^2.4.0"},diff --git a/packages/http-crawler/CHANGELOG.md b/packages/http-crawler/CHANGELOG.mdindex 542d1a96c..2f4c135e5 100644--- a/packages/http-crawler/CHANGELOG.md+++ b/packages/http-crawler/CHANGELOG.md@@ -3,6 +3,17 @@All notable changes to this project will be documented in this file.See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.+# [3.12.0](https://github.com/apify/crawlee/compare/v3.11.5...v3.12.0) (2024-11-04)+++### Features++* allow using other HTTP clients ([#2661](https://github.com/apify/crawlee/issues/2661)) ([568c655](https://github.com/apify/crawlee/commit/568c6556d79ce91654c8a715d1d1729d7d6ed8ef)), closes [#2659](https://github.com/apify/crawlee/issues/2659)+++++## [3.11.5](https://github.com/apify/crawlee/compare/v3.11.4...v3.11.5) (2024-10-04)diff --git a/packages/http-crawler/package.json b/packages/http-crawler/package.jsonindex b4e13699a..a651e996a 100644--- a/packages/http-crawler/package.json+++ b/packages/http-crawler/package.json@@ -1,6 +1,6 @@{"name": "@crawlee/http",- "version": "3.11.5",+ "version": "3.12.0","description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.","engines": {"node": ">=16.0.0"@@ -55,9 +55,9 @@"dependencies": {"@apify/timeout": "^0.3.0","@apify/utilities": "^2.7.10",- "@crawlee/basic": "3.11.5",- "@crawlee/types": "3.11.5",- "@crawlee/utils": "3.11.5",+ "@crawlee/basic": "^3.12.0",+ "@crawlee/types": "^3.12.0",+ "@crawlee/utils": "^3.12.0","@types/content-type": "^1.1.5","cheerio": "1.0.0-rc.12","content-type": "^1.0.4",diff --git a/packages/jsdom-crawler/CHANGELOG.md b/packages/jsdom-crawler/CHANGELOG.mdindex 64344a7a0..1d88e1b49 100644--- a/packages/jsdom-crawler/CHANGELOG.md+++ b/packages/jsdom-crawler/CHANGELOG.md@@ -3,6 +3,14 @@All notable changes to this project will be documented in this file.See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.+# [3.12.0](https://github.com/apify/crawlee/compare/v3.11.5...v3.12.0) (2024-11-04)++**Note:** Version bump only for package @crawlee/jsdom+++++## [3.11.5](https://github.com/apify/crawlee/compare/v3.11.4...v3.11.5) (2024-10-04)**Note:** Version bump only for package @crawlee/jsdomdiff --git a/packages/jsdom-crawler/package.json b/packages/jsdom-crawler/package.jsonindex 887852713..7bfe491ac 100644--- a/packages/jsdom-crawler/package.json+++ b/packages/jsdom-crawler/package.json@@ -1,6 +1,6 @@{"name": "@crawlee/jsdom",- "version": "3.11.5",+ "version": "3.12.0","description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.","engines": {"node": ">=16.0.0"@@ -55,9 +55,9 @@"dependencies": {"@apify/timeout": "^0.3.0","@apify/utilities": "^2.7.10",- "@crawlee/http": "3.11.5",- "@crawlee/types": "3.11.5",- "@crawlee/utils": "3.11.5",+ "@crawlee/http": "^3.12.0",+ "@crawlee/types": "^3.12.0",+ "@crawlee/utils": "^3.12.0","@types/jsdom": "^21.0.0","cheerio": "1.0.0-rc.12","jsdom": "^25.0.0",diff --git a/packages/linkedom-crawler/CHANGELOG.md b/packages/linkedom-crawler/CHANGELOG.mdindex 50cf1b166..2419a6531 100644--- a/packages/linkedom-crawler/CHANGELOG.md+++ b/packages/linkedom-crawler/CHANGELOG.md@@ -3,6 +3,14 @@All notable changes to this project will be documented in this file.See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.+# [3.12.0](https://github.com/apify/crawlee/compare/v3.11.5...v3.12.0) (2024-11-04)++**Note:** Version bump only for package @crawlee/linkedom+++++## [3.11.5](https://github.com/apify/crawlee/compare/v3.11.4...v3.11.5) (2024-10-04)**Note:** Version bump only for package @crawlee/linkedomdiff --git a/packages/linkedom-crawler/package.json b/packages/linkedom-crawler/package.jsonindex a35fd6ba6..b578e3c31 100644--- a/packages/linkedom-crawler/package.json+++ b/packages/linkedom-crawler/package.json@@ -1,6 +1,6 @@{"name": "@crawlee/linkedom",- "version": "3.11.5",+ "version": "3.12.0","description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.","engines": {"node": ">=16.0.0"@@ -55,8 +55,8 @@"dependencies": {"@apify/timeout": "^0.3.0","@apify/utilities": "^2.7.10",- "@crawlee/http": "3.11.5",- "@crawlee/types": "3.11.5",+ "@crawlee/http": "^3.12.0",+ "@crawlee/types": "^3.12.0","linkedom": "^0.18.0","ow": "^0.28.2","tslib": "^2.4.0"diff --git a/packages/memory-storage/CHANGELOG.md b/packages/memory-storage/CHANGELOG.mdindex d0ea8fb12..fe9195a16 100644--- a/packages/memory-storage/CHANGELOG.md+++ b/packages/memory-storage/CHANGELOG.md@@ -3,6 +3,14 @@All notable changes to this project will be documented in this file.See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.+# [3.12.0](https://github.com/apify/crawlee/compare/v3.11.5...v3.12.0) (2024-11-04)++**Note:** Version bump only for package @crawlee/memory-storage+++++## [3.11.5](https://github.com/apify/crawlee/compare/v3.11.4...v3.11.5) (2024-10-04)diff --git a/packages/memory-storage/package.json b/packages/memory-storage/package.jsonindex 93b74c645..ce49d43a4 100644--- a/packages/memory-storage/package.json+++ b/packages/memory-storage/package.json@@ -1,6 +1,6 @@{"name": "@crawlee/memory-storage",- "version": "3.11.5",+ "version": "3.12.0","description": "A simple in-memory storage implementation of the Apify API","engines": {"node": ">= 16"@@ -49,7 +49,7 @@},"dependencies": {"@apify/log": "^2.4.0",- "@crawlee/types": "3.11.5",+ "@crawlee/types": "^3.12.0","@sapphire/async-queue": "^1.5.0","@sapphire/shapeshift": "^3.0.0","content-type": "^1.0.4",diff --git a/packages/playwright-crawler/CHANGELOG.md b/packages/playwright-crawler/CHANGELOG.mdindex c743a3fb0..857e79a1e 100644--- a/packages/playwright-crawler/CHANGELOG.md+++ b/packages/playwright-crawler/CHANGELOG.md@@ -3,6 +3,17 @@All notable changes to this project will be documented in this file.See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.+# [3.12.0](https://github.com/apify/crawlee/compare/v3.11.5...v3.12.0) (2024-11-04)+++### Bug Fixes++* ignore errors from iframe content extraction ([#2714](https://github.com/apify/crawlee/issues/2714)) ([627e5c2](https://github.com/apify/crawlee/commit/627e5c2fbadce63c7e631217cd0e735597c0ce08)), closes [#2708](https://github.com/apify/crawlee/issues/2708)+++++## [3.11.5](https://github.com/apify/crawlee/compare/v3.11.4...v3.11.5) (2024-10-04)diff --git a/packages/playwright-crawler/package.json b/packages/playwright-crawler/package.jsonindex 2339fcc7d..bd266b64a 100644--- a/packages/playwright-crawler/package.json+++ b/packages/playwright-crawler/package.json@@ -1,6 +1,6 @@{"name": "@crawlee/playwright",- "version": "3.11.5",+ "version": "3.12.0","description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.","engines": {"node": ">=16.0.0"@@ -56,11 +56,11 @@"@apify/datastructures": "^2.0.0","@apify/log": "^2.4.0","@apify/timeout": "^0.3.1",- "@crawlee/browser": "3.11.5",- "@crawlee/browser-pool": "3.11.5",- "@crawlee/core": "3.11.5",- "@crawlee/types": "3.11.5",- "@crawlee/utils": "3.11.5",+ "@crawlee/browser": "^3.12.0",+ "@crawlee/browser-pool": "^3.12.0",+ "@crawlee/core": "^3.12.0",+ "@crawlee/types": "^3.12.0",+ "@crawlee/utils": "^3.12.0","cheerio": "1.0.0-rc.12","idcac-playwright": "^0.1.2","jquery": "^3.6.0",diff --git a/packages/puppeteer-crawler/CHANGELOG.md b/packages/puppeteer-crawler/CHANGELOG.mdindex 3cf6a0667..7af5e4780 100644--- a/packages/puppeteer-crawler/CHANGELOG.md+++ b/packages/puppeteer-crawler/CHANGELOG.md@@ -3,6 +3,17 @@All notable changes to this project will be documented in this file.See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.+# [3.12.0](https://github.com/apify/crawlee/compare/v3.11.5...v3.12.0) (2024-11-04)+++### Bug Fixes++* ignore errors from iframe content extraction ([#2714](https://github.com/apify/crawlee/issues/2714)) ([627e5c2](https://github.com/apify/crawlee/commit/627e5c2fbadce63c7e631217cd0e735597c0ce08)), closes [#2708](https://github.com/apify/crawlee/issues/2708)+++++## [3.11.5](https://github.com/apify/crawlee/compare/v3.11.4...v3.11.5) (2024-10-04)diff --git a/packages/puppeteer-crawler/package.json b/packages/puppeteer-crawler/package.jsonindex a4dcefe8b..b50e5b624 100644--- a/packages/puppeteer-crawler/package.json+++ b/packages/puppeteer-crawler/package.json@@ -1,6 +1,6 @@{"name": "@crawlee/puppeteer",- "version": "3.11.5",+ "version": "3.12.0","description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.","engines": {"node": ">=16.0.0"@@ -55,10 +55,10 @@"dependencies": {"@apify/datastructures": "^2.0.0","@apify/log": "^2.4.0",- "@crawlee/browser": "3.11.5",- "@crawlee/browser-pool": "3.11.5",- "@crawlee/types": "3.11.5",- "@crawlee/utils": "3.11.5",+ "@crawlee/browser": "^3.12.0",+ "@crawlee/browser-pool": "^3.12.0",+ "@crawlee/types": "^3.12.0",+ "@crawlee/utils": "^3.12.0","cheerio": "1.0.0-rc.12","devtools-protocol": "*","idcac-playwright": "^0.1.2",diff --git a/packages/templates/CHANGELOG.md b/packages/templates/CHANGELOG.mdindex fc1f9406d..b0965f44c 100644--- a/packages/templates/CHANGELOG.md+++ b/packages/templates/CHANGELOG.md@@ -3,6 +3,14 @@All notable changes to this project will be documented in this file.See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.+# [3.12.0](https://github.com/apify/crawlee/compare/v3.11.5...v3.12.0) (2024-11-04)++**Note:** Version bump only for package @crawlee/templates+++++## [3.11.5](https://github.com/apify/crawlee/compare/v3.11.4...v3.11.5) (2024-10-04)**Note:** Version bump only for package @crawlee/templatesdiff --git a/packages/templates/package.json b/packages/templates/package.jsonindex 81a01a8c3..81eadf690 100644--- a/packages/templates/package.json+++ b/packages/templates/package.json@@ -1,6 +1,6 @@{"name": "@crawlee/templates",- "version": "3.11.5",+ "version": "3.12.0","description": "Templates for the crawlee projects","engines": {"node": ">=16.0.0"diff --git a/packages/types/CHANGELOG.md b/packages/types/CHANGELOG.mdindex 4f160cf0c..7ef0ff143 100644--- a/packages/types/CHANGELOG.md+++ b/packages/types/CHANGELOG.md@@ -3,6 +3,14 @@All notable changes to this project will be documented in this file.See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.+# [3.12.0](https://github.com/apify/crawlee/compare/v3.11.5...v3.12.0) (2024-11-04)++**Note:** Version bump only for package @crawlee/types+++++## [3.11.5](https://github.com/apify/crawlee/compare/v3.11.4...v3.11.5) (2024-10-04)**Note:** Version bump only for package @crawlee/typesdiff --git a/packages/types/package.json b/packages/types/package.jsonindex 31fd96394..7dc878412 100644--- a/packages/types/package.json+++ b/packages/types/package.json@@ -1,6 +1,6 @@{"name": "@crawlee/types",- "version": "3.11.5",+ "version": "3.12.0","description": "Shared types for the crawlee projects","engines": {"node": ">=16.0.0"diff --git a/packages/utils/CHANGELOG.md b/packages/utils/CHANGELOG.mdindex ccc66d059..6086d502b 100644--- a/packages/utils/CHANGELOG.md+++ b/packages/utils/CHANGELOG.md@@ -3,6 +3,22 @@All notable changes to this project will be documented in this file.See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.+# [3.12.0](https://github.com/apify/crawlee/compare/v3.11.5...v3.12.0) (2024-11-04)+++### Bug Fixes++* `.trim()` urls from pretty-printed sitemap.xml files ([#2709](https://github.com/apify/crawlee/issues/2709)) ([802a6fe](https://github.com/apify/crawlee/commit/802a6fea7b2125e2b36d740fc2d5d131de5d53ed)), closes [#2698](https://github.com/apify/crawlee/issues/2698)+++### Features++* allow using other HTTP clients ([#2661](https://github.com/apify/crawlee/issues/2661)) ([568c655](https://github.com/apify/crawlee/commit/568c6556d79ce91654c8a715d1d1729d7d6ed8ef)), closes [#2659](https://github.com/apify/crawlee/issues/2659)+++++## [3.11.5](https://github.com/apify/crawlee/compare/v3.11.4...v3.11.5) (2024-10-04)**Note:** Version bump only for package @crawlee/utilsdiff --git a/packages/utils/package.json b/packages/utils/package.jsonindex 9f37de97f..3d5fb07d9 100644--- a/packages/utils/package.json+++ b/packages/utils/package.json@@ -1,6 +1,6 @@{"name": "@crawlee/utils",- "version": "3.11.5",+ "version": "3.12.0","description": "A set of shared utilities that can be used by crawlers","engines": {"node": ">=16.0.0"@@ -49,7 +49,7 @@"dependencies": {"@apify/log": "^2.4.0","@apify/ps-tree": "^1.2.0",- "@crawlee/types": "3.11.5",+ "@crawlee/types": "^3.12.0","@types/sax": "^1.2.7","cheerio": "1.0.0-rc.12","file-type": "^19.0.0",diff --git a/yarn.lock b/yarn.lockindex 6c5012392..9e813692d 100644--- a/yarn.lock+++ b/yarn.lock@@ -477,16 +477,16 @@ __metadata:languageName: nodelinkType: hard-"@crawlee/basic@npm:3.11.5, @crawlee/basic@workspace:packages/basic-crawler":+"@crawlee/basic@npm:^3.12.0, @crawlee/basic@workspace:packages/basic-crawler":version: 0.0.0-use.localresolution: "@crawlee/basic@workspace:packages/basic-crawler"dependencies:"@apify/log": "npm:^2.4.0""@apify/timeout": "npm:^0.3.0""@apify/utilities": "npm:^2.7.10"- "@crawlee/core": "npm:3.11.5"- "@crawlee/types": "npm:3.11.5"- "@crawlee/utils": "npm:3.11.5"+ "@crawlee/core": "npm:^3.12.0"+ "@crawlee/types": "npm:^3.12.0"+ "@crawlee/utils": "npm:^3.12.0"csv-stringify: "npm:^6.2.0"fs-extra: "npm:^11.0.0"got-scraping: "npm:^4.0.0"@@ -497,14 +497,14 @@ __metadata:languageName: unknownlinkType: soft-"@crawlee/browser-pool@npm:3.11.5, @crawlee/browser-pool@workspace:packages/browser-pool":+"@crawlee/browser-pool@npm:^3.12.0, @crawlee/browser-pool@workspace:packages/browser-pool":version: 0.0.0-use.localresolution: "@crawlee/browser-pool@workspace:packages/browser-pool"dependencies:"@apify/log": "npm:^2.4.0""@apify/timeout": "npm:^0.3.0"- "@crawlee/core": "npm:3.11.5"- "@crawlee/types": "npm:3.11.5"+ "@crawlee/core": "npm:^3.12.0"+ "@crawlee/types": "npm:^3.12.0"fingerprint-generator: "npm:^2.0.6"fingerprint-injector: "npm:^2.0.5"lodash.merge: "npm:^4.6.2"@@ -526,15 +526,15 @@ __metadata:languageName: unknownlinkType: soft-"@crawlee/browser@npm:3.11.5, @crawlee/browser@workspace:packages/browser-crawler":+"@crawlee/browser@npm:^3.12.0, @crawlee/browser@workspace:packages/browser-crawler":version: 0.0.0-use.localresolution: "@crawlee/browser@workspace:packages/browser-crawler"dependencies:"@apify/timeout": "npm:^0.3.0"- "@crawlee/basic": "npm:3.11.5"- "@crawlee/browser-pool": "npm:3.11.5"- "@crawlee/types": "npm:3.11.5"- "@crawlee/utils": "npm:3.11.5"+ "@crawlee/basic": "npm:^3.12.0"+ "@crawlee/browser-pool": "npm:^3.12.0"+ "@crawlee/types": "npm:^3.12.0"+ "@crawlee/utils": "npm:^3.12.0"ow: "npm:^0.28.1"tslib: "npm:^2.4.0"type-fest: "npm:^4.0.0"@@ -549,24 +549,24 @@ __metadata:languageName: unknownlinkType: soft-"@crawlee/cheerio@npm:3.11.5, @crawlee/cheerio@workspace:packages/cheerio-crawler":+"@crawlee/cheerio@npm:^3.12.0, @crawlee/cheerio@workspace:packages/cheerio-crawler":version: 0.0.0-use.localresolution: "@crawlee/cheerio@workspace:packages/cheerio-crawler"dependencies:- "@crawlee/http": "npm:3.11.5"- "@crawlee/types": "npm:3.11.5"- "@crawlee/utils": "npm:3.11.5"+ "@crawlee/http": "npm:^3.12.0"+ "@crawlee/types": "npm:^3.12.0"+ "@crawlee/utils": "npm:^3.12.0"cheerio: "npm:1.0.0-rc.12"htmlparser2: "npm:^9.0.0"tslib: "npm:^2.4.0"languageName: unknownlinkType: soft-"@crawlee/cli@npm:3.11.5, @crawlee/cli@workspace:packages/cli":+"@crawlee/cli@npm:^3.12.0, @crawlee/cli@workspace:packages/cli":version: 0.0.0-use.localresolution: "@crawlee/cli@workspace:packages/cli"dependencies:- "@crawlee/templates": "npm:3.11.5"+ "@crawlee/templates": "npm:^3.12.0"ansi-colors: "npm:^4.1.3"fs-extra: "npm:^11.0.0"inquirer: "npm:^8.2.4"@@ -578,7 +578,7 @@ __metadata:languageName: unknownlinkType: soft-"@crawlee/core@npm:3.11.5, @crawlee/core@npm:^3.9.0, @crawlee/core@workspace:packages/core":+"@crawlee/core@npm:^3.12.0, @crawlee/core@npm:^3.9.0, @crawlee/core@workspace:packages/core":version: 0.0.0-use.localresolution: "@crawlee/core@workspace:packages/core"dependencies:@@ -588,9 +588,9 @@ __metadata:"@apify/pseudo_url": "npm:^2.0.30""@apify/timeout": "npm:^0.3.0""@apify/utilities": "npm:^2.7.10"- "@crawlee/memory-storage": "npm:3.11.5"- "@crawlee/types": "npm:3.11.5"- "@crawlee/utils": "npm:3.11.5"+ "@crawlee/memory-storage": "npm:^3.12.0"+ "@crawlee/types": "npm:^3.12.0"+ "@crawlee/utils": "npm:^3.12.0""@sapphire/async-queue": "npm:^1.5.1""@vladfrangu/async_event_emitter": "npm:^2.2.2"csv-stringify: "npm:^6.2.0"@@ -607,15 +607,15 @@ __metadata:languageName: unknownlinkType: soft-"@crawlee/http@npm:3.11.5, @crawlee/http@workspace:packages/http-crawler":+"@crawlee/http@npm:^3.12.0, @crawlee/http@workspace:packages/http-crawler":version: 0.0.0-use.localresolution: "@crawlee/http@workspace:packages/http-crawler"dependencies:"@apify/timeout": "npm:^0.3.0""@apify/utilities": "npm:^2.7.10"- "@crawlee/basic": "npm:3.11.5"- "@crawlee/types": "npm:3.11.5"- "@crawlee/utils": "npm:3.11.5"+ "@crawlee/basic": "npm:^3.12.0"+ "@crawlee/types": "npm:^3.12.0"+ "@crawlee/utils": "npm:^3.12.0""@types/content-type": "npm:^1.1.5"cheerio: "npm:1.0.0-rc.12"content-type: "npm:^1.0.4"@@ -628,15 +628,15 @@ __metadata:languageName: unknownlinkType: soft-"@crawlee/jsdom@npm:3.11.5, @crawlee/jsdom@workspace:packages/jsdom-crawler":+"@crawlee/jsdom@npm:^3.12.0, @crawlee/jsdom@workspace:packages/jsdom-crawler":version: 0.0.0-use.localresolution: "@crawlee/jsdom@workspace:packages/jsdom-crawler"dependencies:"@apify/timeout": "npm:^0.3.0""@apify/utilities": "npm:^2.7.10"- "@crawlee/http": "npm:3.11.5"- "@crawlee/types": "npm:3.11.5"- "@crawlee/utils": "npm:3.11.5"+ "@crawlee/http": "npm:^3.12.0"+ "@crawlee/types": "npm:^3.12.0"+ "@crawlee/utils": "npm:^3.12.0""@types/jsdom": "npm:^21.0.0"cheerio: "npm:1.0.0-rc.12"jsdom: "npm:^25.0.0"@@ -645,26 +645,26 @@ __metadata:languageName: unknownlinkType: soft-"@crawlee/linkedom@npm:3.11.5, @crawlee/linkedom@workspace:packages/linkedom-crawler":+"@crawlee/linkedom@npm:^3.12.0, @crawlee/linkedom@workspace:packages/linkedom-crawler":version: 0.0.0-use.localresolution: "@crawlee/linkedom@workspace:packages/linkedom-crawler"dependencies:"@apify/timeout": "npm:^0.3.0""@apify/utilities": "npm:^2.7.10"- "@crawlee/http": "npm:3.11.5"- "@crawlee/types": "npm:3.11.5"+ "@crawlee/http": "npm:^3.12.0"+ "@crawlee/types": "npm:^3.12.0"linkedom: "npm:^0.18.0"ow: "npm:^0.28.2"tslib: "npm:^2.4.0"languageName: unknownlinkType: soft-"@crawlee/memory-storage@npm:3.11.5, @crawlee/memory-storage@workspace:packages/memory-storage":+"@crawlee/memory-storage@npm:^3.12.0, @crawlee/memory-storage@workspace:packages/memory-storage":version: 0.0.0-use.localresolution: "@crawlee/memory-storage@workspace:packages/memory-storage"dependencies:"@apify/log": "npm:^2.4.0"- "@crawlee/types": "npm:3.11.5"+ "@crawlee/types": "npm:^3.12.0""@sapphire/async-queue": "npm:^1.5.0""@sapphire/shapeshift": "npm:^3.0.0"content-type: "npm:^1.0.4"@@ -676,18 +676,18 @@ __metadata:languageName: unknownlinkType: soft-"@crawlee/playwright@npm:3.11.5, @crawlee/playwright@workspace:packages/playwright-crawler":+"@crawlee/playwright@npm:^3.12.0, @crawlee/playwright@workspace:packages/playwright-crawler":version: 0.0.0-use.localresolution: "@crawlee/playwright@workspace:packages/playwright-crawler"dependencies:"@apify/datastructures": "npm:^2.0.0""@apify/log": "npm:^2.4.0""@apify/timeout": "npm:^0.3.1"- "@crawlee/browser": "npm:3.11.5"- "@crawlee/browser-pool": "npm:3.11.5"- "@crawlee/core": "npm:3.11.5"- "@crawlee/types": "npm:3.11.5"- "@crawlee/utils": "npm:3.11.5"+ "@crawlee/browser": "npm:^3.12.0"+ "@crawlee/browser-pool": "npm:^3.12.0"+ "@crawlee/core": "npm:^3.12.0"+ "@crawlee/types": "npm:^3.12.0"+ "@crawlee/utils": "npm:^3.12.0"cheerio: "npm:1.0.0-rc.12"idcac-playwright: "npm:^0.1.2"jquery: "npm:^3.6.0"@@ -705,16 +705,16 @@ __metadata:languageName: unknownlinkType: soft-"@crawlee/puppeteer@npm:3.11.5, @crawlee/puppeteer@workspace:packages/puppeteer-crawler":+"@crawlee/puppeteer@npm:^3.12.0, @crawlee/puppeteer@workspace:packages/puppeteer-crawler":version: 0.0.0-use.localresolution: "@crawlee/puppeteer@workspace:packages/puppeteer-crawler"dependencies:"@apify/datastructures": "npm:^2.0.0""@apify/log": "npm:^2.4.0"- "@crawlee/browser": "npm:3.11.5"- "@crawlee/browser-pool": "npm:3.11.5"- "@crawlee/types": "npm:3.11.5"- "@crawlee/utils": "npm:3.11.5"+ "@crawlee/browser": "npm:^3.12.0"+ "@crawlee/browser-pool": "npm:^3.12.0"+ "@crawlee/types": "npm:^3.12.0"+ "@crawlee/utils": "npm:^3.12.0"cheerio: "npm:1.0.0-rc.12"devtools-protocol: "npm:*"idcac-playwright: "npm:^0.1.2"@@ -795,7 +795,7 @@ __metadata:languageName: unknownlinkType: soft-"@crawlee/templates@npm:3.11.5, @crawlee/templates@workspace:packages/templates":+"@crawlee/templates@npm:^3.12.0, @crawlee/templates@workspace:packages/templates":version: 0.0.0-use.localresolution: "@crawlee/templates@workspace:packages/templates"dependencies:@@ -807,7 +807,7 @@ __metadata:languageName: unknownlinkType: soft-"@crawlee/types@npm:3.11.5, @crawlee/types@npm:^3.3.0, @crawlee/types@npm:^3.9.0, @crawlee/types@workspace:packages/types":+"@crawlee/types@npm:^3.12.0, @crawlee/types@npm:^3.3.0, @crawlee/types@npm:^3.9.0, @crawlee/types@workspace:packages/types":version: 0.0.0-use.localresolution: "@crawlee/types@workspace:packages/types"dependencies:@@ -815,13 +815,13 @@ __metadata:languageName: unknownlinkType: soft-"@crawlee/utils@npm:3.11.5, @crawlee/utils@npm:^3.9.0, @crawlee/utils@workspace:packages/utils":+"@crawlee/utils@npm:^3.12.0, @crawlee/utils@npm:^3.9.0, @crawlee/utils@workspace:packages/utils":version: 0.0.0-use.localresolution: "@crawlee/utils@workspace:packages/utils"dependencies:"@apify/log": "npm:^2.4.0""@apify/ps-tree": "npm:^1.2.0"- "@crawlee/types": "npm:3.11.5"+ "@crawlee/types": "npm:^3.12.0""@types/sax": "npm:^1.2.7""@types/whatwg-mimetype": "npm:^3.0.2"cheerio: "npm:1.0.0-rc.12"@@ -4414,18 +4414,18 @@ __metadata:version: 0.0.0-use.localresolution: "crawlee@workspace:packages/crawlee"dependencies:- "@crawlee/basic": "npm:3.11.5"- "@crawlee/browser": "npm:3.11.5"- "@crawlee/browser-pool": "npm:3.11.5"- "@crawlee/cheerio": "npm:3.11.5"- "@crawlee/cli": "npm:3.11.5"- "@crawlee/core": "npm:3.11.5"- "@crawlee/http": "npm:3.11.5"- "@crawlee/jsdom": "npm:3.11.5"- "@crawlee/linkedom": "npm:3.11.5"- "@crawlee/playwright": "npm:3.11.5"- "@crawlee/puppeteer": "npm:3.11.5"- "@crawlee/utils": "npm:3.11.5"+ "@crawlee/basic": "npm:^3.12.0"+ "@crawlee/browser": "npm:^3.12.0"+ "@crawlee/browser-pool": "npm:^3.12.0"+ "@crawlee/cheerio": "npm:^3.12.0"+ "@crawlee/cli": "npm:^3.12.0"+ "@crawlee/core": "npm:^3.12.0"+ "@crawlee/http": "npm:^3.12.0"+ "@crawlee/jsdom": "npm:^3.12.0"+ "@crawlee/linkedom": "npm:^3.12.0"+ "@crawlee/playwright": "npm:^3.12.0"+ "@crawlee/puppeteer": "npm:^3.12.0"+ "@crawlee/utils": "npm:^3.12.0"import-local: "npm:^3.1.0"tslib: "npm:^2.4.0"peerDependencies: