Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
chore(release): v3.1.4
Apify Release Bot4d36bb7e540c
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 7a7dce5ad..cfd4c94b1 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -3,6 +3,17 @@All notable changes to this project will be documented in this file.See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.+## [3.1.4](https://github.com/apify/crawlee/compare/v3.1.3...v3.1.4) (2022-12-14)+++### Bug Fixes++* session.markBad() on requestHandler error ([#1709](https://github.com/apify/crawlee/issues/1709)) ([e87eb1f](https://github.com/apify/crawlee/commit/e87eb1f2ccd9585f8d53cb03ec671cedf23a06b4)), closes [#1635](https://github.com/apify/crawlee/issues/1635) [/github.com/apify/crawlee/blob/5ff04faa85c3a6b6f02cd58a91b46b80610d8ae6/packages/browser-crawler/src/internals/browser-crawler.ts#L524](https://github.com//github.com/apify/crawlee/blob/5ff04faa85c3a6b6f02cd58a91b46b80610d8ae6/packages/browser-crawler/src/internals/browser-crawler.ts/issues/L524)+++++## [3.1.3](https://github.com/apify/crawlee/compare/v3.1.2...v3.1.3) (2022-12-07)diff --git a/lerna.json b/lerna.jsonindex 2abcf9a95..e10feb8e2 100644--- a/lerna.json+++ b/lerna.json@@ -2,7 +2,7 @@"packages": ["packages/*"],- "version": "3.1.3",+ "version": "3.1.4","command": {"version": {"conventionalCommits": true,diff --git a/package-lock.json b/package-lock.jsonindex aa6208ab1..29a333709 100644--- a/package-lock.json+++ b/package-lock.json@@ -16570,7 +16570,7 @@},"packages/basic-crawler": {"name": "@crawlee/basic",- "version": "3.1.3",+ "version": "3.1.4","license": "Apache-2.0","dependencies": {"@apify/log": "^2.0.0",@@ -16589,11 +16589,11 @@},"packages/browser-crawler": {"name": "@crawlee/browser",- "version": "3.1.3",+ "version": "3.1.4","license": "Apache-2.0","dependencies": {"@apify/timeout": "^0.3.0",- "@crawlee/basic": "^3.1.3",+ "@crawlee/basic": "^3.1.4","@crawlee/browser-pool": "^3.1.3","@crawlee/types": "^3.1.2","@crawlee/utils": "^3.1.2",@@ -16640,10 +16640,10 @@},"packages/cheerio-crawler": {"name": "@crawlee/cheerio",- "version": "3.1.3",+ "version": "3.1.4","license": "Apache-2.0","dependencies": {- "@crawlee/http": "^3.1.3",+ "@crawlee/http": "^3.1.4","@crawlee/types": "^3.1.2","cheerio": "1.0.0-rc.12","htmlparser2": "^8.0.1"@@ -16722,18 +16722,18 @@}},"packages/crawlee": {- "version": "3.1.3",+ "version": "3.1.4","license": "Apache-2.0","dependencies": {- "@crawlee/basic": "^3.1.3",- "@crawlee/browser": "^3.1.3",- "@crawlee/cheerio": "^3.1.3",+ "@crawlee/basic": "^3.1.4",+ "@crawlee/browser": "^3.1.4",+ "@crawlee/cheerio": "^3.1.4","@crawlee/cli": "^3.1.2","@crawlee/core": "^3.1.3",- "@crawlee/http": "^3.1.3",- "@crawlee/jsdom": "^3.1.3",- "@crawlee/playwright": "^3.1.3",- "@crawlee/puppeteer": "^3.1.3",+ "@crawlee/http": "^3.1.4",+ "@crawlee/jsdom": "^3.1.4",+ "@crawlee/playwright": "^3.1.4",+ "@crawlee/puppeteer": "^3.1.4","@crawlee/utils": "^3.1.2","import-local": "^3.1.0"},@@ -16758,12 +16758,12 @@},"packages/http-crawler": {"name": "@crawlee/http",- "version": "3.1.3",+ "version": "3.1.4","license": "Apache-2.0","dependencies": {"@apify/timeout": "^0.3.0","@apify/utilities": "^2.0.0",- "@crawlee/basic": "^3.1.3",+ "@crawlee/basic": "^3.1.4","@crawlee/types": "^3.1.2","@types/content-type": "^1.1.5","content-type": "^1.0.4",@@ -16790,11 +16790,11 @@},"packages/jsdom-crawler": {"name": "@crawlee/jsdom",- "version": "3.1.3",+ "version": "3.1.4","license": "Apache-2.0","dependencies": {"@apify/utilities": "^2.1.4",- "@crawlee/http": "^3.1.3",+ "@crawlee/http": "^3.1.4","@crawlee/types": "^3.1.2","@types/jsdom": "^20.0.0","jsdom": "^20.0.0",@@ -16826,12 +16826,12 @@},"packages/playwright-crawler": {"name": "@crawlee/playwright",- "version": "3.1.3",+ "version": "3.1.4","license": "Apache-2.0","dependencies": {"@apify/datastructures": "^2.0.0","@apify/log": "^2.0.0",- "@crawlee/browser": "^3.1.3",+ "@crawlee/browser": "^3.1.4","@crawlee/browser-pool": "^3.1.3","@crawlee/types": "^3.1.2","@crawlee/utils": "^3.1.2",@@ -16853,12 +16853,12 @@},"packages/puppeteer-crawler": {"name": "@crawlee/puppeteer",- "version": "3.1.3",+ "version": "3.1.4","license": "Apache-2.0","dependencies": {"@apify/datastructures": "^2.0.0","@apify/log": "^2.0.0",- "@crawlee/browser": "^3.1.3",+ "@crawlee/browser": "^3.1.4","@crawlee/browser-pool": "^3.1.3","@crawlee/types": "^3.1.2","@crawlee/utils": "^3.1.2",diff --git a/packages/basic-crawler/CHANGELOG.md b/packages/basic-crawler/CHANGELOG.mdindex 66c580c74..6862d7095 100644--- a/packages/basic-crawler/CHANGELOG.md+++ b/packages/basic-crawler/CHANGELOG.md@@ -3,6 +3,17 @@All notable changes to this project will be documented in this file.See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.+## [3.1.4](https://github.com/apify/crawlee/compare/v3.1.3...v3.1.4) (2022-12-14)+++### Bug Fixes++* session.markBad() on requestHandler error ([#1709](https://github.com/apify/crawlee/issues/1709)) ([e87eb1f](https://github.com/apify/crawlee/commit/e87eb1f2ccd9585f8d53cb03ec671cedf23a06b4)), closes [#1635](https://github.com/apify/crawlee/issues/1635) [/github.com/apify/crawlee/blob/5ff04faa85c3a6b6f02cd58a91b46b80610d8ae6/packages/browser-crawler/src/internals/browser-crawler.ts#L524](https://github.com//github.com/apify/crawlee/blob/5ff04faa85c3a6b6f02cd58a91b46b80610d8ae6/packages/browser-crawler/src/internals/browser-crawler.ts/issues/L524)+++++## [3.1.3](https://github.com/apify/crawlee/compare/v3.1.2...v3.1.3) (2022-12-07)diff --git a/packages/basic-crawler/package.json b/packages/basic-crawler/package.jsonindex 4a7134330..73ab5d27a 100644--- a/packages/basic-crawler/package.json+++ b/packages/basic-crawler/package.json@@ -1,6 +1,6 @@{"name": "@crawlee/basic",- "version": "3.1.3",+ "version": "3.1.4","description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.","engines": {"node": ">=16.0.0"diff --git a/packages/browser-crawler/CHANGELOG.md b/packages/browser-crawler/CHANGELOG.mdindex 8a0215dcb..edb668ff6 100644--- a/packages/browser-crawler/CHANGELOG.md+++ b/packages/browser-crawler/CHANGELOG.md@@ -3,6 +3,14 @@All notable changes to this project will be documented in this file.See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.+## [3.1.4](https://github.com/apify/crawlee/compare/v3.1.3...v3.1.4) (2022-12-14)++**Note:** Version bump only for package @crawlee/browser+++++## [3.1.3](https://github.com/apify/crawlee/compare/v3.1.2...v3.1.3) (2022-12-07)**Note:** Version bump only for package @crawlee/browserdiff --git a/packages/browser-crawler/package.json b/packages/browser-crawler/package.jsonindex ac7f7fecf..9fb69f68d 100644--- a/packages/browser-crawler/package.json+++ b/packages/browser-crawler/package.json@@ -1,6 +1,6 @@{"name": "@crawlee/browser",- "version": "3.1.3",+ "version": "3.1.4","description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.","engines": {"node": ">=16.0.0"@@ -54,7 +54,7 @@},"dependencies": {"@apify/timeout": "^0.3.0",- "@crawlee/basic": "^3.1.3",+ "@crawlee/basic": "^3.1.4","@crawlee/browser-pool": "^3.1.3","@crawlee/types": "^3.1.2","@crawlee/utils": "^3.1.2",diff --git a/packages/cheerio-crawler/CHANGELOG.md b/packages/cheerio-crawler/CHANGELOG.mdindex f679eddcc..e2af7ab36 100644--- a/packages/cheerio-crawler/CHANGELOG.md+++ b/packages/cheerio-crawler/CHANGELOG.md@@ -3,6 +3,14 @@All notable changes to this project will be documented in this file.See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.+## [3.1.4](https://github.com/apify/crawlee/compare/v3.1.3...v3.1.4) (2022-12-14)++**Note:** Version bump only for package @crawlee/cheerio+++++## [3.1.3](https://github.com/apify/crawlee/compare/v3.1.2...v3.1.3) (2022-12-07)**Note:** Version bump only for package @crawlee/cheeriodiff --git a/packages/cheerio-crawler/package.json b/packages/cheerio-crawler/package.jsonindex f09ef2958..a102790b3 100644--- a/packages/cheerio-crawler/package.json+++ b/packages/cheerio-crawler/package.json@@ -1,6 +1,6 @@{"name": "@crawlee/cheerio",- "version": "3.1.3",+ "version": "3.1.4","description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.","engines": {"node": ">=16.0.0"@@ -53,7 +53,7 @@"access": "public"},"dependencies": {- "@crawlee/http": "^3.1.3",+ "@crawlee/http": "^3.1.4","@crawlee/types": "^3.1.2","cheerio": "1.0.0-rc.12","htmlparser2": "^8.0.1"diff --git a/packages/crawlee/CHANGELOG.md b/packages/crawlee/CHANGELOG.mdindex 879da2dd9..4d55cf57a 100644--- a/packages/crawlee/CHANGELOG.md+++ b/packages/crawlee/CHANGELOG.md@@ -3,6 +3,14 @@All notable changes to this project will be documented in this file.See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.+## [3.1.4](https://github.com/apify/crawlee/compare/v3.1.3...v3.1.4) (2022-12-14)++**Note:** Version bump only for package crawlee+++++## [3.1.3](https://github.com/apify/crawlee/compare/v3.1.2...v3.1.3) (2022-12-07)**Note:** Version bump only for package crawleediff --git a/packages/crawlee/package.json b/packages/crawlee/package.jsonindex 830b4813a..e4792c8fd 100644--- a/packages/crawlee/package.json+++ b/packages/crawlee/package.json@@ -1,6 +1,6 @@{"name": "crawlee",- "version": "3.1.3",+ "version": "3.1.4","description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.","engines": {"node": ">=16.0.0"@@ -56,15 +56,15 @@"access": "public"},"dependencies": {- "@crawlee/basic": "^3.1.3",- "@crawlee/browser": "^3.1.3",- "@crawlee/cheerio": "^3.1.3",+ "@crawlee/basic": "^3.1.4",+ "@crawlee/browser": "^3.1.4",+ "@crawlee/cheerio": "^3.1.4","@crawlee/cli": "^3.1.2","@crawlee/core": "^3.1.3",- "@crawlee/http": "^3.1.3",- "@crawlee/jsdom": "^3.1.3",- "@crawlee/playwright": "^3.1.3",- "@crawlee/puppeteer": "^3.1.3",+ "@crawlee/http": "^3.1.4",+ "@crawlee/jsdom": "^3.1.4",+ "@crawlee/playwright": "^3.1.4",+ "@crawlee/puppeteer": "^3.1.4","@crawlee/utils": "^3.1.2","import-local": "^3.1.0"},diff --git a/packages/http-crawler/CHANGELOG.md b/packages/http-crawler/CHANGELOG.mdindex d892d518b..102d27008 100644--- a/packages/http-crawler/CHANGELOG.md+++ b/packages/http-crawler/CHANGELOG.md@@ -3,6 +3,14 @@All notable changes to this project will be documented in this file.See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.+## [3.1.4](https://github.com/apify/crawlee/compare/v3.1.3...v3.1.4) (2022-12-14)++**Note:** Version bump only for package @crawlee/http+++++## [3.1.3](https://github.com/apify/crawlee/compare/v3.1.2...v3.1.3) (2022-12-07)**Note:** Version bump only for package @crawlee/httpdiff --git a/packages/http-crawler/package.json b/packages/http-crawler/package.jsonindex f19d43353..01b624ed9 100644--- a/packages/http-crawler/package.json+++ b/packages/http-crawler/package.json@@ -1,6 +1,6 @@{"name": "@crawlee/http",- "version": "3.1.3",+ "version": "3.1.4","description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.","engines": {"node": ">=16.0.0"@@ -55,7 +55,7 @@"dependencies": {"@apify/timeout": "^0.3.0","@apify/utilities": "^2.0.0",- "@crawlee/basic": "^3.1.3",+ "@crawlee/basic": "^3.1.4","@crawlee/types": "^3.1.2","@types/content-type": "^1.1.5","content-type": "^1.0.4",diff --git a/packages/jsdom-crawler/CHANGELOG.md b/packages/jsdom-crawler/CHANGELOG.mdindex 05d768581..2aa049e1c 100644--- a/packages/jsdom-crawler/CHANGELOG.md+++ b/packages/jsdom-crawler/CHANGELOG.md@@ -3,6 +3,14 @@All notable changes to this project will be documented in this file.See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.+## [3.1.4](https://github.com/apify/crawlee/compare/v3.1.3...v3.1.4) (2022-12-14)++**Note:** Version bump only for package @crawlee/jsdom+++++## [3.1.3](https://github.com/apify/crawlee/compare/v3.1.2...v3.1.3) (2022-12-07)diff --git a/packages/jsdom-crawler/package.json b/packages/jsdom-crawler/package.jsonindex eb5e40669..43d051a58 100644--- a/packages/jsdom-crawler/package.json+++ b/packages/jsdom-crawler/package.json@@ -1,6 +1,6 @@{"name": "@crawlee/jsdom",- "version": "3.1.3",+ "version": "3.1.4","description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.","engines": {"node": ">=16.0.0"@@ -54,7 +54,7 @@},"dependencies": {"@apify/utilities": "^2.1.4",- "@crawlee/http": "^3.1.3",+ "@crawlee/http": "^3.1.4","@crawlee/types": "^3.1.2","@types/jsdom": "^20.0.0","jsdom": "^20.0.0",diff --git a/packages/playwright-crawler/CHANGELOG.md b/packages/playwright-crawler/CHANGELOG.mdindex a15eddd30..81f98e22e 100644--- a/packages/playwright-crawler/CHANGELOG.md+++ b/packages/playwright-crawler/CHANGELOG.md@@ -3,6 +3,14 @@All notable changes to this project will be documented in this file.See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.+## [3.1.4](https://github.com/apify/crawlee/compare/v3.1.3...v3.1.4) (2022-12-14)++**Note:** Version bump only for package @crawlee/playwright+++++## [3.1.3](https://github.com/apify/crawlee/compare/v3.1.2...v3.1.3) (2022-12-07)**Note:** Version bump only for package @crawlee/playwrightdiff --git a/packages/playwright-crawler/package.json b/packages/playwright-crawler/package.jsonindex b8b01cd3a..13a93d312 100644--- a/packages/playwright-crawler/package.json+++ b/packages/playwright-crawler/package.json@@ -1,6 +1,6 @@{"name": "@crawlee/playwright",- "version": "3.1.3",+ "version": "3.1.4","description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.","engines": {"node": ">=16.0.0"@@ -55,7 +55,7 @@"dependencies": {"@apify/datastructures": "^2.0.0","@apify/log": "^2.0.0",- "@crawlee/browser": "^3.1.3",+ "@crawlee/browser": "^3.1.4","@crawlee/browser-pool": "^3.1.3","@crawlee/types": "^3.1.2","@crawlee/utils": "^3.1.2",diff --git a/packages/puppeteer-crawler/CHANGELOG.md b/packages/puppeteer-crawler/CHANGELOG.mdindex fa5728925..483e28e85 100644--- a/packages/puppeteer-crawler/CHANGELOG.md+++ b/packages/puppeteer-crawler/CHANGELOG.md@@ -3,6 +3,14 @@All notable changes to this project will be documented in this file.See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.+## [3.1.4](https://github.com/apify/crawlee/compare/v3.1.3...v3.1.4) (2022-12-14)++**Note:** Version bump only for package @crawlee/puppeteer+++++## [3.1.3](https://github.com/apify/crawlee/compare/v3.1.2...v3.1.3) (2022-12-07)**Note:** Version bump only for package @crawlee/puppeteerdiff --git a/packages/puppeteer-crawler/package.json b/packages/puppeteer-crawler/package.jsonindex 14990e48a..3a4f31ca7 100644--- a/packages/puppeteer-crawler/package.json+++ b/packages/puppeteer-crawler/package.json@@ -1,6 +1,6 @@{"name": "@crawlee/puppeteer",- "version": "3.1.3",+ "version": "3.1.4","description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.","engines": {"node": ">=16.0.0"@@ -55,7 +55,7 @@"dependencies": {"@apify/datastructures": "^2.0.0","@apify/log": "^2.0.0",- "@crawlee/browser": "^3.1.3",+ "@crawlee/browser": "^3.1.4","@crawlee/browser-pool": "^3.1.3","@crawlee/types": "^3.1.2","@crawlee/utils": "^3.1.2",