Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
chore: require node 15+ due to http2 issues in older versions (#1099)
Martin Adámeke2a59384b53e
diff --git a/.github/workflows/check-ts-support.yml b/.github/workflows/check-ts-support.ymlindex 088e5de82..81ad28c50 100644--- a/.github/workflows/check-ts-support.yml+++ b/.github/workflows/check-ts-support.yml@@ -15,7 +15,7 @@ jobs:- uses: actions/checkout@v2- uses: actions/setup-node@v1with:- node-version: 14+ node-version: 16registry-url: https://registry.npmjs.org/- name: Check TS supportrun: |diff --git a/.github/workflows/check.yml b/.github/workflows/check.ymlindex 5c68461ba..cbac74930 100644--- a/.github/workflows/check.yml+++ b/.github/workflows/check.yml@@ -16,7 +16,7 @@ jobs:fail-fast: falsematrix:os: [ubuntu-latest, windows-latest]- node-version: [ 10, 12, 14, 15, 16 ]+ node-version: [ 15, 16 ]steps:- uses: actions/checkout@v2diff --git a/.github/workflows/release.yml b/.github/workflows/release.ymlindex 0cb2f7a4f..7ab793703 100644--- a/.github/workflows/release.yml+++ b/.github/workflows/release.yml@@ -19,7 +19,7 @@ jobs:strategy:matrix:os: [ubuntu-latest, windows-latest]- node-version: [10, 12, 14, 15, 16]+ node-version: [15, 16]steps:-@@ -50,10 +50,10 @@ jobs:-uses: actions/checkout@v2-- name: Use Node.js 14+ name: Use Node.js 16uses: actions/setup-node@v1with:- node-version: 14+ node-version: 16-name: Install Dependenciesrun: npm install@@ -74,7 +74,7 @@ jobs:-uses: actions/setup-node@v1with:- node-version: 14+ node-version: 16registry-url: https://registry.npmjs.org/-name: Install Dependenciesdiff --git a/CHANGELOG.md b/CHANGELOG.mdindex 991538ff3..7b21764a0 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -1,8 +1,12 @@+2.0.0 / BETA+====================+- **BREAKING**: Require Node.js >=15.10.0 because HTTP2 support on lower Node.js versions is very buggy.+1.3.1 / 2021/07/13====================- Fix client `/v2` duplication in `apiBaseUrl`.--1.3.0 / BETA++1.3.0 / 2021/07/11====================## Navigation hooks in `CheerioCrawler`diff --git a/README.md b/README.mdindex 6000e7b2b..fe869631b 100644--- a/README.md+++ b/README.md@@ -89,7 +89,7 @@ If you want to learn more, proceed to the [Getting Started](https://sdk.apify.cotutorial that will take you step by step through creating your first scraper.### Local stand-alone usage-Apify SDK requires [Node.js](https://nodejs.org/en/) 10.17 or later, with the exception of Node.js 11.+Apify SDK requires [Node.js](https://nodejs.org/en/) 15.10 or later.Add Apify SDK to any Node.js project by running:```bashdiff --git a/package.json b/package.jsonindex fbf2264fd..3fdc00692 100644--- a/package.json+++ b/package.json@@ -1,9 +1,9 @@{"name": "apify",- "version": "1.3.1",+ "version": "2.0.0","description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.","engines": {- "node": ">=10.17.0"+ "node": ">=15.10.0"},"main": "build/index.js","types": "types/index.d.ts",@@ -53,33 +53,33 @@"dependencies": {"@apify/consts": "^1.1.3","@apify/datastructures": "^1.0.1",- "@apify/log": "^1.0.5",+ "@apify/log": "^1.1.1","@apify/ps-tree": "^1.1.4","@apify/storage-local": "^2.0.1",- "@apify/utilities": "^1.0.6",- "@types/cheerio": "^0.22.28",- "@types/domhandler": "^2.4.1",- "@types/node": "^14",+ "@apify/utilities": "^1.1.2",+ "@types/cheerio": "^0.22.30",+ "@types/domhandler": "^2.4.2",+ "@types/node": "^15.14.2","@types/socket.io": "^2.1.13",- "@types/tough-cookie": "^4.0.0",- "apify-client": "^1.2.4",+ "@types/tough-cookie": "^4.0.1",+ "apify-client": "^1.3.0","browser-pool": "^1.2.0-beta.1","cheerio": "1.0.0-rc.3","content-type": "^1.0.4","express": "^4.17.1",- "fs-extra": "^9.1.0",- "got-scraping": "^1.0.2",- "htmlparser2": "^6.0.0",- "iconv-lite": "^0.6.2",- "jquery": "^3.5.1",- "mime-types": "^2.1.27",- "ow": "^0.23.0",- "semver": "^7.3.2",- "socket.io": "^3.0.3",- "stream-json": "^1.7.1",+ "fs-extra": "^10.0.0",+ "got-scraping": "^2.0.1",+ "htmlparser2": "^6.1.0",+ "iconv-lite": "^0.6.3",+ "jquery": "^3.6.0",+ "mime-types": "^2.1.31",+ "ow": "^0.27.0",+ "semver": "^7.3.5",+ "socket.io": "^3.1.2",+ "stream-json": "^1.7.2","tough-cookie": "^4.0.0","underscore": "^1.13.0",- "ws": "^7.4.6"+ "ws": "^7.5.3"},"peerDependencies": {"playwright": "^1.11.0",@@ -94,43 +94,43 @@}},"devDependencies": {- "@apify/eslint-config": "^0.1.3",- "@babel/cli": "^7.11.6",- "@babel/core": "^7.11.6",- "@babel/eslint-parser": "^7.12.1",- "@babel/plugin-syntax-class-properties": "^7.12.1",- "@babel/preset-env": "^7.11.5",- "@babel/register": "^7.11.5",- "@types/content-type": "^1.1.3",- "@types/fs-extra": "^9.0.8",- "@types/htmlparser2": "^3.10.2",- "@types/jest": "^26.0.21",- "@types/request-promise-native": "^1.0.17",- "@types/rimraf": "^3.0.0",- "@types/semver": "^7.3.4",+ "@apify/eslint-config": "^0.1.4",+ "@babel/cli": "^7.14.8",+ "@babel/core": "^7.14.8",+ "@babel/eslint-parser": "^7.14.7",+ "@babel/plugin-syntax-class-properties": "^7.12.13",+ "@babel/preset-env": "^7.14.8",+ "@babel/register": "^7.14.5",+ "@types/content-type": "^1.1.5",+ "@types/fs-extra": "^9.0.12",+ "@types/htmlparser2": "^3.10.3",+ "@types/jest": "^26.0.24",+ "@types/request-promise-native": "^1.0.18",+ "@types/rimraf": "^3.0.1",+ "@types/semver": "^7.3.7","@types/sinon": "^9.0.11",- "@types/underscore": "^1.11.0",- "@types/ws": "^7.4.0",+ "@types/underscore": "^1.11.3",+ "@types/ws": "^7.4.7","basic-auth-parser": "^0.0.2","body-parser": "^1.19.0",- "csv-parse": "^4.12.0",- "eslint": "^7.11.0",+ "csv-parse": "^4.16.0",+ "eslint": "^7.31.0","fpcollect": "^1.0.4","fpscanner": "^0.1.5","jest": "^26.6.3","jest-circus": "^26.6.3","playwright": "1.11.1","portastic": "^1.0.1",- "prettier": "^2.1.2",+ "prettier": "^2.3.2","proxy": "^1.0.2","puppeteer": "9.1.1","rimraf": "^3.0.2","sinon": "^9.2.0","sinon-stub-promise": "^4.0.0","socket.io-client": "^3.0.3",- "ts-jest": "^26.5.4",- "typescript": "4.2.3",- "ua-parser-js": "^0.7.22",+ "ts-jest": "^26.5.6",+ "typescript": "4.3.5",+ "ua-parser-js": "^0.7.28","why-is-node-running": "^2.2.0"}}diff --git a/test/crawlers/browser_crawler.test.js b/test/crawlers/browser_crawler.test.jsindex 167c00f44..aac942477 100644--- a/test/crawlers/browser_crawler.test.js+++ b/test/crawlers/browser_crawler.test.js@@ -401,6 +401,7 @@ describe('BrowserCrawler', () => {loadedCookies.forEach((cookie) => {// TODO this test is flaky in CI and we need some more info to debug why.if (cookie !== 'TEST=12321312312') {+ // for some reason, the CI failures report the first cookie to be just empty stringconsole.log('loadedCookies:');console.dir(loadedCookies);}