Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

chore: require node 15+ due to http2 issues in older versions (#1099)

Martin Adámeke2a59384b53e
diff --git a/.github/workflows/check-ts-support.yml b/.github/workflows/check-ts-support.yml
index 088e5de82..81ad28c50 100644
--- a/.github/workflows/check-ts-support.yml
+++ b/.github/workflows/check-ts-support.yml
@@ -15,7 +15,7 @@ jobs:
- uses: actions/checkout@v2
- uses: actions/setup-node@v1
with:
- node-version: 14
+ node-version: 16
registry-url: https://registry.npmjs.org/
- name: Check TS support
run: |
diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml
index 5c68461ba..cbac74930 100644
--- a/.github/workflows/check.yml
+++ b/.github/workflows/check.yml
@@ -16,7 +16,7 @@ jobs:
fail-fast: false
matrix:
os: [ubuntu-latest, windows-latest]
- node-version: [ 10, 12, 14, 15, 16 ]
+ node-version: [ 15, 16 ]
steps:
- uses: actions/checkout@v2
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index 0cb2f7a4f..7ab793703 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -19,7 +19,7 @@ jobs:
strategy:
matrix:
os: [ubuntu-latest, windows-latest]
- node-version: [10, 12, 14, 15, 16]
+ node-version: [15, 16]
steps:
-
@@ -50,10 +50,10 @@ jobs:
-
uses: actions/checkout@v2
-
- name: Use Node.js 14
+ name: Use Node.js 16
uses: actions/setup-node@v1
with:
- node-version: 14
+ node-version: 16
-
name: Install Dependencies
run: npm install
@@ -74,7 +74,7 @@ jobs:
-
uses: actions/setup-node@v1
with:
- node-version: 14
+ node-version: 16
registry-url: https://registry.npmjs.org/
-
name: Install Dependencies
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 991538ff3..7b21764a0 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,8 +1,12 @@
+2.0.0 / BETA
+====================
+- **BREAKING**: Require Node.js >=15.10.0 because HTTP2 support on lower Node.js versions is very buggy.
+
1.3.1 / 2021/07/13
====================
- Fix client `/v2` duplication in `apiBaseUrl`.
-
-1.3.0 / BETA
+
+1.3.0 / 2021/07/11
====================
## Navigation hooks in `CheerioCrawler`
diff --git a/README.md b/README.md
index 6000e7b2b..fe869631b 100644
--- a/README.md
+++ b/README.md
@@ -89,7 +89,7 @@ If you want to learn more, proceed to the [Getting Started](https://sdk.apify.co
tutorial that will take you step by step through creating your first scraper.
### Local stand-alone usage
-Apify SDK requires [Node.js](https://nodejs.org/en/) 10.17 or later, with the exception of Node.js 11.
+Apify SDK requires [Node.js](https://nodejs.org/en/) 15.10 or later.
Add Apify SDK to any Node.js project by running:
```bash
diff --git a/package.json b/package.json
index fbf2264fd..3fdc00692 100644
--- a/package.json
+++ b/package.json
@@ -1,9 +1,9 @@
{
"name": "apify",
- "version": "1.3.1",
+ "version": "2.0.0",
"description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
"engines": {
- "node": ">=10.17.0"
+ "node": ">=15.10.0"
},
"main": "build/index.js",
"types": "types/index.d.ts",
@@ -53,33 +53,33 @@
"dependencies": {
"@apify/consts": "^1.1.3",
"@apify/datastructures": "^1.0.1",
- "@apify/log": "^1.0.5",
+ "@apify/log": "^1.1.1",
"@apify/ps-tree": "^1.1.4",
"@apify/storage-local": "^2.0.1",
- "@apify/utilities": "^1.0.6",
- "@types/cheerio": "^0.22.28",
- "@types/domhandler": "^2.4.1",
- "@types/node": "^14",
+ "@apify/utilities": "^1.1.2",
+ "@types/cheerio": "^0.22.30",
+ "@types/domhandler": "^2.4.2",
+ "@types/node": "^15.14.2",
"@types/socket.io": "^2.1.13",
- "@types/tough-cookie": "^4.0.0",
- "apify-client": "^1.2.4",
+ "@types/tough-cookie": "^4.0.1",
+ "apify-client": "^1.3.0",
"browser-pool": "^1.2.0-beta.1",
"cheerio": "1.0.0-rc.3",
"content-type": "^1.0.4",
"express": "^4.17.1",
- "fs-extra": "^9.1.0",
- "got-scraping": "^1.0.2",
- "htmlparser2": "^6.0.0",
- "iconv-lite": "^0.6.2",
- "jquery": "^3.5.1",
- "mime-types": "^2.1.27",
- "ow": "^0.23.0",
- "semver": "^7.3.2",
- "socket.io": "^3.0.3",
- "stream-json": "^1.7.1",
+ "fs-extra": "^10.0.0",
+ "got-scraping": "^2.0.1",
+ "htmlparser2": "^6.1.0",
+ "iconv-lite": "^0.6.3",
+ "jquery": "^3.6.0",
+ "mime-types": "^2.1.31",
+ "ow": "^0.27.0",
+ "semver": "^7.3.5",
+ "socket.io": "^3.1.2",
+ "stream-json": "^1.7.2",
"tough-cookie": "^4.0.0",
"underscore": "^1.13.0",
- "ws": "^7.4.6"
+ "ws": "^7.5.3"
},
"peerDependencies": {
"playwright": "^1.11.0",
@@ -94,43 +94,43 @@
}
},
"devDependencies": {
- "@apify/eslint-config": "^0.1.3",
- "@babel/cli": "^7.11.6",
- "@babel/core": "^7.11.6",
- "@babel/eslint-parser": "^7.12.1",
- "@babel/plugin-syntax-class-properties": "^7.12.1",
- "@babel/preset-env": "^7.11.5",
- "@babel/register": "^7.11.5",
- "@types/content-type": "^1.1.3",
- "@types/fs-extra": "^9.0.8",
- "@types/htmlparser2": "^3.10.2",
- "@types/jest": "^26.0.21",
- "@types/request-promise-native": "^1.0.17",
- "@types/rimraf": "^3.0.0",
- "@types/semver": "^7.3.4",
+ "@apify/eslint-config": "^0.1.4",
+ "@babel/cli": "^7.14.8",
+ "@babel/core": "^7.14.8",
+ "@babel/eslint-parser": "^7.14.7",
+ "@babel/plugin-syntax-class-properties": "^7.12.13",
+ "@babel/preset-env": "^7.14.8",
+ "@babel/register": "^7.14.5",
+ "@types/content-type": "^1.1.5",
+ "@types/fs-extra": "^9.0.12",
+ "@types/htmlparser2": "^3.10.3",
+ "@types/jest": "^26.0.24",
+ "@types/request-promise-native": "^1.0.18",
+ "@types/rimraf": "^3.0.1",
+ "@types/semver": "^7.3.7",
"@types/sinon": "^9.0.11",
- "@types/underscore": "^1.11.0",
- "@types/ws": "^7.4.0",
+ "@types/underscore": "^1.11.3",
+ "@types/ws": "^7.4.7",
"basic-auth-parser": "^0.0.2",
"body-parser": "^1.19.0",
- "csv-parse": "^4.12.0",
- "eslint": "^7.11.0",
+ "csv-parse": "^4.16.0",
+ "eslint": "^7.31.0",
"fpcollect": "^1.0.4",
"fpscanner": "^0.1.5",
"jest": "^26.6.3",
"jest-circus": "^26.6.3",
"playwright": "1.11.1",
"portastic": "^1.0.1",
- "prettier": "^2.1.2",
+ "prettier": "^2.3.2",
"proxy": "^1.0.2",
"puppeteer": "9.1.1",
"rimraf": "^3.0.2",
"sinon": "^9.2.0",
"sinon-stub-promise": "^4.0.0",
"socket.io-client": "^3.0.3",
- "ts-jest": "^26.5.4",
- "typescript": "4.2.3",
- "ua-parser-js": "^0.7.22",
+ "ts-jest": "^26.5.6",
+ "typescript": "4.3.5",
+ "ua-parser-js": "^0.7.28",
"why-is-node-running": "^2.2.0"
}
}
diff --git a/test/crawlers/browser_crawler.test.js b/test/crawlers/browser_crawler.test.js
index 167c00f44..aac942477 100644
--- a/test/crawlers/browser_crawler.test.js
+++ b/test/crawlers/browser_crawler.test.js
@@ -401,6 +401,7 @@ describe('BrowserCrawler', () => {
loadedCookies.forEach((cookie) => {
// TODO this test is flaky in CI and we need some more info to debug why.
if (cookie !== 'TEST=12321312312') {
+ // for some reason, the CI failures report the first cookie to be just empty string
console.log('loadedCookies:');
console.dir(loadedCookies);
}