Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

chore: update playwright, puppeteer and some other dependencies (#975)

Martin Adámek99a6774f9359
diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml
index d1292854c..c1bac443e 100644
--- a/.github/workflows/check.yml
+++ b/.github/workflows/check.yml
@@ -7,60 +7,34 @@ on:
jobs:
# NPM install is done in a separate job and cached to speed up the following jobs.
- build_and_test:
- name: Build & Test
+ test:
+ name: Test
if: ${{ !contains(github.event.head_commit.message, '[skip ci]') }}
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
- node-version: [10, 12, 14, 15]
+ node-version: [ 10, 12, 14, 15 ]
steps:
- -
- uses: actions/checkout@v2
- -
- name: Use Node.js ${{ matrix.node-version }}
+ - uses: actions/checkout@v2
+ - name: Use Node.js ${{ matrix.node-version }}
uses: actions/setup-node@v1
with:
node-version: ${{ matrix.node-version }}
- -
- name: Cache Node Modules
- if: ${{ matrix.node-version == 14 }}
- uses: actions/cache@v2
- with:
- path: |
- node_modules
- build
- key: cache-${{ github.run_id }}-v14
- -
- name: Install Dependencies
+ - name: Install Dependencies
run: npm install
- -
- name: Run Tests
+ - name: Run Tests
run: npm test
-
lint:
name: Lint
- needs: [build_and_test]
+ if: ${{ !contains(github.event.head_commit.message, '[skip ci]') }}
runs-on: ubuntu-latest
-
steps:
- -
- uses: actions/checkout@v2
- -
- name: Use Node.js 14
- uses: actions/setup-node@v1
- with:
- node-version: 14
- -
- name: Load Cache
- uses: actions/cache@v2
- with:
- path: |
- node_modules
- build
- key: cache-${{ github.run_id }}-v14
- -
+ - uses: actions/checkout@v2
+ - uses: actions/setup-node@v1
+ - name: Install Dependencies
+ run: npm install
+ - name: Run Linter
run: npm run lint
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index b01a5b6e9..f5665a215 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -1,4 +1,4 @@
-name: Check & Release Beta
+name: Check & Release
on:
# Push to master will deploy a beta version
diff --git a/jest.config.js b/jest.config.js
index 5de837f5e..74fed1afb 100644
--- a/jest.config.js
+++ b/jest.config.js
@@ -5,6 +5,8 @@ module.exports = {
testRunner: 'jest-circus/runner',
verbose: true,
rootDir: path.join(__dirname, './'),
+ testTimeout: 30e3,
+ maxWorkers: 3,
testMatch: ['**/?(*.)+(spec|test).[tj]s?(x)'],
transform: {
'^.+\\.ts$': 'ts-jest',
diff --git a/package.json b/package.json
index 8169256f0..e81044233 100644
--- a/package.json
+++ b/package.json
@@ -43,7 +43,7 @@
"build": "npm run clean && tsc -p tsconfig.json && node ./tools/typescript_fixes.js",
"build-docs": "npm run build && node ./website/tools/build_docs.js",
"build-readme": "node ./tools/build_readme.js",
- "test": "npm run build && jest --testTimeout=30000 --maxWorkers=3 --silent",
+ "test": "npm run build && jest --silent",
"prepare": "npm run build",
"prepublishOnly": "(test $CI || (echo \"Publishing is reserved to CI!\"; exit 1))",
"clean": "rm -rf ./build ./types",
@@ -54,11 +54,11 @@
"@apify/http-request": "^2.1.1",
"@apify/ps-tree": "^1.1.3",
"@apify/storage-local": "^1.0.4",
- "@types/cheerio": "^0.22.22",
+ "@types/cheerio": "^0.22.28",
"@types/domhandler": "^2.4.1",
"@types/node": "^14",
- "@types/puppeteer": "^5.4.0",
- "@types/socket.io": "^2.1.11",
+ "@types/puppeteer": "^5.4.3",
+ "@types/socket.io": "^2.1.13",
"@types/tough-cookie": "^4.0.0",
"apify-client": "^1.0.3",
"apify-shared": "^0.6.0",
@@ -66,7 +66,7 @@
"cheerio": "1.0.0-rc.3",
"content-type": "^1.0.4",
"express": "^4.17.1",
- "fs-extra": "^9.0.1",
+ "fs-extra": "^9.1.0",
"htmlparser2": "^6.0.0",
"iconv-lite": "^0.6.2",
"jquery": "^3.5.1",
@@ -89,33 +89,33 @@
"@babel/preset-env": "^7.11.5",
"@babel/register": "^7.11.5",
"@types/content-type": "^1.1.3",
- "@types/fs-extra": "^9.0.2",
+ "@types/fs-extra": "^9.0.8",
"@types/htmlparser2": "^3.10.2",
- "@types/jest": "^26.0.14",
+ "@types/jest": "^26.0.21",
"@types/request-promise-native": "^1.0.17",
"@types/rimraf": "^3.0.0",
"@types/semver": "^7.3.4",
- "@types/sinon": "^9.0.8",
- "@types/underscore": "^1.10.24",
- "@types/ws": "^7.2.7",
+ "@types/sinon": "^9.0.11",
+ "@types/underscore": "^1.11.0",
+ "@types/ws": "^7.4.0",
"basic-auth-parser": "^0.0.2",
"body-parser": "^1.19.0",
"csv-parse": "^4.12.0",
"eslint": "^7.11.0",
"fpcollect": "^1.0.4",
"fpscanner": "^0.1.5",
- "jest": "^26.5.3",
- "jest-circus": "^26.5.3",
- "puppeteer": "6.0.0",
- "playwright": "1.7.1",
+ "jest": "^26.6.3",
+ "jest-circus": "^26.6.3",
+ "playwright": "1.10.0",
"portastic": "^1.0.1",
"prettier": "^2.1.2",
"proxy": "^1.0.2",
+ "puppeteer": "8.0.0",
"sinon": "^9.2.0",
"sinon-stub-promise": "^4.0.0",
"socket.io-client": "^3.0.3",
- "ts-jest": "^26.4.1",
- "typescript": "3.8.3",
+ "ts-jest": "^26.5.4",
+ "typescript": "4.2.3",
"ua-parser-js": "^0.7.22",
"why-is-node-running": "^2.2.0"
}
diff --git a/test/stealth/stealth.test.js b/test/stealth/stealth.test.js
index bbeda9258..4468e9335 100644
--- a/test/stealth/stealth.test.js
+++ b/test/stealth/stealth.test.js
@@ -165,18 +165,15 @@ describe('Stealth - testing headless chrome hiding tricks', () => {
expect(deviceMemory).not.toBe(0);
});
- test(
- 'it should bypass all of the known tests for browser fingerprinting',
- async () => {
- await page.goto(testUrl);
- const fingerPrint = await getFingerPrint(page);
- const testedFingerprint = scanner.analyseFingerprint(fingerPrint);
- const failedChecks = Object.values(testedFingerprint).filter((val) => val.consistent < 3);
-
- // webdriver check is failing due to the outdated fp analyzer
- expect(failedChecks.length).toBe(1);
- },
- );
+ test('it should bypass all of the known tests for browser fingerprinting', async () => {
+ await page.goto(testUrl);
+ const fingerPrint = await getFingerPrint(page);
+ const testedFingerprint = scanner.analyseFingerprint(fingerPrint);
+ const failedChecks = Object.values(testedFingerprint).filter((val) => val.consistent < 3);
+
+ // webdriver check is failing due to the outdated fp analyzer
+ expect(failedChecks.length).toBe(1);
+ });
test('logs the evaluation warning in "page" when limit is exceeded', async () => {
const numberOfIframes = 14;