Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
chore: update playwright, puppeteer and some other dependencies (#975)
Martin Adámek99a6774f9359
diff --git a/.github/workflows/check.yml b/.github/workflows/check.ymlindex d1292854c..c1bac443e 100644--- a/.github/workflows/check.yml+++ b/.github/workflows/check.yml@@ -7,60 +7,34 @@ on:jobs:# NPM install is done in a separate job and cached to speed up the following jobs.- build_and_test:- name: Build & Test+ test:+ name: Testif: ${{ !contains(github.event.head_commit.message, '[skip ci]') }}runs-on: ubuntu-lateststrategy:fail-fast: falsematrix:- node-version: [10, 12, 14, 15]+ node-version: [ 10, 12, 14, 15 ]steps:- -- uses: actions/checkout@v2- -- name: Use Node.js ${{ matrix.node-version }}+ - uses: actions/checkout@v2+ - name: Use Node.js ${{ matrix.node-version }}uses: actions/setup-node@v1with:node-version: ${{ matrix.node-version }}- -- name: Cache Node Modules- if: ${{ matrix.node-version == 14 }}- uses: actions/cache@v2- with:- path: |- node_modules- build- key: cache-${{ github.run_id }}-v14- -- name: Install Dependencies+ - name: Install Dependenciesrun: npm install- -- name: Run Tests+ - name: Run Testsrun: npm test-lint:name: Lint- needs: [build_and_test]+ if: ${{ !contains(github.event.head_commit.message, '[skip ci]') }}runs-on: ubuntu-latest-steps:- -- uses: actions/checkout@v2- -- name: Use Node.js 14- uses: actions/setup-node@v1- with:- node-version: 14- -- name: Load Cache- uses: actions/cache@v2- with:- path: |- node_modules- build- key: cache-${{ github.run_id }}-v14- -+ - uses: actions/checkout@v2+ - uses: actions/setup-node@v1+ - name: Install Dependencies+ run: npm install+ - name: Run Linterrun: npm run lintdiff --git a/.github/workflows/release.yml b/.github/workflows/release.ymlindex b01a5b6e9..f5665a215 100644--- a/.github/workflows/release.yml+++ b/.github/workflows/release.yml@@ -1,4 +1,4 @@-name: Check & Release Beta+name: Check & Releaseon:# Push to master will deploy a beta versiondiff --git a/jest.config.js b/jest.config.jsindex 5de837f5e..74fed1afb 100644--- a/jest.config.js+++ b/jest.config.js@@ -5,6 +5,8 @@ module.exports = {testRunner: 'jest-circus/runner',verbose: true,rootDir: path.join(__dirname, './'),+ testTimeout: 30e3,+ maxWorkers: 3,testMatch: ['**/?(*.)+(spec|test).[tj]s?(x)'],transform: {'^.+\\.ts$': 'ts-jest',diff --git a/package.json b/package.jsonindex 8169256f0..e81044233 100644--- a/package.json+++ b/package.json@@ -43,7 +43,7 @@"build": "npm run clean && tsc -p tsconfig.json && node ./tools/typescript_fixes.js","build-docs": "npm run build && node ./website/tools/build_docs.js","build-readme": "node ./tools/build_readme.js",- "test": "npm run build && jest --testTimeout=30000 --maxWorkers=3 --silent",+ "test": "npm run build && jest --silent","prepare": "npm run build","prepublishOnly": "(test $CI || (echo \"Publishing is reserved to CI!\"; exit 1))","clean": "rm -rf ./build ./types",@@ -54,11 +54,11 @@"@apify/http-request": "^2.1.1","@apify/ps-tree": "^1.1.3","@apify/storage-local": "^1.0.4",- "@types/cheerio": "^0.22.22",+ "@types/cheerio": "^0.22.28","@types/domhandler": "^2.4.1","@types/node": "^14",- "@types/puppeteer": "^5.4.0",- "@types/socket.io": "^2.1.11",+ "@types/puppeteer": "^5.4.3",+ "@types/socket.io": "^2.1.13","@types/tough-cookie": "^4.0.0","apify-client": "^1.0.3","apify-shared": "^0.6.0",@@ -66,7 +66,7 @@"cheerio": "1.0.0-rc.3","content-type": "^1.0.4","express": "^4.17.1",- "fs-extra": "^9.0.1",+ "fs-extra": "^9.1.0","htmlparser2": "^6.0.0","iconv-lite": "^0.6.2","jquery": "^3.5.1",@@ -89,33 +89,33 @@"@babel/preset-env": "^7.11.5","@babel/register": "^7.11.5","@types/content-type": "^1.1.3",- "@types/fs-extra": "^9.0.2",+ "@types/fs-extra": "^9.0.8","@types/htmlparser2": "^3.10.2",- "@types/jest": "^26.0.14",+ "@types/jest": "^26.0.21","@types/request-promise-native": "^1.0.17","@types/rimraf": "^3.0.0","@types/semver": "^7.3.4",- "@types/sinon": "^9.0.8",- "@types/underscore": "^1.10.24",- "@types/ws": "^7.2.7",+ "@types/sinon": "^9.0.11",+ "@types/underscore": "^1.11.0",+ "@types/ws": "^7.4.0","basic-auth-parser": "^0.0.2","body-parser": "^1.19.0","csv-parse": "^4.12.0","eslint": "^7.11.0","fpcollect": "^1.0.4","fpscanner": "^0.1.5",- "jest": "^26.5.3",- "jest-circus": "^26.5.3",- "puppeteer": "6.0.0",- "playwright": "1.7.1",+ "jest": "^26.6.3",+ "jest-circus": "^26.6.3",+ "playwright": "1.10.0","portastic": "^1.0.1","prettier": "^2.1.2","proxy": "^1.0.2",+ "puppeteer": "8.0.0","sinon": "^9.2.0","sinon-stub-promise": "^4.0.0","socket.io-client": "^3.0.3",- "ts-jest": "^26.4.1",- "typescript": "3.8.3",+ "ts-jest": "^26.5.4",+ "typescript": "4.2.3","ua-parser-js": "^0.7.22","why-is-node-running": "^2.2.0"}diff --git a/test/stealth/stealth.test.js b/test/stealth/stealth.test.jsindex bbeda9258..4468e9335 100644--- a/test/stealth/stealth.test.js+++ b/test/stealth/stealth.test.js@@ -165,18 +165,15 @@ describe('Stealth - testing headless chrome hiding tricks', () => {expect(deviceMemory).not.toBe(0);});- test(- 'it should bypass all of the known tests for browser fingerprinting',- async () => {- await page.goto(testUrl);- const fingerPrint = await getFingerPrint(page);- const testedFingerprint = scanner.analyseFingerprint(fingerPrint);- const failedChecks = Object.values(testedFingerprint).filter((val) => val.consistent < 3);-- // webdriver check is failing due to the outdated fp analyzer- expect(failedChecks.length).toBe(1);- },- );+ test('it should bypass all of the known tests for browser fingerprinting', async () => {+ await page.goto(testUrl);+ const fingerPrint = await getFingerPrint(page);+ const testedFingerprint = scanner.analyseFingerprint(fingerPrint);+ const failedChecks = Object.values(testedFingerprint).filter((val) => val.consistent < 3);++ // webdriver check is failing due to the outdated fp analyzer+ expect(failedChecks.length).toBe(1);+ });test('logs the evaluation warning in "page" when limit is exceeded', async () => {const numberOfIframes = 14;