Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Optimize actions
Ondra Urbana5c6b83e75ca
diff --git a/.github/workflows/check.yml b/.github/workflows/check.ymlindex 4ede43bb3..4b01ae281 100644--- a/.github/workflows/check.yml+++ b/.github/workflows/check.yml@@ -3,73 +3,63 @@name: Checkon:- pull_request:+ pull_request:jobs:- # NPM install is done in a separate job and cached to speed up the following jobs.- install:- if: ${{ !contains(github.event.head_commit.message, '[skip ci]') }}- runs-on: ubuntu-latest+ # NPM install is done in a separate job and cached to speed up the following jobs.+ build_and_test:+ name: Build & Test+ if: ${{ !contains(github.event.head_commit.message, '[skip ci]') }}+ runs-on: ubuntu-latest- strategy:- fail-fast: false- matrix:- node-version: [10, 12, 14]+ strategy:+ matrix:+ node-version: [10, 12, 14]- steps:- - uses: actions/checkout@v2- - name: Use Node.js ${{ matrix.node-version }}- uses: actions/setup-node@v1- with:- node-version: ${{ matrix.node-version }}- - name: Cache Node Modules- uses: actions/cache@v2- with:- path: |- node_modules- build- key: cache-${{ github.run_id }}-v${{ matrix.node-version }}- - name: Install Dependencies- run: npm install+ steps:+ -+ uses: actions/checkout@v2+ -+ name: Use Node.js ${{ matrix.node-version }}+ uses: actions/setup-node@v1+ with:+ node-version: ${{ matrix.node-version }}+ -+ name: Cache Node Modules+ if: ${{ matrix.node-version == 14 }}+ uses: actions/cache@v2+ with:+ path: |+ node_modules+ build+ key: cache-${{ github.run_id }}-v14+ -+ name: Install Dependencies+ run: npm install+ -+ name: Run Tests+ run: npm test- lint:- needs: [install]- runs-on: ubuntu-latest+ lint:+ name: Lint+ needs: [build_and_test]+ runs-on: ubuntu-latest- steps:- - uses: actions/checkout@v2- - name: Use Node.js 14- uses: actions/setup-node@v1- with:- node-version: 14- - name: Load Cache- uses: actions/cache@v2- with:- path: |- node_modules- build- key: cache-${{ github.run_id }}-v14- - run: npm run lint-- test:- needs: [install]- runs-on: ubuntu-latest-- strategy:- matrix:- node-version: [10, 12, 14]-- steps:- - uses: actions/checkout@v2- - name: Use Node.js ${{ matrix.node-version }}- uses: actions/setup-node@v1- with:- node-version: ${{ matrix.node-version }}- - name: Load Cache- uses: actions/cache@v2- with:- path: |- node_modules- build- key: cache-${{ github.run_id }}-v${{ matrix.node-version }}- - run: npm test+ steps:+ -+ uses: actions/checkout@v2+ -+ name: Use Node.js 14+ uses: actions/setup-node@v1+ with:+ node-version: 14+ -+ name: Load Cache+ uses: actions/cache@v2+ with:+ path: |+ node_modules+ build+ key: cache-${{ github.run_id }}-v14+ -+ run: npm run lintdiff --git a/.github/workflows/release.yml b/.github/workflows/release.ymlindex 100b31c4a..e81b1bfc2 100644--- a/.github/workflows/release.yml+++ b/.github/workflows/release.yml@@ -11,7 +11,8 @@ on:jobs:# NPM install is done in a separate job and cached to speed up the following jobs.- install:+ build_and_test:+ name: Build & Testif: ${{ !contains(github.event.head_commit.message, '[skip ci]') }}runs-on: ubuntu-latest@@ -29,18 +30,23 @@ jobs:node-version: ${{ matrix.node-version }}-name: Cache Node Modules+ if: ${{ matrix.node-version == 14 }}uses: actions/cache@v2with:path: |node_modulesbuild- key: cache-${{ github.run_id }}-v${{ matrix.node-version }}+ key: cache-${{ github.run_id }}-v14-name: Install Dependenciesrun: npm install+ -+ name: Run Tests+ run: npm testlint:- needs: [install]+ name: Lint+ needs: [build_and_test]runs-on: ubuntu-lateststeps:@@ -63,37 +69,11 @@ jobs:run: npm run lint- test:- needs: [install]- runs-on: ubuntu-latest-- strategy:- matrix:- node-version: [10, 12, 14]-- steps:- -- uses: actions/checkout@v2- -- name: Use Node.js ${{ matrix.node-version }}- uses: actions/setup-node@v1- with:- node-version: ${{ matrix.node-version }}- -- name: Load Cache- uses: actions/cache@v2- with:- path: |- node_modules- build- key: cache-${{ github.run_id }}-v${{ matrix.node-version }}- -- run: npm test-# The deploy job is long but there are only 2 important parts. NPM publish# and triggering of docker image builds in the apify-actor-docker repo.deploy:- needs: [lint, test]+ name: Publish to NPM and release images+ needs: [lint]runs-on: ubuntu-lateststeps:-@@ -110,7 +90,7 @@ jobs:path: |node_modulesbuild- key: cache-${{ github.run_id }}-v${{ matrix.node-version }}+ key: cache-${{ github.run_id }}-v14-# Determine if this is a beta or latest releasename: Set Release Tagdiff --git a/test/puppeteer_utils.test.js b/test/puppeteer_utils.test.jsindex a74afe934..03c4352c6 100644--- a/test/puppeteer_utils.test.js+++ b/test/puppeteer_utils.test.js@@ -1,6 +1,5 @@import sinon from 'sinon';import path from 'path';-import { ENV_VARS } from 'apify-shared/consts';import Apify from '../build/index';import * as keyValueStore from '../build/storages/key_value_store';import LocalStorageDirEmulator from './local_storage_dir_emulator';