Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

Optimize actions

Ondra Urbana5c6b83e75ca
diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml
index 4ede43bb3..4b01ae281 100644
--- a/.github/workflows/check.yml
+++ b/.github/workflows/check.yml
@@ -3,73 +3,63 @@
name: Check
on:
- pull_request:
+ pull_request:
jobs:
- # NPM install is done in a separate job and cached to speed up the following jobs.
- install:
- if: ${{ !contains(github.event.head_commit.message, '[skip ci]') }}
- runs-on: ubuntu-latest
+ # NPM install is done in a separate job and cached to speed up the following jobs.
+ build_and_test:
+ name: Build & Test
+ if: ${{ !contains(github.event.head_commit.message, '[skip ci]') }}
+ runs-on: ubuntu-latest
- strategy:
- fail-fast: false
- matrix:
- node-version: [10, 12, 14]
+ strategy:
+ matrix:
+ node-version: [10, 12, 14]
- steps:
- - uses: actions/checkout@v2
- - name: Use Node.js ${{ matrix.node-version }}
- uses: actions/setup-node@v1
- with:
- node-version: ${{ matrix.node-version }}
- - name: Cache Node Modules
- uses: actions/cache@v2
- with:
- path: |
- node_modules
- build
- key: cache-${{ github.run_id }}-v${{ matrix.node-version }}
- - name: Install Dependencies
- run: npm install
+ steps:
+ -
+ uses: actions/checkout@v2
+ -
+ name: Use Node.js ${{ matrix.node-version }}
+ uses: actions/setup-node@v1
+ with:
+ node-version: ${{ matrix.node-version }}
+ -
+ name: Cache Node Modules
+ if: ${{ matrix.node-version == 14 }}
+ uses: actions/cache@v2
+ with:
+ path: |
+ node_modules
+ build
+ key: cache-${{ github.run_id }}-v14
+ -
+ name: Install Dependencies
+ run: npm install
+ -
+ name: Run Tests
+ run: npm test
- lint:
- needs: [install]
- runs-on: ubuntu-latest
+ lint:
+ name: Lint
+ needs: [build_and_test]
+ runs-on: ubuntu-latest
- steps:
- - uses: actions/checkout@v2
- - name: Use Node.js 14
- uses: actions/setup-node@v1
- with:
- node-version: 14
- - name: Load Cache
- uses: actions/cache@v2
- with:
- path: |
- node_modules
- build
- key: cache-${{ github.run_id }}-v14
- - run: npm run lint
-
- test:
- needs: [install]
- runs-on: ubuntu-latest
-
- strategy:
- matrix:
- node-version: [10, 12, 14]
-
- steps:
- - uses: actions/checkout@v2
- - name: Use Node.js ${{ matrix.node-version }}
- uses: actions/setup-node@v1
- with:
- node-version: ${{ matrix.node-version }}
- - name: Load Cache
- uses: actions/cache@v2
- with:
- path: |
- node_modules
- build
- key: cache-${{ github.run_id }}-v${{ matrix.node-version }}
- - run: npm test
+ steps:
+ -
+ uses: actions/checkout@v2
+ -
+ name: Use Node.js 14
+ uses: actions/setup-node@v1
+ with:
+ node-version: 14
+ -
+ name: Load Cache
+ uses: actions/cache@v2
+ with:
+ path: |
+ node_modules
+ build
+ key: cache-${{ github.run_id }}-v14
+ -
+ run: npm run lint
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index 100b31c4a..e81b1bfc2 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -11,7 +11,8 @@ on:
jobs:
# NPM install is done in a separate job and cached to speed up the following jobs.
- install:
+ build_and_test:
+ name: Build & Test
if: ${{ !contains(github.event.head_commit.message, '[skip ci]') }}
runs-on: ubuntu-latest
@@ -29,18 +30,23 @@ jobs:
node-version: ${{ matrix.node-version }}
-
name: Cache Node Modules
+ if: ${{ matrix.node-version == 14 }}
uses: actions/cache@v2
with:
path: |
node_modules
build
- key: cache-${{ github.run_id }}-v${{ matrix.node-version }}
+ key: cache-${{ github.run_id }}-v14
-
name: Install Dependencies
run: npm install
+ -
+ name: Run Tests
+ run: npm test
lint:
- needs: [install]
+ name: Lint
+ needs: [build_and_test]
runs-on: ubuntu-latest
steps:
@@ -63,37 +69,11 @@ jobs:
run: npm run lint
- test:
- needs: [install]
- runs-on: ubuntu-latest
-
- strategy:
- matrix:
- node-version: [10, 12, 14]
-
- steps:
- -
- uses: actions/checkout@v2
- -
- name: Use Node.js ${{ matrix.node-version }}
- uses: actions/setup-node@v1
- with:
- node-version: ${{ matrix.node-version }}
- -
- name: Load Cache
- uses: actions/cache@v2
- with:
- path: |
- node_modules
- build
- key: cache-${{ github.run_id }}-v${{ matrix.node-version }}
- -
- run: npm test
-
# The deploy job is long but there are only 2 important parts. NPM publish
# and triggering of docker image builds in the apify-actor-docker repo.
deploy:
- needs: [lint, test]
+ name: Publish to NPM and release images
+ needs: [lint]
runs-on: ubuntu-latest
steps:
-
@@ -110,7 +90,7 @@ jobs:
path: |
node_modules
build
- key: cache-${{ github.run_id }}-v${{ matrix.node-version }}
+ key: cache-${{ github.run_id }}-v14
-
# Determine if this is a beta or latest release
name: Set Release Tag
diff --git a/test/puppeteer_utils.test.js b/test/puppeteer_utils.test.js
index a74afe934..03c4352c6 100644
--- a/test/puppeteer_utils.test.js
+++ b/test/puppeteer_utils.test.js
@@ -1,6 +1,5 @@
import sinon from 'sinon';
import path from 'path';
-import { ENV_VARS } from 'apify-shared/consts';
import Apify from '../build/index';
import * as keyValueStore from '../build/storages/key_value_store';
import LocalStorageDirEmulator from './local_storage_dir_emulator';