Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

build: unify check and release workflows

Martin Adámek641dcf17a438
diff --git a/.github/workflows/check_pr_title.yml b/.github/workflows/check-pr-title.yml
similarity index 100%
rename from .github/workflows/check_pr_title.yml
rename to .github/workflows/check-pr-title.yml
diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml
deleted file mode 100644
index 28100fb1d..000000000
--- a/.github/workflows/check.yml
+++ /dev/null
@@ -1,43 +0,0 @@
-# This workflow runs for every pull request to lint and test the proposed changes.
-
-name: Check
-
-on:
- pull_request:
-
-jobs:
- # NPM install is done in a separate job and cached to speed up the following jobs.
- test:
- name: Test
- if: ${{ !contains(github.event.head_commit.message, '[skip ci]') }}
- runs-on: ${{ matrix.os }}
-
- strategy:
- fail-fast: false
- matrix:
- os: [ubuntu-latest, windows-latest]
- node-version: [ 15, 16 ]
-
- steps:
- - uses: actions/checkout@v2
- - name: Use Node.js ${{ matrix.node-version }}
- uses: actions/setup-node@v2
- with:
- node-version: ${{ matrix.node-version }}
- - name: Install Dependencies
- run: npm install
- - name: Run Tests
- run: npm test
- lint:
- name: Lint
- if: ${{ !contains(github.event.head_commit.message, '[skip ci]') }}
- runs-on: ubuntu-latest
- steps:
- - uses: actions/checkout@v2
- - uses: actions/setup-node@v2
- with:
- node-version: 16
- - name: Install Dependencies
- run: npm install
- - name: Run Linter
- run: npm run lint
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
deleted file mode 100644
index 32084e2ce..000000000
--- a/.github/workflows/release.yml
+++ /dev/null
@@ -1,117 +0,0 @@
-name: Check & Release
-
-on:
- # Push to master will deploy a beta version
- push:
- branches: [ master, renovate/** ]
- # A release via GitHub releases will deploy a latest version
- release:
- types: [ published ]
-
-jobs:
- # NPM install is done in a separate job and cached to speed up the following jobs.
- build_and_test:
- name: Build & Test
- if: ${{ !contains(github.event.head_commit.message, '[skip ci]') }}
- runs-on: ${{ matrix.os }}
-
- strategy:
- matrix:
- os: [ubuntu-latest, windows-latest]
- node-version: [15, 16]
-
- steps:
- -
- name: Cancel Workflow Action
- uses: styfle/[email protected]
- with:
- access_token: ${{ github.token }}
-
- -
- uses: actions/checkout@v2
- -
- name: Use Node.js ${{ matrix.node-version }}
- uses: actions/setup-node@v2
- with:
- node-version: ${{ matrix.node-version }}
- -
- name: Install Dependencies
- run: npm install
- -
- name: Run Tests
- run: npm test
-
- lint:
- name: Lint
- runs-on: ubuntu-latest
-
- steps:
- -
- uses: actions/checkout@v2
- -
- name: Use Node.js 16
- uses: actions/setup-node@v2
- with:
- node-version: 16
- -
- name: Install Dependencies
- run: npm install
- -
- name: Run Linter
- run: npm run lint
-
-
- # The deploy job is long but there are only 2 important parts. NPM publish
- # and triggering of docker image builds in the apify-actor-docker repo.
- deploy:
- name: Publish to NPM and release images
- if: github.ref == 'refs/heads/master'
- needs: [lint, build_and_test]
- runs-on: ubuntu-latest
- steps:
- -
- uses: actions/checkout@v2
- -
- uses: actions/setup-node@v2
- with:
- node-version: 16
- registry-url: https://registry.npmjs.org/
- -
- name: Install Dependencies
- run: npm install
- -
- # Determine if this is a beta or latest release
- name: Set Release Tag
- run: echo "RELEASE_TAG=$(if [ ${{ github.event_name }} = release ]; then echo latest; else echo beta; fi)" >> $GITHUB_ENV
- -
- # Check version consistency and increment pre-release version number for beta only.
- name: Bump pre-release version
- if: env.RELEASE_TAG == 'beta'
- run: node ./.github/scripts/before-beta-release.js
- -
- name: Collect Versions For Images
- id: versions
- run: |
- apify=`node -p "require('./package.json').version"`
- echo "::set-output name=apify::$apify"
- puppeteer=`node -p "require('./package.json').devDependencies.puppeteer"`
- echo "::set-output name=puppeteer::$puppeteer"
- playwright=`node -p "require('./package.json').devDependencies.playwright"`
- echo "::set-output name=playwright::$playwright"
- -
- name: Publish to NPM
- run: NODE_AUTH_TOKEN=${{secrets.NPM_TOKEN}} npm publish --tag ${{ env.RELEASE_TAG }}
- -
- name: Trigger Docker Image Build
- uses: peter-evans/repository-dispatch@v1
- with:
- token: ${{ secrets.TRIGGER_DOCKER_IMAGE_BUILD_TOKEN }}
- repository: apify/apify-actor-docker
- event-type: build-node-images
- client-payload: >
- {
- "release_tag": "${{ env.RELEASE_TAG }}",
- "apify_version": "${{ steps.versions.outputs.apify }}",
- "puppeteer_version": "${{ steps.versions.outputs.puppeteer }}",
- "playwright_version": "${{ steps.versions.outputs.playwright }}"
- }
diff --git a/.github/workflows/test-and-release.yml b/.github/workflows/test-and-release.yml
new file mode 100644
index 000000000..8a234e24a
--- /dev/null
+++ b/.github/workflows/test-and-release.yml
@@ -0,0 +1,102 @@
+name: Check & Release
+
+on:
+ # Push to master will deploy a beta version
+ push:
+ branches: [ master, renovate/** ]
+ # PR to master branch will skip deployment
+ pull_request:
+ branches: [ master ]
+ # A release via GitHub releases will deploy the latest version
+ release:
+ types: [ published ]
+
+jobs:
+ # NPM install is done in a separate job and cached to speed up the following jobs.
+ build_and_test:
+ name: Build & Test
+ if: ${{ !contains(github.event.head_commit.message, '[skip ci]') }}
+ runs-on: ${{ matrix.os }}
+
+ strategy:
+ matrix:
+ os: [ ubuntu-latest, windows-latest ]
+ node-version: [ 15, 16 ]
+
+ steps:
+ - name: Cancel Workflow Action
+ uses: styfle/[email protected]
+ with:
+ access_token: ${{ github.token }}
+
+ - uses: actions/checkout@v2
+ - name: Use Node.js ${{ matrix.node-version }}
+ uses: actions/setup-node@v2
+ with:
+ node-version: ${{ matrix.node-version }}
+ - name: Install Dependencies
+ run: npm install
+ - name: Run Tests
+ run: npm test
+
+ lint:
+ name: Lint
+ runs-on: ubuntu-latest
+
+ steps:
+ - uses: actions/checkout@v2
+ - name: Use Node.js 16
+ uses: actions/setup-node@v2
+ with:
+ node-version: 16
+ - name: Install Dependencies
+ run: npm install
+ - name: Run Linter
+ run: npm run lint
+
+ # The deploy job is long but there are only 2 important parts. NPM publish
+ # and triggering of docker image builds in the apify-actor-docker repo.
+ deploy:
+ name: Publish to NPM and release images
+ if: github.ref == 'refs/heads/master'
+ needs: [ lint, build_and_test ]
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v2
+ - uses: actions/setup-node@v2
+ with:
+ node-version: 16
+ registry-url: https://registry.npmjs.org/
+ - name: Install Dependencies
+ run: npm install
+ - # Determine if this is a beta or latest release
+ name: Set Release Tag
+ run: echo "RELEASE_TAG=$(if [ ${{ github.event_name }} = release ]; then echo latest; else echo beta; fi)" >> $GITHUB_ENV
+ - # Check version consistency and increment pre-release version number for beta only.
+ name: Bump pre-release version
+ if: env.RELEASE_TAG == 'beta'
+ run: node ./.github/scripts/before-beta-release.js
+ - name: Collect Versions For Images
+ id: versions
+ run: |
+ apify=`node -p "require('./package.json').version"`
+ echo "::set-output name=apify::$apify"
+ puppeteer=`node -p "require('./package.json').devDependencies.puppeteer"`
+ echo "::set-output name=puppeteer::$puppeteer"
+ playwright=`node -p "require('./package.json').devDependencies.playwright"`
+ echo "::set-output name=playwright::$playwright"
+ - name: Publish to NPM
+ run: NODE_AUTH_TOKEN=${{secrets.NPM_TOKEN}} npm publish --tag ${{ env.RELEASE_TAG }}
+ - name: Trigger Docker Image Build
+ uses: peter-evans/repository-dispatch@v1
+ with:
+ token: ${{ secrets.TRIGGER_DOCKER_IMAGE_BUILD_TOKEN }}
+ repository: apify/apify-actor-docker
+ event-type: build-node-images
+ client-payload: >
+ {
+ "release_tag": "${{ env.RELEASE_TAG }}",
+ "apify_version": "${{ steps.versions.outputs.apify }}",
+ "puppeteer_version": "${{ steps.versions.outputs.puppeteer }}",
+ "playwright_version": "${{ steps.versions.outputs.playwright }}"
+ }