Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
build: unify check and release workflows
Martin Adámek641dcf17a438
diff --git a/.github/workflows/check_pr_title.yml b/.github/workflows/check-pr-title.ymlsimilarity index 100%rename from .github/workflows/check_pr_title.ymlrename to .github/workflows/check-pr-title.ymldiff --git a/.github/workflows/check.yml b/.github/workflows/check.ymldeleted file mode 100644index 28100fb1d..000000000--- a/.github/workflows/check.yml+++ /dev/null@@ -1,43 +0,0 @@-# This workflow runs for every pull request to lint and test the proposed changes.--name: Check--on:- pull_request:--jobs:- # NPM install is done in a separate job and cached to speed up the following jobs.- test:- name: Test- if: ${{ !contains(github.event.head_commit.message, '[skip ci]') }}- runs-on: ${{ matrix.os }}-- strategy:- fail-fast: false- matrix:- os: [ubuntu-latest, windows-latest]- node-version: [ 15, 16 ]-- steps:- - uses: actions/checkout@v2- - name: Use Node.js ${{ matrix.node-version }}- uses: actions/setup-node@v2- with:- node-version: ${{ matrix.node-version }}- - name: Install Dependencies- run: npm install- - name: Run Tests- run: npm test- lint:- name: Lint- if: ${{ !contains(github.event.head_commit.message, '[skip ci]') }}- runs-on: ubuntu-latest- steps:- - uses: actions/checkout@v2- - uses: actions/setup-node@v2- with:- node-version: 16- - name: Install Dependencies- run: npm install- - name: Run Linter- run: npm run lintdiff --git a/.github/workflows/release.yml b/.github/workflows/release.ymldeleted file mode 100644index 32084e2ce..000000000--- a/.github/workflows/release.yml+++ /dev/null@@ -1,117 +0,0 @@-name: Check & Release--on:- # Push to master will deploy a beta version- push:- branches: [ master, renovate/** ]- # A release via GitHub releases will deploy a latest version- release:- types: [ published ]--jobs:- # NPM install is done in a separate job and cached to speed up the following jobs.- build_and_test:- name: Build & Test- if: ${{ !contains(github.event.head_commit.message, '[skip ci]') }}- runs-on: ${{ matrix.os }}-- strategy:- matrix:- os: [ubuntu-latest, windows-latest]- node-version: [15, 16]-- steps:- -- name: Cancel Workflow Action- uses: styfle/[email protected]- with:- access_token: ${{ github.token }}-- -- uses: actions/checkout@v2- -- name: Use Node.js ${{ matrix.node-version }}- uses: actions/setup-node@v2- with:- node-version: ${{ matrix.node-version }}- -- name: Install Dependencies- run: npm install- -- name: Run Tests- run: npm test-- lint:- name: Lint- runs-on: ubuntu-latest-- steps:- -- uses: actions/checkout@v2- -- name: Use Node.js 16- uses: actions/setup-node@v2- with:- node-version: 16- -- name: Install Dependencies- run: npm install- -- name: Run Linter- run: npm run lint--- # The deploy job is long but there are only 2 important parts. NPM publish- # and triggering of docker image builds in the apify-actor-docker repo.- deploy:- name: Publish to NPM and release images- if: github.ref == 'refs/heads/master'- needs: [lint, build_and_test]- runs-on: ubuntu-latest- steps:- -- uses: actions/checkout@v2- -- uses: actions/setup-node@v2- with:- node-version: 16- registry-url: https://registry.npmjs.org/- -- name: Install Dependencies- run: npm install- -- # Determine if this is a beta or latest release- name: Set Release Tag- run: echo "RELEASE_TAG=$(if [ ${{ github.event_name }} = release ]; then echo latest; else echo beta; fi)" >> $GITHUB_ENV- -- # Check version consistency and increment pre-release version number for beta only.- name: Bump pre-release version- if: env.RELEASE_TAG == 'beta'- run: node ./.github/scripts/before-beta-release.js- -- name: Collect Versions For Images- id: versions- run: |- apify=`node -p "require('./package.json').version"`- echo "::set-output name=apify::$apify"- puppeteer=`node -p "require('./package.json').devDependencies.puppeteer"`- echo "::set-output name=puppeteer::$puppeteer"- playwright=`node -p "require('./package.json').devDependencies.playwright"`- echo "::set-output name=playwright::$playwright"- -- name: Publish to NPM- run: NODE_AUTH_TOKEN=${{secrets.NPM_TOKEN}} npm publish --tag ${{ env.RELEASE_TAG }}- -- name: Trigger Docker Image Build- uses: peter-evans/repository-dispatch@v1- with:- token: ${{ secrets.TRIGGER_DOCKER_IMAGE_BUILD_TOKEN }}- repository: apify/apify-actor-docker- event-type: build-node-images- client-payload: >- {- "release_tag": "${{ env.RELEASE_TAG }}",- "apify_version": "${{ steps.versions.outputs.apify }}",- "puppeteer_version": "${{ steps.versions.outputs.puppeteer }}",- "playwright_version": "${{ steps.versions.outputs.playwright }}"- }diff --git a/.github/workflows/test-and-release.yml b/.github/workflows/test-and-release.ymlnew file mode 100644index 000000000..8a234e24a--- /dev/null+++ b/.github/workflows/test-and-release.yml@@ -0,0 +1,102 @@+name: Check & Release++on:+ # Push to master will deploy a beta version+ push:+ branches: [ master, renovate/** ]+ # PR to master branch will skip deployment+ pull_request:+ branches: [ master ]+ # A release via GitHub releases will deploy the latest version+ release:+ types: [ published ]++jobs:+ # NPM install is done in a separate job and cached to speed up the following jobs.+ build_and_test:+ name: Build & Test+ if: ${{ !contains(github.event.head_commit.message, '[skip ci]') }}+ runs-on: ${{ matrix.os }}++ strategy:+ matrix:+ os: [ ubuntu-latest, windows-latest ]+ node-version: [ 15, 16 ]++ steps:+ - name: Cancel Workflow Action+ uses: styfle/[email protected]+ with:+ access_token: ${{ github.token }}++ - uses: actions/checkout@v2+ - name: Use Node.js ${{ matrix.node-version }}+ uses: actions/setup-node@v2+ with:+ node-version: ${{ matrix.node-version }}+ - name: Install Dependencies+ run: npm install+ - name: Run Tests+ run: npm test++ lint:+ name: Lint+ runs-on: ubuntu-latest++ steps:+ - uses: actions/checkout@v2+ - name: Use Node.js 16+ uses: actions/setup-node@v2+ with:+ node-version: 16+ - name: Install Dependencies+ run: npm install+ - name: Run Linter+ run: npm run lint++ # The deploy job is long but there are only 2 important parts. NPM publish+ # and triggering of docker image builds in the apify-actor-docker repo.+ deploy:+ name: Publish to NPM and release images+ if: github.ref == 'refs/heads/master'+ needs: [ lint, build_and_test ]+ runs-on: ubuntu-latest+ steps:+ - uses: actions/checkout@v2+ - uses: actions/setup-node@v2+ with:+ node-version: 16+ registry-url: https://registry.npmjs.org/+ - name: Install Dependencies+ run: npm install+ - # Determine if this is a beta or latest release+ name: Set Release Tag+ run: echo "RELEASE_TAG=$(if [ ${{ github.event_name }} = release ]; then echo latest; else echo beta; fi)" >> $GITHUB_ENV+ - # Check version consistency and increment pre-release version number for beta only.+ name: Bump pre-release version+ if: env.RELEASE_TAG == 'beta'+ run: node ./.github/scripts/before-beta-release.js+ - name: Collect Versions For Images+ id: versions+ run: |+ apify=`node -p "require('./package.json').version"`+ echo "::set-output name=apify::$apify"+ puppeteer=`node -p "require('./package.json').devDependencies.puppeteer"`+ echo "::set-output name=puppeteer::$puppeteer"+ playwright=`node -p "require('./package.json').devDependencies.playwright"`+ echo "::set-output name=playwright::$playwright"+ - name: Publish to NPM+ run: NODE_AUTH_TOKEN=${{secrets.NPM_TOKEN}} npm publish --tag ${{ env.RELEASE_TAG }}+ - name: Trigger Docker Image Build+ uses: peter-evans/repository-dispatch@v1+ with:+ token: ${{ secrets.TRIGGER_DOCKER_IMAGE_BUILD_TOKEN }}+ repository: apify/apify-actor-docker+ event-type: build-node-images+ client-payload: >+ {+ "release_tag": "${{ env.RELEASE_TAG }}",+ "apify_version": "${{ steps.versions.outputs.apify }}",+ "puppeteer_version": "${{ steps.versions.outputs.puppeteer }}",+ "playwright_version": "${{ steps.versions.outputs.playwright }}"+ }