Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

build: rework CI workflows to support publishing (#1570)

Vlad Frangu334e72d6d9ed
diff --git a/.github/workflows/release-next.yml b/.github/workflows/release-next.yml
new file mode 100644
index 000000000..b84aca4ec
--- /dev/null
+++ b/.github/workflows/release-next.yml
@@ -0,0 +1,142 @@
+name: Release @next
+
+on:
+ # Pushes to master release a new @next version
+ push:
+ branches:
+ - master
+
+jobs:
+ # Test before releasing to ensure we don't release a broken version
+ build_and_test:
+ name: Build & Test
+ if: (!contains(github.event.head_commit.message, '[skip ci]') && !contains(github.event.head_commit.message, 'docs:'))
+ strategy:
+ fail-fast: true
+ matrix:
+ # We don't test on Windows as the tests are flaky
+ os: [ubuntu-latest]
+ node-version: [16, 18]
+
+ runs-on: ${{ matrix.os }}
+
+ steps:
+ - name: Cancel Workflow Action
+ uses: styfle/[email protected]
+ with:
+ access_token: ${{ github.token }}
+
+ - name: Checkout repository
+ uses: actions/checkout@v3
+
+ - name: Use Node.js ${{ matrix.node-version }}
+ uses: actions/setup-node@v3
+ with:
+ node-version: ${{ matrix.node-version }}
+ cache: 'npm'
+ cache-dependency-path: 'package-lock.json'
+
+ - name: Setup Playwright
+ uses: microsoft/playwright-github-action@v1
+
+ - name: Turbo cache
+ id: turbo-cache
+ uses: actions/cache@v3
+ with:
+ path: .turbo
+ key: turbo-${{ github.job }}-${{ github.ref_name }}-${{ github.sha }}
+ restore-keys: |
+ turbo-${{ github.job }}-${{ github.ref_name }}-
+
+ - name: Install Dependencies
+ run: npm ci
+
+ - name: Build
+ run: npm run ci:build
+
+ - name: Tests
+ run: npm test
+
+ release_next:
+ name: Release @next
+ if: (!contains(github.event.head_commit.message, '[skip ci]') && !contains(github.event.head_commit.message, 'docs:'))
+ needs: build_and_test
+ runs-on: ubuntu-latest
+
+ steps:
+ - name: Checkout repository
+ uses: actions/checkout@v3
+
+ - name: Use Node.js 18
+ uses: actions/setup-node@v3
+ with:
+ node-version: 18
+ cache: 'npm'
+ cache-dependency-path: 'package-lock.json'
+
+ - name: Turbo cache
+ id: turbo-cache
+ uses: actions/cache@v3
+ with:
+ path: .turbo
+ key: turbo-${{ github.job }}-${{ github.ref_name }}-${{ github.sha }}
+ restore-keys: |
+ turbo-${{ github.job }}-${{ github.ref_name }}-
+
+ - name: Install Dependencies
+ run: npm ci
+
+ - name: Generate changed packages list
+ id: changed-packages
+ run: |
+ echo "::set-output name=changed_packages::$(node ./node_modules/.bin/lerna changed -p | wc -l | xargs)"
+
+ - name: Report nothing to release
+ if: steps.changed-packages.outputs.changed_packages == '0'
+ run: echo "Nothing to release"
+
+ - name: Release @next
+ if: steps.changed-packages.outputs.changed_packages != '0'
+ run: |
+ git config --global user.name 'Apify Release Bot'
+ git config --global user.email '[email protected]'
+ npx turbo copy --force -- -- --canary --preid=beta
+ git commit -am "chore: bump canary versions [skip ci]"
+
+ echo "access=public" > .npmrc
+ echo "//registry.npmjs.org/:_authToken=${NPM_TOKEN}" >> .npmrc
+ npm run publish:next -- --yes --no-verify-access
+ env:
+ NPM_TOKEN: ${{ secrets.NPM_TOKEN }}
+ GIT_USER: "[email protected]:${{ secrets.GH_TOKEN }}"
+ GH_TOKEN: ${{ secrets.GH_TOKEN }}
+
+ - name: Collect versions for Docker images
+ id: versions
+ run: |
+ crawlee=`node -p "require('./packages/crawlee/package.json').version"`
+ echo "::set-output name=crawlee::$crawlee"
+ apify=`node -p "require('./package.json').devDependencies.apify"`
+ echo "::set-output name=apify::$apify"
+ puppeteer=`node -p "require('./package.json').devDependencies.puppeteer"`
+ echo "::set-output name=puppeteer::$puppeteer"
+ playwright=`node -p "require('./package.json').devDependencies.playwright"`
+ echo "::set-output name=playwright::$playwright"
+
+ - name: Trigger Docker image builds
+ uses: peter-evans/repository-dispatch@v2
+ # Trigger next images only if we have something new pushed
+ if: steps.changed-packages.outputs.changed_packages != '0'
+ with:
+ token: ${{ secrets.TRIGGER_DOCKER_IMAGE_BUILD_TOKEN }}
+ repository: apify/apify-actor-docker
+ event-type: build-node-images
+ client-payload: >
+ {
+ "release_tag": "next",
+ "apify_version": "${{ steps.versions.outputs.apify }}",
+ "crawlee_version": "${{ steps.versions.outputs.crawlee }}",
+ "puppeteer_version": "${{ steps.versions.outputs.puppeteer }}",
+ "playwright_version": "${{ steps.versions.outputs.playwright }}"
+ }
+
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
new file mode 100644
index 000000000..07addcb0e
--- /dev/null
+++ b/.github/workflows/release.yml
@@ -0,0 +1,131 @@
+name: Release a new Crawlee version
+
+on:
+ # Manual call from UI / automations(?)
+ workflow_dispatch:
+ inputs:
+ version:
+ description: The version to bump (if you choose custom, please include it under custom version)
+ required: true
+ default: 'patch'
+ type: choice
+ options:
+ - "patch"
+ - "minor"
+ - "major"
+ - "custom"
+ custom_version:
+ description: The custom version to bump to (only for "custom" type)
+ required: false
+ type: string
+ default: ''
+ # Eventually this will run on new releases too
+ # release:
+ # types:
+ # - published
+
+jobs:
+ # Test before releasing to ensure we don't release a broken version
+ build_and_test:
+ name: Build & Test
+ if: (!contains(github.event.head_commit.message, '[skip ci]') && !contains(github.event.head_commit.message, 'docs:'))
+ strategy:
+ fail-fast: true
+ matrix:
+ # We don't test on Windows as the tests are flaky
+ os: [ubuntu-latest]
+ node-version: [16, 18]
+
+ runs-on: ${{ matrix.os }}
+
+ steps:
+ - name: Cancel Workflow Action
+ uses: styfle/[email protected]
+ with:
+ access_token: ${{ github.token }}
+
+ - name: Checkout repository
+ uses: actions/checkout@v3
+
+ - name: Use Node.js ${{ matrix.node-version }}
+ uses: actions/setup-node@v3
+ with:
+ node-version: ${{ matrix.node-version }}
+ cache: 'npm'
+ cache-dependency-path: 'package-lock.json'
+
+ - name: Setup Playwright
+ uses: microsoft/playwright-github-action@v1
+
+ - name: Turbo cache
+ id: turbo-cache
+ uses: actions/cache@v3
+ with:
+ path: .turbo
+ key: turbo-${{ github.job }}-${{ github.ref_name }}-${{ github.sha }}
+ restore-keys: |
+ turbo-${{ github.job }}-${{ github.ref_name }}-
+
+ - name: Install Dependencies
+ run: npm ci
+
+ - name: Build
+ run: npm run ci:build
+
+ - name: Tests
+ run: npm test
+
+ release:
+ name: "Bump Crawlee: ${{ inputs.version }} version (${{ inputs.custom_version || 'n/a' }} custom version)"
+ if: (!contains(github.event.head_commit.message, '[skip ci]') && !contains(github.event.head_commit.message, 'docs:'))
+ needs: build_and_test
+ runs-on: ubuntu-latest
+
+ steps:
+ - name: Checkout repository
+ uses: actions/checkout@v3
+
+ - name: Use Node.js 18
+ uses: actions/setup-node@v3
+ with:
+ node-version: 18
+ cache: 'npm'
+ cache-dependency-path: 'package-lock.json'
+
+ - name: Turbo cache
+ id: turbo-cache
+ uses: actions/cache@v3
+ with:
+ path: .turbo
+ key: turbo-${{ github.job }}-${{ github.ref_name }}-${{ github.sha }}
+ restore-keys: |
+ turbo-${{ github.job }}-${{ github.ref_name }}-
+
+ - name: Install Dependencies
+ run: npm ci
+
+ - name: Build packages
+ run: npm run ci:build
+
+ - name: Setup git user and npm
+ run: |
+ git config --global user.name "Apify Release Bot"
+ git config --global user.email "[email protected]"
+
+ echo "access=public" > .npmrc
+ echo "//registry.npmjs.org/:_authToken=${{ secrets.NPM_TOKEN }}" >> .npmrc
+
+ - name: Bump version to custom version
+ if: ${{ github.event.inputs.version == 'custom' && github.event.inputs.custom_version != '' }}
+ run: npx lerna version ${{ github.event.inputs.custom_version}}
+
+ - name: Bump version to ${{ github.event.inputs.version }} version
+ if: ${{ github.event.inputs.version != 'custom' }}
+ run: npx lerna version ${{ github.event.inputs.version}}
+
+ - name: Publish packages
+ run: npx lerna publish from-package --contents dist --force-publish
+ env:
+ NPM_TOKEN: ${{ secrets.NPM_TOKEN }}
+ GIT_USER: "[email protected]:${{ secrets.GH_TOKEN }}"
+ GH_TOKEN: ${{ secrets.GH_TOKEN }}
diff --git a/.github/workflows/test-and-release.yml b/.github/workflows/test-and-release.yml
deleted file mode 100644
index 5afb1a33b..000000000
--- a/.github/workflows/test-and-release.yml
+++ /dev/null
@@ -1,218 +0,0 @@
-name: Check & Release
-
-on:
- # Push to master will deploy a dev version
- push:
- branches: [ master, renovate/** ]
- # PR to master branch will skip deployment
- pull_request:
- branches: [ master ]
- # A release via GitHub releases will deploy the latest version
- release:
- types: [ published ]
-
-jobs:
- # NPM install is done in a separate job and cached to speed up the following jobs.
- build_and_test:
- name: Build & Test
- if: (!contains(github.event.head_commit.message, '[skip ci]') && !contains(github.event.head_commit.message, 'docs:'))
- runs-on: ${{ matrix.os }}
-
- strategy:
- fail-fast: false
- matrix:
- # tests on windows are extremely unstable
- # os: [ ubuntu-latest, windows-2019 ]
- os: [ ubuntu-latest ]
- node-version: [ 16, 18 ]
-
- steps:
- - name: Cancel Workflow Action
- uses: styfle/[email protected]
- with:
- access_token: ${{ github.token }}
-
- - name: Checkout repository
- uses: actions/checkout@v3
-
- - name: Use Node.js ${{ matrix.node-version }}
- uses: actions/setup-node@v3
- with:
- node-version: ${{ matrix.node-version }}
- cache: 'npm'
- cache-dependency-path: 'package-lock.json'
-
- - uses: microsoft/playwright-github-action@v1
-
- - name: Turbo cache
- id: turbo-cache
- uses: actions/cache@v3
- with:
- path: .turbo
- key: turbo-${{ github.job }}-${{ github.ref_name }}-${{ github.sha }}
- restore-keys: |
- turbo-${{ github.job }}-${{ github.ref_name }}-
-
- - name: Install Dependencies
- run: npm ci
-
- - name: Build
- run: npm run ci:build
-
- - name: Tests
- run: npm test
-
- docs:
- name: Docs build
- if: ${{ !contains(github.event.head_commit.message, '[skip ci]') }}
- runs-on: ubuntu-latest
- steps:
- - name: Checkout Source code
- uses: actions/checkout@v3
-
- - name: Use Node.js 16
- uses: actions/setup-node@v3
- with:
- node-version: 16
- cache: 'npm'
- cache-dependency-path: 'package-lock.json'
-
- - name: Turbo cache
- id: turbo-cache
- uses: actions/cache@v3
- with:
- path: .turbo
- key: turbo-${{ github.job }}-${{ github.ref_name }}-${{ github.sha }}
- restore-keys: |
- turbo-${{ github.job }}-${{ github.ref_name }}-
-
- - name: Install Dependencies
- run: npm ci
-
- - name: Build & deploy docs
- run: |
- cd website
- npm ci
- npm run build
-
- lint:
- name: Lint
- runs-on: ubuntu-latest
-
- steps:
- - name: Checkout repository
- uses: actions/checkout@v3
-
- - name: Use Node.js 16
- uses: actions/setup-node@v3
- with:
- node-version: 16
- cache: 'npm'
- cache-dependency-path: 'package-lock.json'
-
- - name: Turbo cache
- id: turbo-cache
- uses: actions/cache@v3
- with:
- path: .turbo
- key: turbo-${{ github.job }}-${{ github.ref_name }}-${{ github.sha }}
- restore-keys: |
- turbo-${{ github.job }}-${{ github.ref_name }}-
-
- - name: Install Dependencies
- run: npm ci
-
- - name: ESLint
- run: npm run lint
-
- publish:
- name: Publish
- if: (github.ref == 'refs/heads/master' || github.event_name == 'release') && !contains(github.event.head_commit.message, '[skip ci]') && !contains(github.event.head_commit.message, 'docs:')
- runs-on: ubuntu-latest
- needs: [ lint, build_and_test ]
- steps:
- - name: Checkout Source code
- uses: actions/checkout@v3
-
- - name: Use Node.js 16
- uses: actions/setup-node@v3
- with:
- node-version: 16
- cache: 'npm'
- cache-dependency-path: 'package-lock.json'
-
- - name: Turbo cache
- id: turbo-cache
- uses: actions/cache@v3
- with:
- path: .turbo
- key: turbo-${{ github.job }}-${{ github.ref_name }}-${{ github.sha }}
- restore-keys: |
- turbo-${{ github.job }}-${{ github.ref_name }}-
-
- - name: Install Dependencies
- run: npm ci
-
- - name: Check for changes
- id: changed_packages
- run: |
- echo "::set-output name=changed_packages::$(node ./node_modules/.bin/lerna changed -p | wc -l | xargs)"
-
- - name: Check for changes 2
- if: steps.changed_packages.outputs.changed_packages == '0'
- run: |
- echo "no changes detected by lerna"
-
- - name: Build
- if: steps.changed_packages.outputs.changed_packages != '0'
- run: npm run build
-
- - name: Set Release Tag
- run: echo "RELEASE_TAG=$(if [ ${{ github.event_name }} = release ]; then echo latest; else echo next; fi)" >> $GITHUB_ENV
-
- - name: Release dev version for testing
- if: env.RELEASE_TAG == 'next' && steps.changed_packages.outputs.changed_packages != '0'
- run: |
- git config --global user.name 'Apify Release Bot'
- git config --global user.email '[email protected]'
- npx turbo run copy --force -- -- --canary --preid=beta
- git commit -am 'chore: bump canary versions [skip ci]'
-
- echo "access=public" >> .npmrc
- echo "//registry.npmjs.org/:_authToken=${NPM_TOKEN}" >> .npmrc
- npm run publish:next -- --yes --no-verify-access
- env:
- NPM_TOKEN: ${{ secrets.NPM_TOKEN }}
- GIT_USER: "[email protected]:${{ secrets.GH_TOKEN }}"
- GH_TOKEN: ${{ secrets.GH_TOKEN }}
-
- # trigger of docker image builds in the apify-actor-docker repo.
- - name: Collect Versions For Images
- id: versions
- run: |
- crawlee=`node -p "require('./packages/crawlee/package.json').version"`
- echo "::set-output name=crawlee::$crawlee"
- apify=`node -p "require('./package.json').devDependencies.apify"`
- echo "::set-output name=apify::$apify"
- puppeteer=`node -p "require('./package.json').devDependencies.puppeteer"`
- echo "::set-output name=puppeteer::$puppeteer"
- playwright=`node -p "require('./package.json').devDependencies.playwright"`
- echo "::set-output name=playwright::$playwright"
- env:
- NPM_TOKEN: ${{ secrets.NPM_TOKEN }}
- GIT_USER: "[email protected]:${{ secrets.GH_TOKEN }}"
- GH_TOKEN: ${{ secrets.GH_TOKEN }}
- - name: Trigger Docker Image Build
- uses: peter-evans/repository-dispatch@v2
- with:
- token: ${{ secrets.TRIGGER_DOCKER_IMAGE_BUILD_TOKEN }}
- repository: apify/apify-actor-docker
- event-type: build-node-images
- client-payload: >
- {
- "release_tag": "${{ env.RELEASE_TAG }}",
- "apify_version": "${{ steps.versions.outputs.apify }}",
- "crawlee_version": "${{ steps.versions.outputs.crawlee }}",
- "puppeteer_version": "${{ steps.versions.outputs.puppeteer }}",
- "playwright_version": "${{ steps.versions.outputs.playwright }}"
- }
diff --git a/.github/workflows/test-ci.yml b/.github/workflows/test-ci.yml
new file mode 100644
index 000000000..f2f41108a
--- /dev/null
+++ b/.github/workflows/test-ci.yml
@@ -0,0 +1,121 @@
+name: Check
+
+on:
+ push:
+ branches: [ master, renovate/** ]
+ pull_request:
+ branches: [ master ]
+
+jobs:
+ # NPM install is done in a separate job and cached to speed up the following jobs.
+ build_and_test:
+ name: Build & Test
+ if: (!contains(github.event.head_commit.message, '[skip ci]') && !contains(github.event.head_commit.message, 'docs:'))
+ runs-on: ${{ matrix.os }}
+
+ strategy:
+ fail-fast: false
+ matrix:
+ # tests on windows are extremely unstable
+ # os: [ ubuntu-latest, windows-2019 ]
+ os: [ ubuntu-latest ]
+ node-version: [ 16, 18 ]
+
+ steps:
+ - name: Cancel Workflow Action
+ uses: styfle/[email protected]
+ with:
+ access_token: ${{ github.token }}
+
+ - name: Checkout repository
+ uses: actions/checkout@v3
+
+ - name: Use Node.js ${{ matrix.node-version }}
+ uses: actions/setup-node@v3
+ with:
+ node-version: ${{ matrix.node-version }}
+ cache: 'npm'
+ cache-dependency-path: 'package-lock.json'
+
+ - uses: microsoft/playwright-github-action@v1
+
+ - name: Turbo cache
+ id: turbo-cache
+ uses: actions/cache@v3
+ with:
+ path: .turbo
+ key: turbo-${{ github.job }}-${{ github.ref_name }}-${{ github.sha }}
+ restore-keys: |
+ turbo-${{ github.job }}-${{ github.ref_name }}-
+
+ - name: Install Dependencies
+ run: npm ci
+
+ - name: Build
+ run: npm run ci:build
+
+ - name: Tests
+ run: npm test
+
+ docs:
+ name: Docs build
+ if: ${{ !contains(github.event.head_commit.message, '[skip ci]') }}
+ runs-on: ubuntu-latest
+ steps:
+ - name: Checkout Source code
+ uses: actions/checkout@v3
+
+ - name: Use Node.js 18
+ uses: actions/setup-node@v3
+ with:
+ node-version: 18
+ cache: 'npm'
+ cache-dependency-path: 'package-lock.json'
+
+ - name: Turbo cache
+ id: turbo-cache
+ uses: actions/cache@v3
+ with:
+ path: .turbo
+ key: turbo-${{ github.job }}-${{ github.ref_name }}-${{ github.sha }}
+ restore-keys: |
+ turbo-${{ github.job }}-${{ github.ref_name }}-
+
+ - name: Install Dependencies
+ run: npm ci
+
+ - name: Build & deploy docs
+ run: |
+ cd website
+ npm ci
+ npm run build
+
+ lint:
+ name: Lint
+ runs-on: ubuntu-latest
+
+ steps:
+ - name: Checkout repository
+ uses: actions/checkout@v3
+
+ - name: Use Node.js 18
+ uses: actions/setup-node@v3
+ with:
+ node-version: 18
+ cache: 'npm'
+ cache-dependency-path: 'package-lock.json'
+
+ - name: Turbo cache
+ id: turbo-cache
+ uses: actions/cache@v3
+ with:
+ path: .turbo
+ key: turbo-${{ github.job }}-${{ github.ref_name }}-${{ github.sha }}
+ restore-keys: |
+ turbo-${{ github.job }}-${{ github.ref_name }}-
+
+ - name: Install Dependencies
+ run: npm ci
+
+ - name: ESLint
+ run: npm run lint
diff --git a/package.json b/package.json
index ea9566231..ccb4c9c87 100644
--- a/package.json
+++ b/package.json
@@ -44,7 +44,6 @@
"test": "jest --silent",
"test:e2e": "node test/e2e/run.mjs",
"coverage": "jest --coverage",
- "release": "npm run build && lerna version patch && lerna publish from-package --contents dist --force-publish",
"publish:next": "lerna publish from-package --contents dist --dist-tag next --force-publish",
"release:next": "npm run build && npm run publish:next",
"lint": "eslint packages/browser-pool/tab-as-a-container packages/*/src packages/*/test test",