Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

ci: merge test flow with release next flow (#1651)

Vlad Frangu430199cec478
diff --git a/.github/workflows/release-next.yml b/.github/workflows/release-next.yml
deleted file mode 100644
index 9c482ad74..000000000
--- a/.github/workflows/release-next.yml
+++ /dev/null
@@ -1,145 +0,0 @@
-name: Release @next
-
-on:
- # Pushes to master release a new @next version
- push:
- branches:
- - master
-
-jobs:
- # Test before releasing to ensure we don't release a broken version
- build_and_test:
- name: Build & Test
- if: (!contains(github.event.head_commit.message, '[skip ci]') && !contains(github.event.head_commit.message, 'docs:'))
- strategy:
- fail-fast: true
- matrix:
- # We don't test on Windows as the tests are flaky
- os: [ubuntu-latest]
- node-version: [16, 18]
-
- runs-on: ${{ matrix.os }}
-
- steps:
- - name: Cancel Workflow Action
- uses: styfle/[email protected]
- with:
- access_token: ${{ github.token }}
-
- - name: Checkout repository
- uses: actions/checkout@v3
-
- - name: Use Node.js ${{ matrix.node-version }}
- uses: actions/setup-node@v3
- with:
- node-version: ${{ matrix.node-version }}
- cache: 'npm'
- cache-dependency-path: 'package-lock.json'
-
- - name: Setup Playwright
- uses: microsoft/playwright-github-action@v1
-
- - name: Turbo cache
- id: turbo-cache
- uses: actions/cache@v3
- with:
- path: .turbo
- key: turbo-${{ github.job }}-${{ github.ref_name }}-${{ github.sha }}
- restore-keys: |
- turbo-${{ github.job }}-${{ github.ref_name }}-
-
- - name: Install Dependencies
- run: npm ci
-
- - name: Build
- run: npm run ci:build
-
- - name: Tests
- run: npm test
-
- release_next:
- name: Release @next
- if: (!contains(github.event.head_commit.message, '[skip ci]') && !contains(github.event.head_commit.message, 'docs:'))
- needs: build_and_test
- runs-on: ubuntu-latest
-
- steps:
- - name: Checkout repository
- uses: actions/checkout@v3
-
- - name: Use Node.js 18
- uses: actions/setup-node@v3
- with:
- node-version: 18
- cache: 'npm'
- cache-dependency-path: 'package-lock.json'
-
- - name: Turbo cache
- id: turbo-cache
- uses: actions/cache@v3
- with:
- path: .turbo
- key: turbo-${{ github.job }}-${{ github.ref_name }}-${{ github.sha }}
- restore-keys: |
- turbo-${{ github.job }}-${{ github.ref_name }}-
-
- - name: Install Dependencies
- run: npm ci
-
- - name: Build
- run: npm run ci:build
-
- - name: Generate changed packages list
- id: changed-packages
- run: |
- echo "::set-output name=changed_packages::$(node ./node_modules/.bin/lerna changed -p | wc -l | xargs)"
-
- - name: Report nothing to release
- if: steps.changed-packages.outputs.changed_packages == '0'
- run: echo "Nothing to release"
-
- - name: Release @next
- if: steps.changed-packages.outputs.changed_packages != '0'
- run: |
- git config --global user.name 'Apify Release Bot'
- git config --global user.email '[email protected]'
- npx turbo copy --force -- -- --canary --preid=beta
- git commit -am "chore: bump canary versions [skip ci]"
-
- echo "access=public" > .npmrc
- echo "//registry.npmjs.org/:_authToken=${NPM_TOKEN}" >> .npmrc
- npm run publish:next -- --yes --no-verify-access
- env:
- NPM_TOKEN: ${{ secrets.NPM_TOKEN }}
- GIT_USER: "[email protected]:${{ secrets.GH_TOKEN }}"
- GH_TOKEN: ${{ secrets.GH_TOKEN }}
-
- - name: Collect versions for Docker images
- id: versions
- run: |
- crawlee=`node -p "require('./packages/crawlee/package.json').version"`
- echo "::set-output name=crawlee::$crawlee"
- apify=`node -p "require('./package.json').devDependencies.apify"`
- echo "::set-output name=apify::$apify"
- puppeteer=`node -p "require('./package.json').devDependencies.puppeteer"`
- echo "::set-output name=puppeteer::$puppeteer"
- playwright=`node -p "require('./package.json').devDependencies.playwright"`
- echo "::set-output name=playwright::$playwright"
-
- - name: Trigger Docker image builds
- uses: peter-evans/repository-dispatch@v2
- # Trigger next images only if we have something new pushed
- if: steps.changed-packages.outputs.changed_packages != '0'
- with:
- token: ${{ secrets.TRIGGER_DOCKER_IMAGE_BUILD_TOKEN }}
- repository: apify/apify-actor-docker
- event-type: build-node-images
- client-payload: >
- {
- "release_tag": "next",
- "apify_version": "${{ steps.versions.outputs.apify }}",
- "crawlee_version": "${{ steps.versions.outputs.crawlee }}",
- "puppeteer_version": "${{ steps.versions.outputs.puppeteer }}",
- "playwright_version": "${{ steps.versions.outputs.playwright }}"
- }
-
diff --git a/.github/workflows/test-ci.yml b/.github/workflows/test-ci.yml
index bbafe00c9..5084db106 100644
--- a/.github/workflows/test-ci.yml
+++ b/.github/workflows/test-ci.yml
@@ -119,3 +119,89 @@ jobs:
- name: ESLint
run: npm run lint
+
+ release_next:
+ name: Release @next
+ if: "github.event_name == 'push' && contains(github.event.ref, 'master') && (!contains(github.event.head_commit.message, '[skip ci]') && !contains(github.event.head_commit.message, 'docs:'))"
+ needs: build_and_test
+ runs-on: ubuntu-latest
+
+ steps:
+ - name: Checkout repository
+ uses: actions/checkout@v3
+
+ - name: Use Node.js 18
+ uses: actions/setup-node@v3
+ with:
+ node-version: 18
+ cache: 'npm'
+ cache-dependency-path: 'package-lock.json'
+
+ - name: Turbo cache
+ id: turbo-cache
+ uses: actions/cache@v3
+ with:
+ path: .turbo
+ key: turbo-${{ github.job }}-${{ github.ref_name }}-${{ github.sha }}
+ restore-keys: |
+ turbo-${{ github.job }}-${{ github.ref_name }}-
+
+ - name: Install Dependencies
+ run: npm ci
+
+ - name: Build
+ run: npm run ci:build
+
+ - name: Generate changed packages list
+ id: changed-packages
+ run: |
+ echo "::set-output name=changed_packages::$(node ./node_modules/.bin/lerna changed -p | wc -l | xargs)"
+
+ - name: Report nothing to release
+ if: steps.changed-packages.outputs.changed_packages == '0'
+ run: echo "Nothing to release"
+
+ - name: Release @next
+ if: steps.changed-packages.outputs.changed_packages != '0'
+ run: |
+ git config --global user.name 'Apify Release Bot'
+ git config --global user.email '[email protected]'
+ npx turbo copy --force -- -- --canary --preid=beta
+ git commit -am "chore: bump canary versions [skip ci]"
+
+ echo "access=public" > .npmrc
+ echo "//registry.npmjs.org/:_authToken=${NPM_TOKEN}" >> .npmrc
+ npm run publish:next -- --yes --no-verify-access
+ env:
+ NPM_TOKEN: ${{ secrets.NPM_TOKEN }}
+ GIT_USER: "[email protected]:${{ secrets.GH_TOKEN }}"
+ GH_TOKEN: ${{ secrets.GH_TOKEN }}
+
+ - name: Collect versions for Docker images
+ id: versions
+ run: |
+ crawlee=`node -p "require('./packages/crawlee/package.json').version"`
+ echo "::set-output name=crawlee::$crawlee"
+ apify=`node -p "require('./package.json').devDependencies.apify"`
+ echo "::set-output name=apify::$apify"
+ puppeteer=`node -p "require('./package.json').devDependencies.puppeteer"`
+ echo "::set-output name=puppeteer::$puppeteer"
+ playwright=`node -p "require('./package.json').devDependencies.playwright"`
+ echo "::set-output name=playwright::$playwright"
+
+ - name: Trigger Docker image builds
+ uses: peter-evans/repository-dispatch@v2
+ # Trigger next images only if we have something new pushed
+ if: steps.changed-packages.outputs.changed_packages != '0'
+ with:
+ token: ${{ secrets.TRIGGER_DOCKER_IMAGE_BUILD_TOKEN }}
+ repository: apify/apify-actor-docker
+ event-type: build-node-images
+ client-payload: >
+ {
+ "release_tag": "next",
+ "apify_version": "${{ steps.versions.outputs.apify }}",
+ "crawlee_version": "${{ steps.versions.outputs.crawlee }}",
+ "puppeteer_version": "${{ steps.versions.outputs.puppeteer }}",
+ "playwright_version": "${{ steps.versions.outputs.playwright }}"
+ }