Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
ci: merge test flow with release next flow (#1651)
Vlad Frangu430199cec478
diff --git a/.github/workflows/release-next.yml b/.github/workflows/release-next.ymldeleted file mode 100644index 9c482ad74..000000000--- a/.github/workflows/release-next.yml+++ /dev/null@@ -1,145 +0,0 @@-name: Release @next--on:- # Pushes to master release a new @next version- push:- branches:- - master--jobs:- # Test before releasing to ensure we don't release a broken version- build_and_test:- name: Build & Test- if: (!contains(github.event.head_commit.message, '[skip ci]') && !contains(github.event.head_commit.message, 'docs:'))- strategy:- fail-fast: true- matrix:- # We don't test on Windows as the tests are flaky- os: [ubuntu-latest]- node-version: [16, 18]-- runs-on: ${{ matrix.os }}-- steps:- - name: Cancel Workflow Action- uses: styfle/[email protected]- with:- access_token: ${{ github.token }}-- - name: Checkout repository- uses: actions/checkout@v3-- - name: Use Node.js ${{ matrix.node-version }}- uses: actions/setup-node@v3- with:- node-version: ${{ matrix.node-version }}- cache: 'npm'- cache-dependency-path: 'package-lock.json'-- - name: Setup Playwright- uses: microsoft/playwright-github-action@v1-- - name: Turbo cache- id: turbo-cache- uses: actions/cache@v3- with:- path: .turbo- key: turbo-${{ github.job }}-${{ github.ref_name }}-${{ github.sha }}- restore-keys: |- turbo-${{ github.job }}-${{ github.ref_name }}--- - name: Install Dependencies- run: npm ci-- - name: Build- run: npm run ci:build-- - name: Tests- run: npm test-- release_next:- name: Release @next- if: (!contains(github.event.head_commit.message, '[skip ci]') && !contains(github.event.head_commit.message, 'docs:'))- needs: build_and_test- runs-on: ubuntu-latest-- steps:- - name: Checkout repository- uses: actions/checkout@v3-- - name: Use Node.js 18- uses: actions/setup-node@v3- with:- node-version: 18- cache: 'npm'- cache-dependency-path: 'package-lock.json'-- - name: Turbo cache- id: turbo-cache- uses: actions/cache@v3- with:- path: .turbo- key: turbo-${{ github.job }}-${{ github.ref_name }}-${{ github.sha }}- restore-keys: |- turbo-${{ github.job }}-${{ github.ref_name }}--- - name: Install Dependencies- run: npm ci-- - name: Build- run: npm run ci:build-- - name: Generate changed packages list- id: changed-packages- run: |- echo "::set-output name=changed_packages::$(node ./node_modules/.bin/lerna changed -p | wc -l | xargs)"-- - name: Report nothing to release- if: steps.changed-packages.outputs.changed_packages == '0'- run: echo "Nothing to release"-- - name: Release @next- if: steps.changed-packages.outputs.changed_packages != '0'- run: |- git config --global user.name 'Apify Release Bot'- git config --global user.email '[email protected]'- npx turbo copy --force -- -- --canary --preid=beta- git commit -am "chore: bump canary versions [skip ci]"-- echo "access=public" > .npmrc- echo "//registry.npmjs.org/:_authToken=${NPM_TOKEN}" >> .npmrc- npm run publish:next -- --yes --no-verify-access- env:- NPM_TOKEN: ${{ secrets.NPM_TOKEN }}- GIT_USER: "[email protected]:${{ secrets.GH_TOKEN }}"- GH_TOKEN: ${{ secrets.GH_TOKEN }}-- - name: Collect versions for Docker images- id: versions- run: |- crawlee=`node -p "require('./packages/crawlee/package.json').version"`- echo "::set-output name=crawlee::$crawlee"- apify=`node -p "require('./package.json').devDependencies.apify"`- echo "::set-output name=apify::$apify"- puppeteer=`node -p "require('./package.json').devDependencies.puppeteer"`- echo "::set-output name=puppeteer::$puppeteer"- playwright=`node -p "require('./package.json').devDependencies.playwright"`- echo "::set-output name=playwright::$playwright"-- - name: Trigger Docker image builds- uses: peter-evans/repository-dispatch@v2- # Trigger next images only if we have something new pushed- if: steps.changed-packages.outputs.changed_packages != '0'- with:- token: ${{ secrets.TRIGGER_DOCKER_IMAGE_BUILD_TOKEN }}- repository: apify/apify-actor-docker- event-type: build-node-images- client-payload: >- {- "release_tag": "next",- "apify_version": "${{ steps.versions.outputs.apify }}",- "crawlee_version": "${{ steps.versions.outputs.crawlee }}",- "puppeteer_version": "${{ steps.versions.outputs.puppeteer }}",- "playwright_version": "${{ steps.versions.outputs.playwright }}"- }-diff --git a/.github/workflows/test-ci.yml b/.github/workflows/test-ci.ymlindex bbafe00c9..5084db106 100644--- a/.github/workflows/test-ci.yml+++ b/.github/workflows/test-ci.yml@@ -119,3 +119,89 @@ jobs:- name: ESLintrun: npm run lint++ release_next:+ name: Release @next+ if: "github.event_name == 'push' && contains(github.event.ref, 'master') && (!contains(github.event.head_commit.message, '[skip ci]') && !contains(github.event.head_commit.message, 'docs:'))"+ needs: build_and_test+ runs-on: ubuntu-latest++ steps:+ - name: Checkout repository+ uses: actions/checkout@v3++ - name: Use Node.js 18+ uses: actions/setup-node@v3+ with:+ node-version: 18+ cache: 'npm'+ cache-dependency-path: 'package-lock.json'++ - name: Turbo cache+ id: turbo-cache+ uses: actions/cache@v3+ with:+ path: .turbo+ key: turbo-${{ github.job }}-${{ github.ref_name }}-${{ github.sha }}+ restore-keys: |+ turbo-${{ github.job }}-${{ github.ref_name }}-++ - name: Install Dependencies+ run: npm ci++ - name: Build+ run: npm run ci:build++ - name: Generate changed packages list+ id: changed-packages+ run: |+ echo "::set-output name=changed_packages::$(node ./node_modules/.bin/lerna changed -p | wc -l | xargs)"++ - name: Report nothing to release+ if: steps.changed-packages.outputs.changed_packages == '0'+ run: echo "Nothing to release"++ - name: Release @next+ if: steps.changed-packages.outputs.changed_packages != '0'+ run: |+ git config --global user.name 'Apify Release Bot'+ git config --global user.email '[email protected]'+ npx turbo copy --force -- -- --canary --preid=beta+ git commit -am "chore: bump canary versions [skip ci]"++ echo "access=public" > .npmrc+ echo "//registry.npmjs.org/:_authToken=${NPM_TOKEN}" >> .npmrc+ npm run publish:next -- --yes --no-verify-access+ env:+ NPM_TOKEN: ${{ secrets.NPM_TOKEN }}+ GIT_USER: "[email protected]:${{ secrets.GH_TOKEN }}"+ GH_TOKEN: ${{ secrets.GH_TOKEN }}++ - name: Collect versions for Docker images+ id: versions+ run: |+ crawlee=`node -p "require('./packages/crawlee/package.json').version"`+ echo "::set-output name=crawlee::$crawlee"+ apify=`node -p "require('./package.json').devDependencies.apify"`+ echo "::set-output name=apify::$apify"+ puppeteer=`node -p "require('./package.json').devDependencies.puppeteer"`+ echo "::set-output name=puppeteer::$puppeteer"+ playwright=`node -p "require('./package.json').devDependencies.playwright"`+ echo "::set-output name=playwright::$playwright"++ - name: Trigger Docker image builds+ uses: peter-evans/repository-dispatch@v2+ # Trigger next images only if we have something new pushed+ if: steps.changed-packages.outputs.changed_packages != '0'+ with:+ token: ${{ secrets.TRIGGER_DOCKER_IMAGE_BUILD_TOKEN }}+ repository: apify/apify-actor-docker+ event-type: build-node-images+ client-payload: >+ {+ "release_tag": "next",+ "apify_version": "${{ steps.versions.outputs.apify }}",+ "crawlee_version": "${{ steps.versions.outputs.crawlee }}",+ "puppeteer_version": "${{ steps.versions.outputs.puppeteer }}",+ "playwright_version": "${{ steps.versions.outputs.playwright }}"+ }