Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
chore: try to fix canary and latest builds
Martin Adámek3a03cff4a9d3
diff --git a/.github/workflows/release.yml b/.github/workflows/release.ymlindex 6ca3259de..a22b6d1a7 100644--- a/.github/workflows/release.yml+++ b/.github/workflows/release.yml@@ -1,169 +1,172 @@name: Release @lateston:- # Manual call from UI / automations(?)- workflow_dispatch:- inputs:- version:- description: The version to bump (if you choose custom, please include it under custom version)- required: true- default: 'patch'- type: choice- options:- - "patch"- - "minor"- - "major"- - "custom"- custom_version:- description: The custom version to bump to (only for "custom" type)- required: false- type: string- default: ''+ workflow_dispatch:+ inputs:+ version:+ description: The version to bump (if you choose custom, please include it under custom version)+ required: true+ default: 'patch'+ type: choice+ options:+ - "patch"+ - "minor"+ - "major"+ - "custom"+ custom_version:+ description: The custom version to bump to (only for "custom" type)+ required: false+ type: string+ default: ''jobs:- # Test before releasing to ensure we don't release a broken version- build_and_test:- name: Build & Test- if: (!contains(github.event.head_commit.message, '[skip ci]') && !contains(github.event.head_commit.message, 'docs:'))- strategy:- fail-fast: true- matrix:- # We don't test on Windows as the tests are flaky- os: [ubuntu-latest]- node-version: [16, 18]-- runs-on: ${{ matrix.os }}-- steps:- - name: Cancel Workflow Action- uses: styfle/[email protected]- with:- access_token: ${{ github.token }}-- - name: Checkout repository- uses: actions/checkout@v3-- - name: Use Node.js ${{ matrix.node-version }}- uses: actions/setup-node@v3- with:- node-version: ${{ matrix.node-version }}- cache: 'yarn'- cache-dependency-path: 'yarn.lock'-- - name: Enable corepack- run: |- corepack enable- corepack prepare yarn@stable --activate-- - name: Setup Playwright- uses: microsoft/playwright-github-action@v1-- - name: Turbo cache- id: turbo-cache- uses: actions/cache@v3- with:- path: .turbo- key: turbo-${{ github.job }}-${{ github.ref_name }}-${{ github.sha }}- restore-keys: |- turbo-${{ github.job }}-${{ github.ref_name }}--- - name: Install Dependencies- run: yarn-- - name: Build- run: yarn ci:build-- - name: Tests- run: yarn test-- release:- name: "Bump Crawlee: ${{ inputs.version }} version (${{ inputs.custom_version || 'n/a' }} custom version)"- if: (!contains(github.event.head_commit.message, '[skip ci]') && !contains(github.event.head_commit.message, 'docs:'))- needs: build_and_test- runs-on: ubuntu-latest-- steps:- - name: Checkout repository- uses: actions/checkout@v3- with:- fetch-depth: 0-- - name: Use Node.js 18- uses: actions/setup-node@v3- with:- node-version: 18- cache: 'yarn'- cache-dependency-path: 'yarn.lock'-- - name: Turbo cache- id: turbo-cache- uses: actions/cache@v3- with:- path: .turbo- key: turbo-${{ github.job }}-${{ github.ref_name }}-${{ github.sha }}- restore-keys: |- turbo-${{ github.job }}-${{ github.ref_name }}--- - name: Install Dependencies- run: yarn-- - name: Build packages- run: yarn build-- - name: Setup git user and npm- run: |- git config --global user.name "Apify Release Bot"- git config --global user.email "[email protected]"-- echo "access=public" > .npmrc- echo "//registry.npmjs.org/:_authToken=${{ secrets.NPM_TOKEN }}" >> .npmrc-- - name: Bump version to custom version- if: ${{ github.event.inputs.version == 'custom' && github.event.inputs.custom_version != '' }}- run: yarn version ${{ github.event.inputs.custom_version}} --force-publish --yes- env:- NPM_TOKEN: ${{ secrets.NPM_TOKEN }}- GIT_USER: "[email protected]:${{ secrets.GH_TOKEN }}"- GH_TOKEN: ${{ secrets.GH_TOKEN }}-- - name: Bump version to ${{ github.event.inputs.version }} version- if: ${{ github.event.inputs.version != 'custom' }}- run: yarn version ${{ github.event.inputs.version}} --force-publish --yes- env:- NPM_TOKEN: ${{ secrets.NPM_TOKEN }}- GIT_USER: "[email protected]:${{ secrets.GH_TOKEN }}"- GH_TOKEN: ${{ secrets.GH_TOKEN }}-- - name: Publish packages- run: yarn publish --yes --no-verify-access- env:- NPM_TOKEN: ${{ secrets.NPM_TOKEN }}- GIT_USER: "[email protected]:${{ secrets.GH_TOKEN }}"- GH_TOKEN: ${{ secrets.GH_TOKEN }}-- - name: Collect versions for Docker images- id: versions- run: |- crawlee=`node -p "require('./packages/crawlee/package.json').version"`- echo "crawlee=$crawlee" >> $GITHUB_OUTPUT- apify=`node -p "require('./package.json').devDependencies.apify"`- echo "apify=$apify" >> $GITHUB_OUTPUT- puppeteer=`node -p "require('./package.json').devDependencies.puppeteer"`- echo "puppeteer=$puppeteer" >> $GITHUB_OUTPUT- playwright=`node -p "require('./package.json').devDependencies.playwright"`- echo "playwright=$playwright" >> $GITHUB_OUTPUT-- - name: Trigger Docker image builds- uses: peter-evans/repository-dispatch@v2- with:- token: ${{ secrets.TRIGGER_DOCKER_IMAGE_BUILD_TOKEN }}- repository: apify/apify-actor-docker- event-type: build-node-images- client-payload: >- {- "release_tag": "latest",- "apify_version": "${{ steps.versions.outputs.apify }}",- "crawlee_version": "${{ steps.versions.outputs.crawlee }}",- "puppeteer_version": "${{ steps.versions.outputs.puppeteer }}",- "playwright_version": "${{ steps.versions.outputs.playwright }}"- }+ # Test before releasing to ensure we don't release a broken version+ build_and_test:+ name: Build & Test+ if: (!contains(github.event.head_commit.message, '[skip ci]') && !contains(github.event.head_commit.message, 'docs:'))+ strategy:+ fail-fast: true+ matrix:+ # We don't test on Windows as the tests are flaky+ os: [ ubuntu-latest ]+ node-version: [ 16, 18 ]++ runs-on: ${{ matrix.os }}++ steps:+ - name: Cancel Workflow Action+ uses: styfle/[email protected]+ with:+ access_token: ${{ github.token }}++ - name: Checkout repository+ uses: actions/checkout@v3++ - name: Use Node.js ${{ matrix.node-version }}+ uses: actions/setup-node@v3+ with:+ node-version: ${{ matrix.node-version }}+ cache: 'yarn'++ - name: Enable corepack+ run: |+ corepack enable+ corepack prepare yarn@stable --activate++ - name: Setup Playwright+ uses: microsoft/playwright-github-action@v1++ - name: Turbo cache+ id: turbo-cache+ uses: actions/cache@v3+ with:+ path: .turbo+ key: turbo-${{ github.job }}-${{ github.ref_name }}-${{ github.sha }}+ restore-keys: |+ turbo-${{ github.job }}-${{ github.ref_name }}-++ - name: Install Dependencies+ run: yarn++ - name: Build+ run: yarn ci:build++ - name: Tests+ run: yarn test++ release:+ name: "Bump Crawlee: ${{ inputs.version }} version (${{ inputs.custom_version || 'n/a' }} custom version)"+ if: (!contains(github.event.head_commit.message, '[skip ci]') && !contains(github.event.head_commit.message, 'docs:'))+ needs: build_and_test+ runs-on: ubuntu-latest++ steps:+ - name: Checkout repository+ uses: actions/checkout@v3+ with:+ token: ${{ secrets.GH_TOKEN }}+ fetch-depth: 0++ - name: Use Node.js 18+ uses: actions/setup-node@v3+ with:+ node-version: 18+ cache: 'yarn'++ - name: Enable corepack+ run: |+ corepack enable+ corepack prepare yarn@stable --activate++ - name: Turbo cache+ id: turbo-cache+ uses: actions/cache@v3+ with:+ path: .turbo+ key: turbo-${{ github.job }}-${{ github.ref_name }}-${{ github.sha }}+ restore-keys: |+ turbo-${{ github.job }}-${{ github.ref_name }}-++ - name: Install Dependencies+ run: yarn++ - name: Build+ run: yarn ci:build++ - name: Setup git user and npm+ run: |+ git config --global user.name "Apify Release Bot"+ git config --global user.email "[email protected]"++ echo "access=public" > .npmrc+ echo "//registry.npmjs.org/:_authToken=${{ secrets.NPM_TOKEN }}" >> .npmrc++ - name: Bump version to custom version+ if: ${{ github.event.inputs.version == 'custom' && github.event.inputs.custom_version != '' }}+ run: yarn lerna version ${{ github.event.inputs.custom_version }} --force-publish --yes+ env:+ NPM_TOKEN: ${{ secrets.NPM_TOKEN }}+ GIT_USER: "[email protected]:${{ secrets.GH_TOKEN }}"+ GH_TOKEN: ${{ secrets.GH_TOKEN }}++ - name: Bump version to ${{ github.event.inputs.version }} version+ if: ${{ github.event.inputs.version != 'custom' }}+ run: yarn lerna version ${{ github.event.inputs.version }} --force-publish --yes+ env:+ NPM_TOKEN: ${{ secrets.NPM_TOKEN }}+ GIT_USER: "[email protected]:${{ secrets.GH_TOKEN }}"+ GH_TOKEN: ${{ secrets.GH_TOKEN }}++ - name: Publish packages+ run: yarn publish:prod --yes --no-verify-access+ env:+ NPM_TOKEN: ${{ secrets.NPM_TOKEN }}+ GIT_USER: "[email protected]:${{ secrets.GH_TOKEN }}"+ GH_TOKEN: ${{ secrets.GH_TOKEN }}++ - name: Collect versions for Docker images+ id: versions+ run: |+ crawlee=`node -p "require('./packages/crawlee/package.json').version"`+ echo "crawlee=$crawlee" >> $GITHUB_OUTPUT+ apify=`node -p "require('./package.json').devDependencies.apify"`+ echo "apify=$apify" >> $GITHUB_OUTPUT+ puppeteer=`node -p "require('./package.json').devDependencies.puppeteer"`+ echo "puppeteer=$puppeteer" >> $GITHUB_OUTPUT+ playwright=`node -p "require('./package.json').devDependencies.playwright"`+ echo "playwright=$playwright" >> $GITHUB_OUTPUT++ - name: Trigger Docker image builds+ uses: peter-evans/repository-dispatch@v2+ with:+ token: ${{ secrets.TRIGGER_DOCKER_IMAGE_BUILD_TOKEN }}+ repository: apify/apify-actor-docker+ event-type: build-node-images+ client-payload: >+ {+ "release_tag": "latest",+ "apify_version": "${{ steps.versions.outputs.apify }}",+ "crawlee_version": "${{ steps.versions.outputs.crawlee }}",+ "puppeteer_version": "${{ steps.versions.outputs.puppeteer }}",+ "playwright_version": "${{ steps.versions.outputs.playwright }}"+ }diff --git a/package.json b/package.jsonindex 076b2dfc3..adaeed5e9 100644--- a/package.json+++ b/package.json@@ -36,19 +36,16 @@"postinstall": "npx husky install","prepublishOnly": "turbo run copy","clean": "turbo run clean",- "build": "turbo run build",- "postbuild": "node ./scripts/typescript_fixes.mjs",- "ci:build": "turbo run build --cache-dir=\".turbo\"",- "postci:build": "node ./scripts/typescript_fixes.mjs",+ "build": "turbo run build && node ./scripts/typescript_fixes.mjs",+ "ci:build": "turbo run build --cache-dir=\".turbo\" && node ./scripts/typescript_fixes.mjs","test": "jest --silent","test:e2e": "node test/e2e/run.mjs","test:full": "cross-env CRAWLEE_DIFFICULT_TESTS=1 jest --silent","coverage": "jest --coverage","publish:next": "lerna publish from-package --contents dist --dist-tag next --force-publish","release:next": "yarn build && yarn publish:next",- "version": "lerna version",- "postversion": "turbo run copy --force",- "publish": "lerna publish from-package --contents dist --force-publish",+ "publish:prod": "lerna publish from-package --contents dist --force-publish",+ "release:prod": "yarn build && yarn publish:prod","lint": "eslint \"packages/**/*.ts\" \"test/**/*.ts\"","lint:fix": "eslint \"packages/**/*.ts\" \"test/**/*.ts\" --fix"},