Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
chore(ci): new automated docker build pipeline (#2171)
Vlad Frangu4e0362e10d2b
diff --git a/.github/workflows/docker-images.yml b/.github/workflows/docker-images.ymlnew file mode 100644index 000000000..38eb9aa6d--- /dev/null+++ b/.github/workflows/docker-images.yml@@ -0,0 +1,69 @@+name: Docker images++# The action runs on pushes to master and new releases only if their respective flows pass+on:+ workflow_dispatch:+ inputs:+ crawlee_version:+ description: Custom crawlee version to deploy+ type: string+ default: ''+ repository_dispatch:+ types:+ - crawlee-docker-image-bump+ schedule:+ - cron: 0 */12 * * *++env:+ ACTIONS_TOKEN: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }}+ CRAWLEE_BETA_VERSION: ${{ inputs.crawlee_version || '' }}++jobs:+ trigger_ci:+ name: Trigger CI+ runs-on: ubuntu-latest++ if: github.repository == 'apify/crawlee'++ steps:+ - name: Checkout repository+ uses: actions/checkout@v4+ with:+ token: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }}+ fetch-depth: 0++ - name: Use Bun+ uses: oven-sh/setup-bun@v1++ # No caching for now, bun's already mega fast+ # - name: Cache Bun Dependencies+ # uses: actions/cache@v3+ # with:+ # path: ./scripts/actions/docker-images/node_modules+ # key: actions-bun-${{ hashFiles('scripts/actions/docker-images/bun.lockb') }}++ - name: Install Bun Dependencies+ run: |+ pushd ./scripts/actions/docker-images+ bun install+ popd++ - name: Setup git user+ run: |+ git config --global user.name "Apify Release Bot"+ git config --global user.email "[email protected]"++ - name: Run Docker Action+ run: |+ pushd ./scripts/actions/docker-images+ bun run-ci+ popd+ env:+ ACTIONS_TOKEN: ${{ env.ACTIONS_TOKEN }}+ CRAWLEE_BETA_VERSION: ${{ env.CRAWLEE_BETA_VERSION }}++ - name: Update Docker state+ run: |+ git add .+ git diff-index --quiet HEAD || git commit -m "chore(docker): update docker state [skip ci]"+ git pushdiff --git a/.github/workflows/release.yml b/.github/workflows/release.ymlindex a10ebb0d7..74b672bf0 100644--- a/.github/workflows/release.yml+++ b/.github/workflows/release.yml@@ -9,10 +9,10 @@ on:default: 'patch'type: choiceoptions:- - "patch"- - "minor"- - "major"- - "custom"+ - 'patch'+ - 'minor'+ - 'major'+ - 'custom'custom_version:description: The custom version to bump to (only for "custom" type)required: false@@ -28,51 +28,51 @@ jobs:fail-fast: truematrix:# We don't test on Windows as the tests are flaky- os: [ ubuntu-latest ]- node-version: [ 16, 18, 20 ]+ os: [ubuntu-latest]+ node-version: [16, 18, 20]runs-on: ${{ matrix.os }}steps:- - name: Cancel Workflow Action- uses: styfle/[email protected]- with:- access_token: ${{ github.token }}-- - name: Checkout repository- uses: actions/checkout@v4-- - name: Use Node.js ${{ matrix.node-version }}- uses: actions/setup-node@v4- with:- node-version: ${{ matrix.node-version }}- cache: 'yarn'-- - name: Enable corepack- run: |- corepack enable- corepack prepare yarn@stable --activate-- - name: Setup Playwright- uses: microsoft/playwright-github-action@v1-- - name: Turbo cache- id: turbo-cache- uses: actions/cache@v3- with:- path: .turbo- key: turbo-${{ github.job }}-${{ github.ref_name }}-${{ github.sha }}- restore-keys: |- turbo-${{ github.job }}-${{ github.ref_name }}--- - name: Install Dependencies- run: yarn-- - name: Build- run: yarn ci:build-- - name: Tests- run: yarn test+ - name: Cancel Workflow Action+ uses: styfle/[email protected]+ with:+ access_token: ${{ github.token }}++ - name: Checkout repository+ uses: actions/checkout@v4++ - name: Use Node.js ${{ matrix.node-version }}+ uses: actions/setup-node@v4+ with:+ node-version: ${{ matrix.node-version }}+ cache: 'yarn'++ - name: Enable corepack+ run: |+ corepack enable+ corepack prepare yarn@stable --activate++ - name: Setup Playwright+ uses: microsoft/playwright-github-action@v1++ - name: Turbo cache+ id: turbo-cache+ uses: actions/cache@v3+ with:+ path: .turbo+ key: turbo-${{ github.job }}-${{ github.ref_name }}-${{ github.sha }}+ restore-keys: |+ turbo-${{ github.job }}-${{ github.ref_name }}-++ - name: Install Dependencies+ run: yarn++ - name: Build+ run: yarn ci:build++ - name: Tests+ run: yarn testrelease:name: "Bump Crawlee: ${{ inputs.version }} version (${{ inputs.custom_version || 'n/a' }} custom version)"@@ -81,100 +81,80 @@ jobs:runs-on: ubuntu-lateststeps:- - name: Checkout repository- uses: actions/checkout@v4- with:- token: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }}- fetch-depth: 0-- - name: Use Node.js 18- uses: actions/setup-node@v4- with:- node-version: 18- cache: 'yarn'-- - name: Enable corepack- run: |- corepack enable- corepack prepare yarn@stable --activate-- - name: Turbo cache- id: turbo-cache- uses: actions/cache@v3- with:- path: .turbo- key: turbo-${{ github.job }}-${{ github.ref_name }}-${{ github.sha }}- restore-keys: |- turbo-${{ github.job }}-${{ github.ref_name }}--- - name: Install Dependencies- run: yarn-- - name: Build- run: yarn ci:build-- - name: Setup git user and npm- run: |- git config --global user.name "Apify Release Bot"- git config --global user.email "[email protected]"-- echo "access=public" > .npmrc- echo "//registry.npmjs.org/:_authToken=${{ secrets.APIFY_SERVICE_ACCOUNT_NPM_TOKEN }}" >> .npmrc-- - name: Bump version to custom version- if: ${{ github.event.inputs.version == 'custom' && github.event.inputs.custom_version != '' }}- run: yarn lerna version ${{ github.event.inputs.custom_version }} --force-publish --yes- env:- NPM_TOKEN: ${{ secrets.APIFY_SERVICE_ACCOUNT_NPM_TOKEN }}- GIT_USER: "[email protected]:${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }}"- GH_TOKEN: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }}-- - name: Bump version to ${{ github.event.inputs.version }} version- if: ${{ github.event.inputs.version != 'custom' }}- run: yarn lerna version ${{ github.event.inputs.version }} --force-publish --yes- env:- NPM_TOKEN: ${{ secrets.APIFY_SERVICE_ACCOUNT_NPM_TOKEN }}- GIT_USER: "[email protected]:${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }}"- GH_TOKEN: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }}-- - name: Pin versions in internal dependencies and update lockfile- run: |- yarn release:pin-versions- yarn install --no-immutable- git add .- git diff-index --quiet HEAD || git commit -m 'chore(release): update internal dependencies [skip ci]'- git push-- - name: Publish packages- run: yarn publish:prod --yes --no-verify-access- env:- NPM_TOKEN: ${{ secrets.APIFY_SERVICE_ACCOUNT_NPM_TOKEN }}- GIT_USER: "[email protected]:${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }}"- GH_TOKEN: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }}-- - name: Collect versions for Docker images- id: versions- run: |- crawlee=`node -p "require('./packages/crawlee/package.json').version"`- echo "crawlee=$crawlee" >> $GITHUB_OUTPUT- apify=`node -p "require('./package.json').devDependencies.apify"`- echo "apify=$apify" >> $GITHUB_OUTPUT- puppeteer=`node -p "require('./package.json').devDependencies.puppeteer"`- echo "puppeteer=$puppeteer" >> $GITHUB_OUTPUT- playwright=`node -p "require('./package.json').devDependencies.playwright"`- echo "playwright=$playwright" >> $GITHUB_OUTPUT-- - name: Trigger Docker image builds- uses: peter-evans/repository-dispatch@v2- with:- token: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }}- repository: apify/apify-actor-docker- event-type: build-node-images- client-payload: >- {- "release_tag": "latest",- "apify_version": "${{ steps.versions.outputs.apify }}",- "crawlee_version": "${{ steps.versions.outputs.crawlee }}",- "puppeteer_version": "${{ steps.versions.outputs.puppeteer }}",- "playwright_version": "${{ steps.versions.outputs.playwright }}"- }+ - name: Checkout repository+ uses: actions/checkout@v4+ with:+ token: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }}+ fetch-depth: 0++ - name: Use Node.js 18+ uses: actions/setup-node@v4+ with:+ node-version: 18+ cache: 'yarn'++ - name: Enable corepack+ run: |+ corepack enable+ corepack prepare yarn@stable --activate++ - name: Turbo cache+ id: turbo-cache+ uses: actions/cache@v3+ with:+ path: .turbo+ key: turbo-${{ github.job }}-${{ github.ref_name }}-${{ github.sha }}+ restore-keys: |+ turbo-${{ github.job }}-${{ github.ref_name }}-++ - name: Install Dependencies+ run: yarn++ - name: Build+ run: yarn ci:build++ - name: Setup git user and npm+ run: |+ git config --global user.name "Apify Release Bot"+ git config --global user.email "[email protected]"++ echo "access=public" > .npmrc+ echo "//registry.npmjs.org/:_authToken=${{ secrets.APIFY_SERVICE_ACCOUNT_NPM_TOKEN }}" >> .npmrc++ - name: Bump version to custom version+ if: ${{ github.event.inputs.version == 'custom' && github.event.inputs.custom_version != '' }}+ run: yarn lerna version ${{ github.event.inputs.custom_version }} --force-publish --yes+ env:+ NPM_TOKEN: ${{ secrets.APIFY_SERVICE_ACCOUNT_NPM_TOKEN }}+ GIT_USER: '[email protected]:${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }}'+ GH_TOKEN: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }}++ - name: Bump version to ${{ github.event.inputs.version }} version+ if: ${{ github.event.inputs.version != 'custom' }}+ run: yarn lerna version ${{ github.event.inputs.version }} --force-publish --yes+ env:+ NPM_TOKEN: ${{ secrets.APIFY_SERVICE_ACCOUNT_NPM_TOKEN }}+ GIT_USER: '[email protected]:${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }}'+ GH_TOKEN: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }}++ - name: Pin versions in internal dependencies and update lockfile+ run: |+ yarn release:pin-versions+ yarn install --no-immutable+ git add .+ git diff-index --quiet HEAD || git commit -m 'chore(release): update internal dependencies [skip ci]'+ git push++ - name: Publish packages+ run: yarn publish:prod --yes --no-verify-access+ env:+ NPM_TOKEN: ${{ secrets.APIFY_SERVICE_ACCOUNT_NPM_TOKEN }}+ GIT_USER: '[email protected]:${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }}'+ GH_TOKEN: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }}++ - name: Trigger Docker image builds+ uses: peter-evans/repository-dispatch@v2+ with:+ token: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }}+ repository: apify/crawlee+ event-type: crawlee-docker-image-bumpdiff --git a/.github/workflows/test-ci.yml b/.github/workflows/test-ci.ymlindex c6130d09b..126e81d21 100644--- a/.github/workflows/test-ci.yml+++ b/.github/workflows/test-ci.yml@@ -2,9 +2,9 @@ name: Checkon:push:- branches: [ master, renovate/** ]+ branches: [master, renovate/**]pull_request:- branches: [ master ]+ branches: [master]env:YARN_IGNORE_NODE: 1@@ -21,208 +21,198 @@ jobs:matrix:# tests on windows are extremely unstable# os: [ ubuntu-latest, windows-2019 ]- os: [ ubuntu-latest ]- node-version: [ 16, 18, 20 ]+ os: [ubuntu-latest]+ node-version: [16, 18, 20]steps:- - name: Cancel Workflow Action- uses: styfle/[email protected]- with:- access_token: ${{ github.token }}+ - name: Cancel Workflow Action+ uses: styfle/[email protected]+ with:+ access_token: ${{ github.token }}- - name: Checkout repository- uses: actions/checkout@v4+ - name: Checkout repository+ uses: actions/checkout@v4- - name: Use Node.js ${{ matrix.node-version }}- uses: actions/setup-node@v4- with:- node-version: ${{ matrix.node-version }}- cache: 'yarn'+ - name: Use Node.js ${{ matrix.node-version }}+ uses: actions/setup-node@v4+ with:+ node-version: ${{ matrix.node-version }}+ cache: 'yarn'- - name: Enable corepack- run: |- corepack enable- corepack prepare yarn@stable --activate+ - name: Enable corepack+ run: |+ corepack enable+ corepack prepare yarn@stable --activate- - uses: microsoft/playwright-github-action@v1+ - uses: microsoft/playwright-github-action@v1- - name: Turbo cache- id: turbo-cache- uses: actions/cache@v3- with:- path: .turbo- key: turbo-${{ github.job }}-${{ github.ref_name }}-${{ github.sha }}- restore-keys: |- turbo-${{ github.job }}-${{ github.ref_name }}-+ - name: Turbo cache+ id: turbo-cache+ uses: actions/cache@v3+ with:+ path: .turbo+ key: turbo-${{ github.job }}-${{ github.ref_name }}-${{ github.sha }}+ restore-keys: |+ turbo-${{ github.job }}-${{ github.ref_name }}-- - name: Install Dependencies- run: yarn+ - name: Install Dependencies+ run: yarn- - name: Build- run: yarn ci:build+ - name: Build+ run: yarn ci:build- - name: Tests- run: yarn test+ - name: Tests+ run: yarn testdocs:name: Docs buildif: ${{ !contains(github.event.head_commit.message, '[skip ci]') }}runs-on: ubuntu-lateststeps:- - name: Checkout Source code- uses: actions/checkout@v4-- - name: Use Node.js 18- uses: actions/setup-node@v4- with:- node-version: 18- cache: 'yarn'-- - name: Enable corepack- run: |- corepack enable- corepack prepare yarn@stable --activate-- - name: Turbo cache- id: turbo-cache- uses: actions/cache@v3- with:- path: .turbo- key: turbo-${{ github.job }}-${{ github.ref_name }}-${{ github.sha }}- restore-keys: |- turbo-${{ github.job }}-${{ github.ref_name }}--- - name: Install Dependencies- run: yarn-- - name: Build & deploy docs- run: |- cd website- yarn- yarn build- env:- APIFY_SIGNING_TOKEN: ${{ secrets.APIFY_SIGNING_TOKEN }}+ - name: Checkout Source code+ uses: actions/checkout@v4++ - name: Use Node.js 18+ uses: actions/setup-node@v4+ with:+ node-version: 18+ cache: 'yarn'++ - name: Enable corepack+ run: |+ corepack enable+ corepack prepare yarn@stable --activate++ - name: Turbo cache+ id: turbo-cache+ uses: actions/cache@v3+ with:+ path: .turbo+ key: turbo-${{ github.job }}-${{ github.ref_name }}-${{ github.sha }}+ restore-keys: |+ turbo-${{ github.job }}-${{ github.ref_name }}-++ - name: Install Dependencies+ run: yarn++ - name: Build & deploy docs+ run: |+ cd website+ yarn+ yarn build+ env:+ APIFY_SIGNING_TOKEN: ${{ secrets.APIFY_SIGNING_TOKEN }}lint:name: Lintruns-on: ubuntu-lateststeps:- - name: Checkout repository- uses: actions/checkout@v4-- - name: Use Node.js 18- uses: actions/setup-node@v4- with:- node-version: 18- cache: 'yarn'-- - name: Enable corepack- run: |- corepack enable- corepack prepare yarn@stable --activate-- - name: Turbo cache- id: turbo-cache- uses: actions/cache@v3- with:- path: .turbo- key: turbo-${{ github.job }}-${{ github.ref_name }}-${{ github.sha }}- restore-keys: |- turbo-${{ github.job }}-${{ github.ref_name }}--- - name: Install Dependencies- run: yarn-- - name: ESLint- run: yarn lint+ - name: Checkout repository+ uses: actions/checkout@v4++ - name: Use Node.js 18+ uses: actions/setup-node@v4+ with:+ node-version: 18+ cache: 'yarn'++ - name: Enable corepack+ run: |+ corepack enable+ corepack prepare yarn@stable --activate++ - name: Turbo cache+ id: turbo-cache+ uses: actions/cache@v3+ with:+ path: .turbo+ key: turbo-${{ github.job }}-${{ github.ref_name }}-${{ github.sha }}+ restore-keys: |+ turbo-${{ github.job }}-${{ github.ref_name }}-++ - name: Install Dependencies+ run: yarn++ - name: ESLint+ run: yarn lintrelease_next:name: Release @next- if: "github.event_name == 'push' && contains(github.event.ref, 'master') && (!contains(github.event.head_commit.message, '[skip ci]') && !contains(github.event.head_commit.message, 'docs:'))"+ if: github.event_name == 'push' && contains(github.event.ref, 'master') && (!contains(github.event.head_commit.message, '[skip ci]') && !contains(github.event.head_commit.message, 'docs:'))needs: build_and_testruns-on: ubuntu-lateststeps:- - name: Checkout repository- uses: actions/checkout@v4-- - name: Use Node.js 18- uses: actions/setup-node@v4- with:- node-version: 18- cache: 'yarn'-- - name: Enable corepack- run: |- corepack enable- corepack prepare yarn@stable --activate-- - name: Turbo cache- id: turbo-cache- uses: actions/cache@v3- with:- path: .turbo- key: turbo-${{ github.job }}-${{ github.ref_name }}-${{ github.sha }}- restore-keys: |- turbo-${{ github.job }}-${{ github.ref_name }}--- - name: Install Dependencies- run: yarn-- - name: Build- run: yarn ci:build-- - name: Generate changed packages list- id: changed-packages- run: |- echo "changed_packages=$(node ./node_modules/.bin/lerna changed -p | wc -l | xargs)" >> $GITHUB_OUTPUT-- - name: Report nothing to release- if: steps.changed-packages.outputs.changed_packages == '0'- run: echo "Nothing to release"-- - name: Release @next- if: steps.changed-packages.outputs.changed_packages != '0'- run: |- git config --global user.name 'Apify Release Bot'- git config --global user.email '[email protected]'- yarn turbo copy --force -- --canary --preid=beta- git commit -am "chore: bump canary versions [skip ci]"-- echo "access=public" > .npmrc- echo "//registry.npmjs.org/:_authToken=${NPM_TOKEN}" >> .npmrc- yarn publish:next --yes --no-verify-access- env:- NPM_TOKEN: ${{ secrets.APIFY_SERVICE_ACCOUNT_NPM_TOKEN }}- GIT_USER: "[email protected]:${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }}"- GH_TOKEN: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }}-- - name: Collect versions for Docker images- id: versions- run: |- crawlee=`node -p "require('./packages/crawlee/package.json').version"`- echo "crawlee=$crawlee" >> $GITHUB_OUTPUT- apify=`node -p "require('./package.json').devDependencies.apify"`- echo "apify=$apify" >> $GITHUB_OUTPUT- puppeteer=`node -p "require('./package.json').devDependencies.puppeteer"`- echo "puppeteer=$puppeteer" >> $GITHUB_OUTPUT- playwright=`node -p "require('./package.json').devDependencies.playwright"`- echo "playwright=$playwright" >> $GITHUB_OUTPUT-- - name: Trigger Docker image builds- uses: peter-evans/repository-dispatch@v2- # Trigger next images only if we have something new pushed- if: steps.changed-packages.outputs.changed_packages != '0'- with:- token: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }}- repository: apify/apify-actor-docker- event-type: build-node-images- client-payload: >- {- "release_tag": "beta",- "apify_version": "${{ steps.versions.outputs.apify }}",- "crawlee_version": "${{ steps.versions.outputs.crawlee }}",- "puppeteer_version": "${{ steps.versions.outputs.puppeteer }}",- "playwright_version": "${{ steps.versions.outputs.playwright }}"- }+ - name: Checkout repository+ uses: actions/checkout@v4++ - name: Use Node.js 18+ uses: actions/setup-node@v4+ with:+ node-version: 18+ cache: 'yarn'++ - name: Enable corepack+ run: |+ corepack enable+ corepack prepare yarn@stable --activate++ - name: Turbo cache+ id: turbo-cache+ uses: actions/cache@v3+ with:+ path: .turbo+ key: turbo-${{ github.job }}-${{ github.ref_name }}-${{ github.sha }}+ restore-keys: |+ turbo-${{ github.job }}-${{ github.ref_name }}-++ - name: Install Dependencies+ run: yarn++ - name: Build+ run: yarn ci:build++ - name: Generate changed packages list+ id: changed-packages+ run: |+ echo "changed_packages=$(node ./node_modules/.bin/lerna changed -p | wc -l | xargs)" >> $GITHUB_OUTPUT++ - name: Report nothing to release+ if: steps.changed-packages.outputs.changed_packages == '0'+ run: echo "Nothing to release"++ - name: Release @next+ if: steps.changed-packages.outputs.changed_packages != '0'+ run: |+ git config --global user.name 'Apify Release Bot'+ git config --global user.email '[email protected]'+ yarn turbo copy --force -- --canary --preid=beta+ git commit -am "chore: bump canary versions [skip ci]"++ echo "access=public" > .npmrc+ echo "//registry.npmjs.org/:_authToken=${NPM_TOKEN}" >> .npmrc+ yarn publish:next --yes --no-verify-access+ env:+ NPM_TOKEN: ${{ secrets.APIFY_SERVICE_ACCOUNT_NPM_TOKEN }}+ GIT_USER: '[email protected]:${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }}'+ GH_TOKEN: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }}++ - name: Collect versions for Docker images+ id: versions+ run: |+ crawlee=`node -p "require('./packages/crawlee/package.json').version"`+ echo "crawlee=$crawlee" >> $GITHUB_OUTPUT++ - name: Trigger Docker image builds+ uses: peter-evans/repository-dispatch@v2+ # Trigger next images only if we have something new pushed+ if: steps.changed-packages.outputs.changed_packages != '0'+ with:+ token: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }}+ repository: apify/crawlee+ event-type: crawlee-docker-image-bump+ client-payload: >+ {+ "crawlee_version": "${{ steps.versions.outputs.crawlee }}"+ }diff --git a/scripts/actions/docker-images/api.ts b/scripts/actions/docker-images/api.tsnew file mode 100644index 000000000..d7f5e8191--- /dev/null+++ b/scripts/actions/docker-images/api.ts@@ -0,0 +1,115 @@+/// <reference types="bun-types" />++import { inspect } from 'node:util';++import * as core from '@actions/core';+import * as github from '@actions/github';+import { satisfies } from 'semver';++if (!process.env.ACTIONS_TOKEN) {+ throw new Error('ACTIONS_TOKEN not set');+}++core.debug('Creating octokit instance');+export const octokit = github.getOctokit(process.env.ACTIONS_TOKEN);++const minimumPlaywrightVersion = '>= 1.x';+const minimumPuppeteerVersion = '>= 10.x';++export async function fetchModuleVersions(module: string, limit?: number) {+ const res = await fetch(`https://registry.npmjs.org/${module}`, {+ headers: {+ accept: 'application/vnd.npm.install-v1+json',+ },+ });++ if (!res.ok) {+ throw new Error(`Failed to fetch ${module} versions`);+ }++ const json = await res.json();++ // Get all stable releases (they don't include a `-` in their version string)+ const versions = Object.keys(json.versions).filter((v) => !v.includes('-'));++ switch (module) {+ case 'playwright': {+ const filtered = versions.filter((v) => satisfies(v, minimumPlaywrightVersion));++ if (limit) {+ return filtered.slice(limit * -1);+ }++ return filtered;+ }+ case 'puppeteer': {+ const filtered = versions.filter((v) => satisfies(v, minimumPuppeteerVersion));++ if (limit) {+ return filtered.slice(limit * -1);+ }++ return filtered;+ }+ default: {+ if (limit) {+ return versions.slice(limit * -1);+ }++ return versions;+ }+ }+}++let owner = 'apify';+let repo = 'apify-actor-docker';++if (process.env.CUSTOM_DISPATCH_REPOSITORY) {+ [owner, repo] = process.env.CUSTOM_DISPATCH_REPOSITORY.split('/');+}++core.debug(`Repository: ${owner}/${repo}`);++export async function triggerAction({ eventType, ...inputs }: ActionInputs) {+ core.debug(`Inputs: ${inspect(inputs)}; eventType: ${eventType}`);++ try {+ await octokit.rest.repos.createDispatchEvent({+ owner,+ repo,+ event_type: eventType,+ client_payload: {+ ...inputs,+ },+ });+ } catch (err) {+ core.debug(inspect(err));+ if (err instanceof Error) {+ if ('status' in err && err.status === 404) {+ core.setFailed(`Token has missing permissions possibly`);+ } else {+ core.setFailed(err.message);+ }+ } else {+ core.setFailed(`Failed to trigger action: ${inspect(err)}`);+ }+ }+}++export interface ActionInputs {+ eventType: string;+ release_tag: 'latest' | 'beta';+ apify_version: string;+ crawlee_version: string;+ puppeteer_version: string;+ playwright_version: string;+ is_latest_browser_image: boolean;+}++export enum EventType {+ // Deploys all images, across all browsers. USE WITH CARE+ AllImages = 'build-node-images',+ Playwright = 'build-node-images-playwright',+ Puppeteer = 'build-node-images-puppeteer',+ Node = 'build-node-image-only',+}diff --git a/scripts/actions/docker-images/bun.lockb b/scripts/actions/docker-images/bun.lockbnew file mode 100755index 000000000..84ae96cf0Binary files /dev/null and b/scripts/actions/docker-images/bun.lockb differdiff --git a/scripts/actions/docker-images/main.ts b/scripts/actions/docker-images/main.tsnew file mode 100644index 000000000..d75ea9208--- /dev/null+++ b/scripts/actions/docker-images/main.ts@@ -0,0 +1,168 @@+/*+The main entry point of the docker action.++The following env variables are available:+- ACTIONS_TOKEN - required; a GitHub PAT+- CRAWLEE_BETA_VERSION - if set, the action will deploy to the beta channel, and for the specified crawlee version+- CUSTOM_DISPATCH_REPOSITORY - if set, the action will dispatch to the specified repository instead of apify/apify-actor-docker++Run with `bun run-ci` in the actions/docker-images directory.+*/++/// <reference types="bun-types" />++import { readFile, writeFile } from 'node:fs/promises';+import { join } from 'node:path';+import { inspect } from 'node:util';++import { debug, info } from '@actions/core';++import { fetchModuleVersions, type ActionInputs, EventType, triggerAction } from './api.ts';++const statePath = join(import.meta.dir, 'state.json');++interface StateFile {+ crawleeVersion?: string;+ puppeteerVersions?: string[];+ playwrightVersions?: string[];+}++const state = JSON.parse(await readFile(statePath, 'utf-8')) as StateFile;+debug(`State file: ${inspect(state)}`);++const crawleeVersion: string = JSON.parse(+ await readFile(new URL('../../../packages/crawlee/package.json', import.meta.url), 'utf-8'),+).version;+debug(`Crawlee version: ${crawleeVersion}`);++const apifyVersion: string = JSON.parse(await readFile(new URL('../../../package.json', import.meta.url), 'utf-8')).devDependencies?.apify+ ?? 'latest';+debug(`Apify version: ${apifyVersion}`);++const lastPlaywrightVersions = await fetchModuleVersions('playwright', 5);+debug(`Last 5 playwright versions: ${lastPlaywrightVersions.join(', ')}`);++const lastPuppeteerVersions = await fetchModuleVersions('puppeteer', 5);+debug(`Last 5 puppeteer versions: ${lastPuppeteerVersions.join(', ')}`);++const apiCalls: ActionInputs[] = [];+const newState: StateFile = {+ playwrightVersions: [],+ puppeteerVersions: [],+ crawleeVersion: state.crawleeVersion,+};++if (process.env.CRAWLEE_BETA_VERSION) {+ info(`👀 Crawlee beta version detected, deploying to beta channel`);+ debug(+ `Crawlee:${process.env.CRAWLEE_BETA_VERSION} Puppeteer:${lastPuppeteerVersions.at(+ -1,+ )} Playwright:${lastPlaywrightVersions.at(-1)} Apify:${apifyVersion}`,+ );++ apiCalls.push({+ eventType: EventType.AllImages,+ apify_version: 'beta',+ crawlee_version: process.env.CRAWLEE_BETA_VERSION,+ playwright_version: lastPlaywrightVersions.at(-1)!,+ puppeteer_version: lastPuppeteerVersions.at(-1)!,+ release_tag: 'beta',+ is_latest_browser_image: true,+ });++ // Keep the old state in place+ newState.playwrightVersions = state.playwrightVersions;+ newState.puppeteerVersions = state.puppeteerVersions;+} else {+ // Step 1. Adjust the playwright/puppeteer versions to the latest 5+ newState.playwrightVersions = lastPlaywrightVersions;+ newState.puppeteerVersions = lastPuppeteerVersions;++ // Step 2. If crawlee versions differ, we deploy images regardless of the browser versions+ newState.crawleeVersion = crawleeVersion;++ if (state.crawleeVersion === crawleeVersion) {+ // Step 3. Find all versions that are not yet deployed for each browser+ for (const [index, newPlaywrightVersion] of lastPlaywrightVersions.entries()) {+ if (!state.playwrightVersions?.includes(newPlaywrightVersion)) {+ info(`👀 New playwright version detected: ${newPlaywrightVersion}, scheduling for deploy`);+ apiCalls.push({+ eventType: EventType.Playwright,+ apify_version: apifyVersion,+ crawlee_version: crawleeVersion,+ playwright_version: newPlaywrightVersion,+ // Doesn't matter as this will only trigger playwright images+ puppeteer_version: '0.0.0',+ release_tag: 'latest',+ is_latest_browser_image: index === lastPlaywrightVersions.length - 1,+ });+ }+ }++ for (const [index, newPuppeteerVersion] of lastPuppeteerVersions.entries()) {+ if (!state.puppeteerVersions?.includes(newPuppeteerVersion)) {+ info(`👀 New puppeteer version detected: ${newPuppeteerVersion}, scheduling for deploy`);+ apiCalls.push({+ eventType: EventType.Puppeteer,+ apify_version: apifyVersion,+ crawlee_version: crawleeVersion,+ // Doesn't matter as this will only trigger puppeteer images+ playwright_version: '0.0.0',+ puppeteer_version: newPuppeteerVersion,+ release_tag: 'latest',+ is_latest_browser_image: index === lastPuppeteerVersions.length - 1,+ });+ }+ }+ } else {+ for (const [index, newPuppeteerVersion] of lastPuppeteerVersions.entries()) {+ info(`👀 Scheduling build for puppeteer: ${newPuppeteerVersion} and crawlee ${crawleeVersion} for deploy`);++ apiCalls.push({+ eventType: EventType.Puppeteer,+ apify_version: apifyVersion,+ crawlee_version: crawleeVersion,+ // Doesn't matter as this will only trigger puppeteer images+ playwright_version: '0.0.0',+ puppeteer_version: newPuppeteerVersion,+ release_tag: 'latest',+ is_latest_browser_image: index === lastPuppeteerVersions.length - 1,+ });+ }++ for (const [index, newPlaywrightVersion] of lastPlaywrightVersions.entries()) {+ info(+ `👀 Scheduling build for playwright: ${newPlaywrightVersion} and crawlee ${crawleeVersion} for deploy`,+ );++ apiCalls.push({+ eventType: EventType.Playwright,+ apify_version: apifyVersion,+ crawlee_version: crawleeVersion,+ playwright_version: newPlaywrightVersion,+ // Doesn't matter as this will only trigger playwright images+ puppeteer_version: '0.0.0',+ release_tag: 'latest',+ is_latest_browser_image: index === lastPlaywrightVersions.length - 1,+ });+ }++ info(`👀 Scheduling build for node image with crawlee ${crawleeVersion} for deploy`);+ apiCalls.push({+ eventType: EventType.Node,+ apify_version: apifyVersion,+ crawlee_version: crawleeVersion,+ is_latest_browser_image: true,+ playwright_version: '0.0.0',+ puppeteer_version: '0.0.0',+ release_tag: 'latest',+ });+ }+}++info(`🚀 Triggering ${apiCalls.length} actions`);+await Promise.all(apiCalls.map(async (data) => triggerAction(data)));+info(`📝 Updating state file`);+await writeFile(statePath, JSON.stringify(newState, null, '\t'));++info(`✅ Done`);diff --git a/scripts/actions/docker-images/package.json b/scripts/actions/docker-images/package.jsonnew file mode 100644index 000000000..1779cc238--- /dev/null+++ b/scripts/actions/docker-images/package.json@@ -0,0 +1,13 @@+{+ "dependencies": {+ "@actions/core": "^1.10.1",+ "@actions/github": "^6.0.0",+ "semver": "^7.5.4"+ },+ "scripts": {+ "run-ci": "bun ./main.ts"+ },+ "devDependencies": {+ "bun-types": "^1.0.9"+ }+}diff --git a/scripts/actions/docker-images/state.json b/scripts/actions/docker-images/state.jsonnew file mode 100644index 000000000..0967ef424--- /dev/null+++ b/scripts/actions/docker-images/state.json@@ -0,0 +1 @@+{}