Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

chore(ci): Set up OIDC trusted publishing (#3223)

Jan Bucharc8ee6f582a0a
diff --git a/.github/workflows/publish-to-npm.yml b/.github/workflows/publish-to-npm.yml
new file mode 100644
index 000000000..94f7453a2
--- /dev/null
+++ b/.github/workflows/publish-to-npm.yml
@@ -0,0 +1,75 @@
+name: Publish to NPM
+
+on:
+ workflow_call:
+ inputs:
+ ref:
+ description: Git ref to publish (branch, tag, or commit SHA)
+ required: true
+ type: string
+ dist-tag:
+ description: NPM dist-tag (prod or next)
+ required: false
+ type: string
+ default: 'prod'
+ workflow_dispatch:
+ inputs:
+ ref:
+ description: Git ref to publish (branch, tag, or commit SHA)
+ required: true
+ type: string
+ dist-tag:
+ description: NPM dist-tag (prod or next)
+ required: false
+ type: string
+ default: 'prod'
+
+permissions:
+ id-token: write # Required for OIDC
+ contents: write # Required to push updates
+
+jobs:
+ publish_to_npm:
+ name: Publish to NPM (${{ inputs.dist-tag }})
+ runs-on: ubuntu-22.04
+ steps:
+ - uses: actions/checkout@v5
+ with:
+ ref: ${{ inputs.ref }}
+ token: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }}
+ fetch-depth: 0
+ - name: Use Node.js 24
+ uses: actions/setup-node@v6
+ with:
+ node-version: 24
+ package-manager-cache: false
+ - name: Enable corepack
+ run: |
+ corepack enable
+ corepack prepare yarn@stable --activate
+ - name: Activate cache for Node.js 24
+ uses: actions/setup-node@v6
+ with:
+ cache: 'yarn'
+ - name: Turbo cache
+ id: turbo-cache
+ uses: actions/cache@v4
+ with:
+ path: .turbo
+ key: turbo-${{ github.job }}-${{ github.ref_name }}-${{ github.sha }}
+ restore-keys: |
+ turbo-${{ github.job }}-${{ github.ref_name }}-
+ - name: Install dependencies
+ run: yarn
+ - name: Build packages
+ run: yarn ci:build
+ - name: Publish to NPM (@latest)
+ if: inputs.dist-tag == 'prod'
+ run: yarn publish:prod --yes --no-verify-access
+ env:
+ GH_TOKEN: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }}
+ - name: Publish to NPM (@next)
+ if: inputs.dist-tag == 'next'
+ run: yarn publish:next --yes --no-verify-access
+ env:
+ GH_TOKEN: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }}
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index beacc15b7..237cd72a7 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -126,44 +126,40 @@ jobs:
- name: Build
run: yarn ci:build
- - name: Setup git user and npm
- run: |
- git config --global user.name "Apify Release Bot"
- git config --global user.email "[email protected]"
-
- echo "access=public" > .npmrc
- echo "//registry.npmjs.org/:_authToken=${{ secrets.APIFY_SERVICE_ACCOUNT_NPM_TOKEN }}" >> .npmrc
-
- name: Bump version to custom version
if: ${{ github.event.inputs.version == 'custom' && github.event.inputs.custom_version != '' }}
run: yarn lerna version ${{ github.event.inputs.custom_version }} --force-publish --yes
env:
- NPM_TOKEN: ${{ secrets.APIFY_SERVICE_ACCOUNT_NPM_TOKEN }}
- GIT_USER: '[email protected]:${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }}'
GH_TOKEN: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }}
- name: Bump version to ${{ github.event.inputs.version }} version
if: ${{ github.event.inputs.version != 'custom' }}
run: yarn lerna version ${{ github.event.inputs.version }} --force-publish --yes
env:
- NPM_TOKEN: ${{ secrets.APIFY_SERVICE_ACCOUNT_NPM_TOKEN }}
- GIT_USER: '[email protected]:${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }}'
GH_TOKEN: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }}
- name: Pin versions in internal dependencies and update lockfile
run: |
yarn release:pin-versions
yarn install --no-immutable
- git add .
- git diff-index --quiet HEAD || git commit -m 'chore(release): update internal dependencies [skip ci]'
- git push
+
+ - name: Commit changes
+ id: commit
+ uses: EndBug/add-and-commit@v9
+ with:
+ author_name: Apify Release Bot
+ author_email: [email protected]
+ message: 'chore(release): update internal dependencies [skip ci]'
- name: Publish packages
- run: yarn publish:prod --yes --no-verify-access
- env:
- NPM_TOKEN: ${{ secrets.APIFY_SERVICE_ACCOUNT_NPM_TOKEN }}
- GIT_USER: '[email protected]:${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }}'
- GH_TOKEN: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }}
+ uses: apify/workflows/execute-workflow@main
+ with:
+ workflow: publish-to-npm.yml
+ inputs: >
+ {
+ "ref": ${{ steps.commit.outputs.commit_long_sha || github.sha }},
+ "dist-tag": "prod"
+ }
- name: Trigger Docker image builds
uses: peter-evans/repository-dispatch@v4
diff --git a/.github/workflows/test-ci.yml b/.github/workflows/test-ci.yml
index 6c9d39aac..217087ee6 100644
--- a/.github/workflows/test-ci.yml
+++ b/.github/workflows/test-ci.yml
@@ -177,8 +177,11 @@ jobs:
steps:
- name: Checkout repository
uses: actions/checkout@v5
+ with:
+ token: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }}
+ fetch-depth: 0
- - name: Use Node.js 20
+ - name: Use Node.js 24
uses: actions/setup-node@v6
with:
node-version: 24
@@ -189,7 +192,7 @@ jobs:
corepack enable
corepack prepare yarn@stable --activate
- - name: Activate cache for Node.js 20
+ - name: Activate cache for Node.js 24
uses: actions/setup-node@v6
with:
cache: 'yarn'
@@ -218,21 +221,29 @@ jobs:
if: steps.changed-packages.outputs.changed_packages == '0'
run: echo "Nothing to release"
- - name: Release @next
+ - name: Bump canary versions
if: steps.changed-packages.outputs.changed_packages != '0'
run: |
- git config --global user.name 'Apify Release Bot'
- git config --global user.email '[email protected]'
yarn turbo copy --force -- --canary --preid=beta
- git commit -am "chore: bump canary versions [skip ci]"
- echo "access=public" > .npmrc
- echo "//registry.npmjs.org/:_authToken=${NPM_TOKEN}" >> .npmrc
- yarn publish:next --yes --no-verify-access
- env:
- NPM_TOKEN: ${{ secrets.APIFY_SERVICE_ACCOUNT_NPM_TOKEN }}
- GIT_USER: '[email protected]:${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }}'
- GH_TOKEN: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }}
+ - name: Commit changes
+ uses: EndBug/add-and-commit@v9
+ id: commit
+ with:
+ author_name: Apify Release Bot
+ author_email: [email protected]
+ message: 'chore: bump canary versions [skip ci]'
+
+ - name: Publish packages
+ if: steps.changed-packages.outputs.changed_packages != '0'
+ uses: apify/workflows/execute-workflow@main
+ with:
+ workflow: publish-to-npm.yml
+ inputs: >
+ {
+ "ref": ${{ steps.commit.outputs.commit_long_sha || github.sha }},
+ "dist-tag": "next"
+ }
- name: Collect versions for Docker images
id: versions