Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
chore(ci): Set up OIDC trusted publishing (#3223)
Jan Bucharc8ee6f582a0a
diff --git a/.github/workflows/publish-to-npm.yml b/.github/workflows/publish-to-npm.ymlnew file mode 100644index 000000000..94f7453a2--- /dev/null+++ b/.github/workflows/publish-to-npm.yml@@ -0,0 +1,75 @@+name: Publish to NPM++on:+ workflow_call:+ inputs:+ ref:+ description: Git ref to publish (branch, tag, or commit SHA)+ required: true+ type: string+ dist-tag:+ description: NPM dist-tag (prod or next)+ required: false+ type: string+ default: 'prod'+ workflow_dispatch:+ inputs:+ ref:+ description: Git ref to publish (branch, tag, or commit SHA)+ required: true+ type: string+ dist-tag:+ description: NPM dist-tag (prod or next)+ required: false+ type: string+ default: 'prod'++permissions:+ id-token: write # Required for OIDC+ contents: write # Required to push updates++jobs:+ publish_to_npm:+ name: Publish to NPM (${{ inputs.dist-tag }})+ runs-on: ubuntu-22.04+ steps:+ - uses: actions/checkout@v5+ with:+ ref: ${{ inputs.ref }}+ token: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }}+ fetch-depth: 0+ - name: Use Node.js 24+ uses: actions/setup-node@v6+ with:+ node-version: 24+ package-manager-cache: false+ - name: Enable corepack+ run: |+ corepack enable+ corepack prepare yarn@stable --activate+ - name: Activate cache for Node.js 24+ uses: actions/setup-node@v6+ with:+ cache: 'yarn'+ - name: Turbo cache+ id: turbo-cache+ uses: actions/cache@v4+ with:+ path: .turbo+ key: turbo-${{ github.job }}-${{ github.ref_name }}-${{ github.sha }}+ restore-keys: |+ turbo-${{ github.job }}-${{ github.ref_name }}-+ - name: Install dependencies+ run: yarn+ - name: Build packages+ run: yarn ci:build+ - name: Publish to NPM (@latest)+ if: inputs.dist-tag == 'prod'+ run: yarn publish:prod --yes --no-verify-access+ env:+ GH_TOKEN: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }}+ - name: Publish to NPM (@next)+ if: inputs.dist-tag == 'next'+ run: yarn publish:next --yes --no-verify-access+ env:+ GH_TOKEN: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }}diff --git a/.github/workflows/release.yml b/.github/workflows/release.ymlindex beacc15b7..237cd72a7 100644--- a/.github/workflows/release.yml+++ b/.github/workflows/release.yml@@ -126,44 +126,40 @@ jobs:- name: Buildrun: yarn ci:build- - name: Setup git user and npm- run: |- git config --global user.name "Apify Release Bot"- git config --global user.email "[email protected]"-- echo "access=public" > .npmrc- echo "//registry.npmjs.org/:_authToken=${{ secrets.APIFY_SERVICE_ACCOUNT_NPM_TOKEN }}" >> .npmrc-- name: Bump version to custom versionif: ${{ github.event.inputs.version == 'custom' && github.event.inputs.custom_version != '' }}run: yarn lerna version ${{ github.event.inputs.custom_version }} --force-publish --yesenv:- NPM_TOKEN: ${{ secrets.APIFY_SERVICE_ACCOUNT_NPM_TOKEN }}- GIT_USER: '[email protected]:${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }}'GH_TOKEN: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }}- name: Bump version to ${{ github.event.inputs.version }} versionif: ${{ github.event.inputs.version != 'custom' }}run: yarn lerna version ${{ github.event.inputs.version }} --force-publish --yesenv:- NPM_TOKEN: ${{ secrets.APIFY_SERVICE_ACCOUNT_NPM_TOKEN }}- GIT_USER: '[email protected]:${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }}'GH_TOKEN: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }}- name: Pin versions in internal dependencies and update lockfilerun: |yarn release:pin-versionsyarn install --no-immutable- git add .- git diff-index --quiet HEAD || git commit -m 'chore(release): update internal dependencies [skip ci]'- git push++ - name: Commit changes+ id: commit+ uses: EndBug/add-and-commit@v9+ with:+ author_name: Apify Release Bot+ author_email: [email protected]+ message: 'chore(release): update internal dependencies [skip ci]'- name: Publish packages- run: yarn publish:prod --yes --no-verify-access- env:- NPM_TOKEN: ${{ secrets.APIFY_SERVICE_ACCOUNT_NPM_TOKEN }}- GIT_USER: '[email protected]:${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }}'- GH_TOKEN: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }}+ uses: apify/workflows/execute-workflow@main+ with:+ workflow: publish-to-npm.yml+ inputs: >+ {+ "ref": ${{ steps.commit.outputs.commit_long_sha || github.sha }},+ "dist-tag": "prod"+ }- name: Trigger Docker image buildsuses: peter-evans/repository-dispatch@v4diff --git a/.github/workflows/test-ci.yml b/.github/workflows/test-ci.ymlindex 6c9d39aac..217087ee6 100644--- a/.github/workflows/test-ci.yml+++ b/.github/workflows/test-ci.yml@@ -177,8 +177,11 @@ jobs:steps:- name: Checkout repositoryuses: actions/checkout@v5+ with:+ token: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }}+ fetch-depth: 0- - name: Use Node.js 20+ - name: Use Node.js 24uses: actions/setup-node@v6with:node-version: 24@@ -189,7 +192,7 @@ jobs:corepack enablecorepack prepare yarn@stable --activate- - name: Activate cache for Node.js 20+ - name: Activate cache for Node.js 24uses: actions/setup-node@v6with:cache: 'yarn'@@ -218,21 +221,29 @@ jobs:if: steps.changed-packages.outputs.changed_packages == '0'run: echo "Nothing to release"- - name: Release @next+ - name: Bump canary versionsif: steps.changed-packages.outputs.changed_packages != '0'run: |- git config --global user.name 'Apify Release Bot'- git config --global user.email '[email protected]'yarn turbo copy --force -- --canary --preid=beta- git commit -am "chore: bump canary versions [skip ci]"- echo "access=public" > .npmrc- echo "//registry.npmjs.org/:_authToken=${NPM_TOKEN}" >> .npmrc- yarn publish:next --yes --no-verify-access- env:- NPM_TOKEN: ${{ secrets.APIFY_SERVICE_ACCOUNT_NPM_TOKEN }}- GIT_USER: '[email protected]:${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }}'- GH_TOKEN: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }}+ - name: Commit changes+ uses: EndBug/add-and-commit@v9+ id: commit+ with:+ author_name: Apify Release Bot+ author_email: [email protected]+ message: 'chore: bump canary versions [skip ci]'++ - name: Publish packages+ if: steps.changed-packages.outputs.changed_packages != '0'+ uses: apify/workflows/execute-workflow@main+ with:+ workflow: publish-to-npm.yml+ inputs: >+ {+ "ref": ${{ steps.commit.outputs.commit_long_sha || github.sha }},+ "dist-tag": "next"+ }- name: Collect versions for Docker imagesid: versions