Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

ci: build and publish SDK releases in separate jobs from a release environment (#5368)

David Tippetta779ddac4316
diff --git a/.github/workflows/deploy-go-service.yaml b/.github/workflows/deploy-go-service.yaml
index 94768910f..cf0159704 100644
--- a/.github/workflows/deploy-go-service.yaml
+++ b/.github/workflows/deploy-go-service.yaml
@@ -12,9 +12,14 @@ on:
- .github/workflows/deploy-go-service.yaml
workflow_dispatch:
+permissions: {}
+
jobs:
push-app-image:
runs-on: blacksmith-4vcpu-ubuntu-2404
+ permissions:
+ contents: read
+ packages: write
defaults:
run:
working-directory: "./apps/go-html-to-md-service"
diff --git a/.github/workflows/deploy-playwright.yml b/.github/workflows/deploy-playwright.yml
index 7b9e9cb03..a27d7c20f 100644
--- a/.github/workflows/deploy-playwright.yml
+++ b/.github/workflows/deploy-playwright.yml
@@ -9,9 +9,14 @@ on:
- .github/workflows/deploy-playwright.yml
workflow_dispatch:
+permissions: {}
+
jobs:
build:
runs-on: ${{ matrix.runner }}
+ permissions:
+ contents: read
+ packages: write
strategy:
matrix:
include:
@@ -64,6 +69,8 @@ jobs:
manifest:
runs-on: blacksmith-2vcpu-ubuntu-2404
needs: build
+ permissions:
+ packages: write
steps:
- name: 'Login to GitHub Container Registry'
uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
diff --git a/.github/workflows/deploy-redis.yml b/.github/workflows/deploy-redis.yml
index f4c209889..e92ff36ad 100644
--- a/.github/workflows/deploy-redis.yml
+++ b/.github/workflows/deploy-redis.yml
@@ -11,9 +11,14 @@ on:
- apps/redis/**
workflow_dispatch:
+permissions: {}
+
jobs:
push-app-image:
runs-on: blacksmith-4vcpu-ubuntu-2404
+ permissions:
+ contents: read
+ packages: write
defaults:
run:
working-directory: './apps/redis'
diff --git a/.github/workflows/ghcr-clean.yml b/.github/workflows/ghcr-clean.yml
index 05ca0ab57..3c1078da4 100644
--- a/.github/workflows/ghcr-clean.yml
+++ b/.github/workflows/ghcr-clean.yml
@@ -3,16 +3,19 @@ name: Cleanup Untagged Images
on:
workflow_dispatch:
+permissions: {}
+
jobs:
delete-untagged-images:
name: Delete Untagged Images
runs-on: blacksmith-2vcpu-ubuntu-2404
+ permissions:
+ packages: write
steps:
- uses: bots-house/ghcr-delete-image-action@3827559c68cb4dcdf54d813ea9853be6d468d3a4 # v1.1.0
with:
owner: firecrawl
name: firecrawl
- # NOTE: using Personal Access Token
token: ${{secrets.GITHUB_TOKEN}}
# Keep latest N untagged images
untagged-keep-latest: 5
\ No newline at end of file
diff --git a/.github/workflows/publish-dotnet-sdk.yml b/.github/workflows/publish-dotnet-sdk.yml
index b02e2b6be..de823bef9 100644
--- a/.github/workflows/publish-dotnet-sdk.yml
+++ b/.github/workflows/publish-dotnet-sdk.yml
@@ -8,13 +8,21 @@ on:
paths:
- 'apps/dot-net-sdk/**'
+permissions: {}
+
jobs:
publish:
name: Publish to NuGet
+ if: github.ref == 'refs/heads/main'
runs-on: blacksmith-2vcpu-ubuntu-2404
+ environment: release
+ permissions:
+ contents: read
steps:
- name: Checkout repository
uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
+ with:
+ persist-credentials: false
- name: Set up Python
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
diff --git a/.github/workflows/publish-elixir-sdk.yml b/.github/workflows/publish-elixir-sdk.yml
index cddf5de7c..91adf9bc9 100644
--- a/.github/workflows/publish-elixir-sdk.yml
+++ b/.github/workflows/publish-elixir-sdk.yml
@@ -15,6 +15,8 @@ defaults:
run:
working-directory: ./apps/elixir-sdk
+permissions: {}
+
jobs:
regenerate:
if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
@@ -142,12 +144,17 @@ jobs:
--base main
publish:
- if: github.event_name == 'push'
+ if: github.event_name == 'push' && github.ref == 'refs/heads/main'
runs-on: blacksmith-2vcpu-ubuntu-2404
+ environment: release
+ permissions:
+ contents: read
steps:
- name: Checkout repository
uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
+ with:
+ persist-credentials: false
- name: Set up Elixir
uses: erlef/setup-beam@ee09b1e59bb240681c382eb1f0abc6a04af72764 # v1
diff --git a/.github/workflows/publish-go-sdk.yml b/.github/workflows/publish-go-sdk.yml
index 29da2d69e..0ef58335d 100644
--- a/.github/workflows/publish-go-sdk.yml
+++ b/.github/workflows/publish-go-sdk.yml
@@ -16,10 +16,14 @@ on:
# This workflow reads Version from apps/go-sdk/version.go, checks whether the
# corresponding tag already exists, and if not creates+pushes it, then warms
# the proxy.golang.org cache to trigger pkg.go.dev indexing.
+permissions: {}
+
jobs:
publish:
name: Tag and index on pkg.go.dev
+ if: github.ref == 'refs/heads/main'
runs-on: blacksmith-2vcpu-ubuntu-2404
+ environment: release
permissions:
contents: write
steps:
diff --git a/.github/workflows/publish-java-sdk.yml b/.github/workflows/publish-java-sdk.yml
index fbc2e76c5..207ff0403 100644
--- a/.github/workflows/publish-java-sdk.yml
+++ b/.github/workflows/publish-java-sdk.yml
@@ -8,13 +8,21 @@ on:
paths:
- 'apps/java-sdk/**'
+permissions: {}
+
jobs:
publish:
name: Publish to Maven Central
+ if: github.ref == 'refs/heads/main'
runs-on: blacksmith-2vcpu-ubuntu-2404
+ environment: release
+ permissions:
+ contents: read
steps:
- name: Checkout repository
uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
+ with:
+ persist-credentials: false
- name: Set up Python
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
diff --git a/.github/workflows/publish-js-sdk.yml b/.github/workflows/publish-js-sdk.yml
index d0900ccb6..810255c89 100644
--- a/.github/workflows/publish-js-sdk.yml
+++ b/.github/workflows/publish-js-sdk.yml
@@ -8,19 +8,23 @@ on:
- apps/js-sdk/firecrawl/package.json
workflow_dispatch:
-env:
- TEST_API_KEY: ${{ secrets.TEST_API_KEY }}
-
-permissions:
- contents: read
- id-token: write
+permissions: {}
jobs:
- publish:
- name: Publish
+ build:
+ name: Build
runs-on: ubuntu-latest
+ permissions:
+ contents: read
+ outputs:
+ version: ${{ steps.pack.outputs.version }}
+ defaults:
+ run:
+ working-directory: ./apps/js-sdk/firecrawl
steps:
- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
+ with:
+ persist-credentials: false
- name: Install pnpm
uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v4
with:
@@ -29,18 +33,58 @@ jobs:
uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4
with:
node-version: "24"
- cache: "pnpm"
- cache-dependency-path: './apps/js-sdk/firecrawl/pnpm-lock.yaml'
- - name: Publish
+ - name: Install dependencies
+ run: pnpm install --frozen-lockfile
+ - name: Build
+ run: pnpm run build
+ - name: Pack the three package names
+ id: pack
run: |
- pnpm install
- pnpm run build
VERSION=$(node -p "require('./package.json').version")
+ echo "version=$VERSION" >> "$GITHUB_OUTPUT"
+ mkdir -p "$RUNNER_TEMP/dist"
npm pkg set name=firecrawl
npm pkg delete dependencies.firecrawl
- pnpm publish --access public --no-git-checks
+ pnpm pack --pack-destination "$RUNNER_TEMP/dist"
npm pkg set name=@mendable/firecrawl-js dependencies.firecrawl=$VERSION
- pnpm publish --access public --no-git-checks
+ pnpm pack --pack-destination "$RUNNER_TEMP/dist"
npm pkg set name=@mendable/firecrawl
- pnpm publish --access public --no-git-checks
- working-directory: ./apps/js-sdk/firecrawl
+ pnpm pack --pack-destination "$RUNNER_TEMP/dist"
+ ls -l "$RUNNER_TEMP/dist"
+ - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
+ with:
+ name: js-sdk-dist
+ path: ${{ runner.temp }}/dist
+ if-no-files-found: error
+ retention-days: 7
+
+ publish:
+ name: Publish
+ needs: build
+ if: github.ref == 'refs/heads/main'
+ runs-on: ubuntu-latest
+ environment: release
+ permissions:
+ id-token: write
+ steps:
+ - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
+ with:
+ name: js-sdk-dist
+ path: dist
+ - name: Set up Node.js
+ uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4
+ with:
+ node-version: "24"
+ registry-url: "https://registry.npmjs.org"
+ - name: Publish
+ env:
+ VERSION: ${{ needs.build.outputs.version }}
+ run: |
+ for name in firecrawl @mendable/firecrawl-js @mendable/firecrawl; do
+ file="dist/$(echo "$name" | sed 's#^@##; s#/#-#')-$VERSION.tgz"
+ if [ "$(npm view "$name@$VERSION" version 2>/dev/null)" = "$VERSION" ]; then
+ echo "$name@$VERSION is already on npm; skipping"
+ continue
+ fi
+ npm publish "$file" --provenance --access public
+ done
diff --git a/.github/workflows/publish-php-sdk.yml b/.github/workflows/publish-php-sdk.yml
index 8b203781c..0ec477213 100644
--- a/.github/workflows/publish-php-sdk.yml
+++ b/.github/workflows/publish-php-sdk.yml
@@ -9,15 +9,22 @@ on:
- 'apps/php-sdk/**'
- '.github/workflows/publish-php-sdk.yml'
+permissions: {}
+
jobs:
publish:
name: Publish to Packagist
+ if: github.ref == 'refs/heads/main'
runs-on: blacksmith-2vcpu-ubuntu-2404
+ environment: release
+ permissions:
+ contents: read
steps:
- name: Checkout repository
uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
with:
fetch-depth: 0
+ persist-credentials: false
- name: Set up Python
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
diff --git a/.github/workflows/publish-python-sdk.yml b/.github/workflows/publish-python-sdk.yml
index 6ea20c2c8..0821fe894 100644
--- a/.github/workflows/publish-python-sdk.yml
+++ b/.github/workflows/publish-python-sdk.yml
@@ -1,4 +1,4 @@
-name: Publish Python SDK
+name: Publish Python SDK
on:
push:
@@ -7,45 +7,77 @@ on:
paths:
- 'apps/python-sdk/**'
-env:
- PYPI_USERNAME: ${{ secrets.PYPI_USERNAME }}
- PYPI_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
+permissions: {}
jobs:
- build-and-publish:
+ build:
+ name: Build
runs-on: blacksmith-2vcpu-ubuntu-2404
-
+ permissions:
+ contents: read
+ outputs:
+ publish: ${{ steps.version_check.outputs.publish }}
steps:
- name: Checkout repository
uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
+ with:
+ persist-credentials: false
- name: Set up Python
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
with:
python-version: '3.x'
- - name: Install dependencies
- run: |
- python -m pip install --upgrade pip
- pip install setuptools wheel twine build requests packaging
+ - name: Install build tools
+ run: python -m pip install build==1.6.1 requests==2.34.2 packaging==26.3
- name: Run version check script
- id: version_check_script
+ id: version_check
run: |
VERSION_INCREMENTED=$(python .github/scripts/check_version_has_incremented.py python ./apps/python-sdk/firecrawl firecrawl-py)
- echo "VERSION_INCREMENTED=$VERSION_INCREMENTED" >> $GITHUB_ENV
+ echo "publish=$VERSION_INCREMENTED" >> "$GITHUB_OUTPUT"
- - name: Build and publish to PyPI
- if: ${{ env.VERSION_INCREMENTED == 'true' }}
- env:
- TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }}
- TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
+ - name: Build firecrawl-py and firecrawl
+ if: steps.version_check.outputs.publish == 'true'
+ working-directory: ./apps/python-sdk
run: |
- python -m build
- twine upload dist/*
+ python -m build --outdir "$RUNNER_TEMP/dist/firecrawl-py"
sed -i 's/name = "firecrawl-py"/name = "firecrawl"/g' pyproject.toml
sed -i 's/name="firecrawl-py"/name="firecrawl"/g' setup.py
- python -m build
- twine upload dist/*
- working-directory: ./apps/python-sdk
+ rm -rf build *.egg-info
+ python -m build --outdir "$RUNNER_TEMP/dist/firecrawl"
+ ls -lR "$RUNNER_TEMP/dist"
+
+ - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
+ if: steps.version_check.outputs.publish == 'true'
+ with:
+ name: python-sdk-dist
+ path: ${{ runner.temp }}/dist
+ if-no-files-found: error
+ retention-days: 7
+ publish:
+ name: Publish to PyPI
+ needs: build
+ if: needs.build.outputs.publish == 'true' && github.ref == 'refs/heads/main'
+ runs-on: ubuntu-latest
+ environment: release
+ permissions:
+ id-token: write
+ steps:
+ - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
+ with:
+ name: python-sdk-dist
+ path: dist
+
+ - name: Publish firecrawl-py
+ uses: pypa/gh-action-pypi-publish@7245906fb61fa33a13554f9d5fb856fe61758086 # v1.14.3
+ with:
+ packages-dir: dist/firecrawl-py/
+ attestations: true
+
+ - name: Publish firecrawl
+ uses: pypa/gh-action-pypi-publish@7245906fb61fa33a13554f9d5fb856fe61758086 # v1.14.3
+ with:
+ packages-dir: dist/firecrawl/
+ attestations: true
diff --git a/.github/workflows/publish-ruby-sdk.yml b/.github/workflows/publish-ruby-sdk.yml
index d56d01253..cbaa6deab 100644
--- a/.github/workflows/publish-ruby-sdk.yml
+++ b/.github/workflows/publish-ruby-sdk.yml
@@ -8,13 +8,21 @@ on:
paths:
- 'apps/ruby-sdk/**'
+permissions: {}
+
jobs:
publish:
name: Publish to RubyGems
+ if: github.ref == 'refs/heads/main'
runs-on: blacksmith-2vcpu-ubuntu-2404
+ environment: release
+ permissions:
+ contents: read
steps:
- name: Checkout repository
uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
+ with:
+ persist-credentials: false
- name: Set up Python
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
diff --git a/.github/workflows/publish-rust-sdk.yml b/.github/workflows/publish-rust-sdk.yml
index 8491f26cd..25da2e8b2 100644
--- a/.github/workflows/publish-rust-sdk.yml
+++ b/.github/workflows/publish-rust-sdk.yml
@@ -8,13 +8,22 @@ on:
paths:
- 'apps/rust-sdk/**'
+permissions: {}
+
jobs:
- publish:
- name: Publish to crates.io
+ build:
+ name: Build and package
runs-on: blacksmith-2vcpu-ubuntu-2404
+ permissions:
+ contents: read
+ outputs:
+ publish: ${{ steps.version_check.outputs.publish }}
+ crate_sha256: ${{ steps.package.outputs.sha256 }}
steps:
- name: Checkout repository
uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
+ with:
+ persist-credentials: false
- name: Set up Python
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
@@ -22,43 +31,90 @@ jobs:
python-version: '3.x'
- name: Install version check dependencies
- run: pip install requests packaging toml
+ run: pip install requests==2.34.2 packaging==26.3 toml==0.10.2
- name: Run version check script
id: version_check
run: |
VERSION_INCREMENTED=$(python .github/scripts/check_version_has_incremented.py rust ./apps/rust-sdk firecrawl)
- echo "VERSION_INCREMENTED=$VERSION_INCREMENTED" >> $GITHUB_ENV
+ echo "publish=$VERSION_INCREMENTED" >> "$GITHUB_OUTPUT"
- name: Install Rust toolchain
- if: ${{ env.VERSION_INCREMENTED == 'true' }}
+ if: steps.version_check.outputs.publish == 'true'
uses: dtolnay/rust-toolchain@631a55b12751854ce901bb631d5902ceb48146f7 # stable
- - name: Cache cargo registry
- if: ${{ env.VERSION_INCREMENTED == 'true' }}
- uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4
- with:
- path: |
- ~/.cargo/registry
- ~/.cargo/git
- apps/rust-sdk/target
- key: ${{ runner.os }}-cargo-${{ hashFiles('apps/rust-sdk/Cargo.lock') }}
- restore-keys: |
- ${{ runner.os }}-cargo-
+ - name: Package
+ id: package
+ if: steps.version_check.outputs.publish == 'true'
+ working-directory: ./apps/rust-sdk
+ run: |
+ cargo package --allow-dirty
+ crate=$(ls target/package/*.crate)
+ echo "sha256=$(sha256sum "$crate" | cut -d' ' -f1)" >> "$GITHUB_OUTPUT"
+ mkdir -p "$RUNNER_TEMP/dist" && cp "$crate" "$RUNNER_TEMP/dist/"
- name: Build
- if: ${{ env.VERSION_INCREMENTED == 'true' }}
+ if: steps.version_check.outputs.publish == 'true'
working-directory: ./apps/rust-sdk
run: cargo build --release
- name: Run tests
- if: ${{ env.VERSION_INCREMENTED == 'true' }}
+ if: steps.version_check.outputs.publish == 'true'
working-directory: ./apps/rust-sdk
run: cargo test --lib
+ - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
+ if: steps.version_check.outputs.publish == 'true'
+ with:
+ name: rust-sdk-crate
+ path: ${{ runner.temp }}/dist
+ if-no-files-found: error
+ retention-days: 7
+
+ publish:
+ name: Publish to crates.io
+ needs: build
+ if: needs.build.outputs.publish == 'true' && github.ref == 'refs/heads/main'
+ runs-on: ubuntu-latest
+ environment: release
+ permissions:
+ contents: read
+ id-token: write
+ steps:
+ - name: Checkout repository
+ uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
+ with:
+ persist-credentials: false
+
+ - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
+ with:
+ name: rust-sdk-crate
+ path: ${{ runner.temp }}/dist
+
+ - name: Install Rust toolchain
+ uses: dtolnay/rust-toolchain@631a55b12751854ce901bb631d5902ceb48146f7 # stable
+
+ # cargo publish uploads what cargo package produces from this commit; refuse
+ # unless that is byte-identical to the crate the build job tested.
+ - name: Check the crate matches the build
+ working-directory: ./apps/rust-sdk
+ env:
+ EXPECTED: ${{ needs.build.outputs.crate_sha256 }}
+ run: |
+ cargo package --no-verify --allow-dirty
+ actual=$(sha256sum target/package/*.crate | cut -d' ' -f1)
+ artifact=$(sha256sum "$RUNNER_TEMP"/dist/*.crate | cut -d' ' -f1)
+ if [ "$actual" != "$EXPECTED" ] || [ "$artifact" != "$EXPECTED" ]; then
+ echo "::error::Crate mismatch: build $EXPECTED, artifact $artifact, publish $actual"
+ exit 1
+ fi
+
+ - name: Authenticate to crates.io
+ id: auth
+ uses: rust-lang/crates-io-auth-action@3268eb413133e2960fa4ae0e963ca76380c54cd9 # v1.0.6
+
- name: Publish to crates.io
- if: ${{ env.VERSION_INCREMENTED == 'true' }}
working-directory: ./apps/rust-sdk
env:
- CARGO_REGISTRY_TOKEN: ${{ secrets.CRATES_IO_TOKEN }}
- run: cargo publish --allow-dirty
+ CARGO_REGISTRY_TOKEN: ${{ steps.auth.outputs.token }}
+ run: cargo publish --no-verify --allow-dirty
diff --git a/.github/workflows/test-js-sdk.yml b/.github/workflows/test-js-sdk.yml
index 9227a7ed5..fe7611be7 100644
--- a/.github/workflows/test-js-sdk.yml
+++ b/.github/workflows/test-js-sdk.yml
@@ -61,7 +61,7 @@ jobs:
tags: tag:ci
use-cache: 'true'
- name: Install dependencies
- run: pnpm install
+ run: pnpm install --frozen-lockfile
working-directory: ./apps/js-sdk/firecrawl
- name: Build
run: pnpm run build
diff --git a/apps/js-sdk/firecrawl/pnpm-workspace.yaml b/apps/js-sdk/firecrawl/pnpm-workspace.yaml
index 1bf703b8e..9fc00710d 100644
--- a/apps/js-sdk/firecrawl/pnpm-workspace.yaml
+++ b/apps/js-sdk/firecrawl/pnpm-workspace.yaml
@@ -1,5 +1,8 @@
allowBuilds:
esbuild: true
+minimumReleaseAge: 4320 # 3 days
+minimumReleaseAgeExclude:
+ - firecrawl
overrides:
"@isaacs/brace-expansion@<=5.0.0": ">=5.0.1"
"minimatch@<10.2.3": ">=10.2.3"