Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
ci: build and publish SDK releases in separate jobs from a release environment (#5368)
David Tippetta779ddac4316
diff --git a/.github/workflows/deploy-go-service.yaml b/.github/workflows/deploy-go-service.yamlindex 94768910f..cf0159704 100644--- a/.github/workflows/deploy-go-service.yaml+++ b/.github/workflows/deploy-go-service.yaml@@ -12,9 +12,14 @@ on:- .github/workflows/deploy-go-service.yamlworkflow_dispatch:+permissions: {}+jobs:push-app-image:runs-on: blacksmith-4vcpu-ubuntu-2404+ permissions:+ contents: read+ packages: writedefaults:run:working-directory: "./apps/go-html-to-md-service"diff --git a/.github/workflows/deploy-playwright.yml b/.github/workflows/deploy-playwright.ymlindex 7b9e9cb03..a27d7c20f 100644--- a/.github/workflows/deploy-playwright.yml+++ b/.github/workflows/deploy-playwright.yml@@ -9,9 +9,14 @@ on:- .github/workflows/deploy-playwright.ymlworkflow_dispatch:+permissions: {}+jobs:build:runs-on: ${{ matrix.runner }}+ permissions:+ contents: read+ packages: writestrategy:matrix:include:@@ -64,6 +69,8 @@ jobs:manifest:runs-on: blacksmith-2vcpu-ubuntu-2404needs: build+ permissions:+ packages: writesteps:- name: 'Login to GitHub Container Registry'uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3diff --git a/.github/workflows/deploy-redis.yml b/.github/workflows/deploy-redis.ymlindex f4c209889..e92ff36ad 100644--- a/.github/workflows/deploy-redis.yml+++ b/.github/workflows/deploy-redis.yml@@ -11,9 +11,14 @@ on:- apps/redis/**workflow_dispatch:+permissions: {}+jobs:push-app-image:runs-on: blacksmith-4vcpu-ubuntu-2404+ permissions:+ contents: read+ packages: writedefaults:run:working-directory: './apps/redis'diff --git a/.github/workflows/ghcr-clean.yml b/.github/workflows/ghcr-clean.ymlindex 05ca0ab57..3c1078da4 100644--- a/.github/workflows/ghcr-clean.yml+++ b/.github/workflows/ghcr-clean.yml@@ -3,16 +3,19 @@ name: Cleanup Untagged Imageson:workflow_dispatch:+permissions: {}+jobs:delete-untagged-images:name: Delete Untagged Imagesruns-on: blacksmith-2vcpu-ubuntu-2404+ permissions:+ packages: writesteps:- uses: bots-house/ghcr-delete-image-action@3827559c68cb4dcdf54d813ea9853be6d468d3a4 # v1.1.0with:owner: firecrawlname: firecrawl- # NOTE: using Personal Access Tokentoken: ${{secrets.GITHUB_TOKEN}}# Keep latest N untagged imagesuntagged-keep-latest: 5\ No newline at end of filediff --git a/.github/workflows/publish-dotnet-sdk.yml b/.github/workflows/publish-dotnet-sdk.ymlindex b02e2b6be..de823bef9 100644--- a/.github/workflows/publish-dotnet-sdk.yml+++ b/.github/workflows/publish-dotnet-sdk.yml@@ -8,13 +8,21 @@ on:paths:- 'apps/dot-net-sdk/**'+permissions: {}+jobs:publish:name: Publish to NuGet+ if: github.ref == 'refs/heads/main'runs-on: blacksmith-2vcpu-ubuntu-2404+ environment: release+ permissions:+ contents: readsteps:- name: Checkout repositoryuses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5+ with:+ persist-credentials: false- name: Set up Pythonuses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5diff --git a/.github/workflows/publish-elixir-sdk.yml b/.github/workflows/publish-elixir-sdk.ymlindex cddf5de7c..91adf9bc9 100644--- a/.github/workflows/publish-elixir-sdk.yml+++ b/.github/workflows/publish-elixir-sdk.yml@@ -15,6 +15,8 @@ defaults:run:working-directory: ./apps/elixir-sdk+permissions: {}+jobs:regenerate:if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'@@ -142,12 +144,17 @@ jobs:--base mainpublish:- if: github.event_name == 'push'+ if: github.event_name == 'push' && github.ref == 'refs/heads/main'runs-on: blacksmith-2vcpu-ubuntu-2404+ environment: release+ permissions:+ contents: readsteps:- name: Checkout repositoryuses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5+ with:+ persist-credentials: false- name: Set up Elixiruses: erlef/setup-beam@ee09b1e59bb240681c382eb1f0abc6a04af72764 # v1diff --git a/.github/workflows/publish-go-sdk.yml b/.github/workflows/publish-go-sdk.ymlindex 29da2d69e..0ef58335d 100644--- a/.github/workflows/publish-go-sdk.yml+++ b/.github/workflows/publish-go-sdk.yml@@ -16,10 +16,14 @@ on:# This workflow reads Version from apps/go-sdk/version.go, checks whether the# corresponding tag already exists, and if not creates+pushes it, then warms# the proxy.golang.org cache to trigger pkg.go.dev indexing.+permissions: {}+jobs:publish:name: Tag and index on pkg.go.dev+ if: github.ref == 'refs/heads/main'runs-on: blacksmith-2vcpu-ubuntu-2404+ environment: releasepermissions:contents: writesteps:diff --git a/.github/workflows/publish-java-sdk.yml b/.github/workflows/publish-java-sdk.ymlindex fbc2e76c5..207ff0403 100644--- a/.github/workflows/publish-java-sdk.yml+++ b/.github/workflows/publish-java-sdk.yml@@ -8,13 +8,21 @@ on:paths:- 'apps/java-sdk/**'+permissions: {}+jobs:publish:name: Publish to Maven Central+ if: github.ref == 'refs/heads/main'runs-on: blacksmith-2vcpu-ubuntu-2404+ environment: release+ permissions:+ contents: readsteps:- name: Checkout repositoryuses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5+ with:+ persist-credentials: false- name: Set up Pythonuses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5diff --git a/.github/workflows/publish-js-sdk.yml b/.github/workflows/publish-js-sdk.ymlindex d0900ccb6..810255c89 100644--- a/.github/workflows/publish-js-sdk.yml+++ b/.github/workflows/publish-js-sdk.yml@@ -8,19 +8,23 @@ on:- apps/js-sdk/firecrawl/package.jsonworkflow_dispatch:-env:- TEST_API_KEY: ${{ secrets.TEST_API_KEY }}--permissions:- contents: read- id-token: write+permissions: {}jobs:- publish:- name: Publish+ build:+ name: Buildruns-on: ubuntu-latest+ permissions:+ contents: read+ outputs:+ version: ${{ steps.pack.outputs.version }}+ defaults:+ run:+ working-directory: ./apps/js-sdk/firecrawlsteps:- uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5+ with:+ persist-credentials: false- name: Install pnpmuses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v4with:@@ -29,18 +33,58 @@ jobs:uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4with:node-version: "24"- cache: "pnpm"- cache-dependency-path: './apps/js-sdk/firecrawl/pnpm-lock.yaml'- - name: Publish+ - name: Install dependencies+ run: pnpm install --frozen-lockfile+ - name: Build+ run: pnpm run build+ - name: Pack the three package names+ id: packrun: |- pnpm install- pnpm run buildVERSION=$(node -p "require('./package.json').version")+ echo "version=$VERSION" >> "$GITHUB_OUTPUT"+ mkdir -p "$RUNNER_TEMP/dist"npm pkg set name=firecrawlnpm pkg delete dependencies.firecrawl- pnpm publish --access public --no-git-checks+ pnpm pack --pack-destination "$RUNNER_TEMP/dist"npm pkg set name=@mendable/firecrawl-js dependencies.firecrawl=$VERSION- pnpm publish --access public --no-git-checks+ pnpm pack --pack-destination "$RUNNER_TEMP/dist"npm pkg set name=@mendable/firecrawl- pnpm publish --access public --no-git-checks- working-directory: ./apps/js-sdk/firecrawl+ pnpm pack --pack-destination "$RUNNER_TEMP/dist"+ ls -l "$RUNNER_TEMP/dist"+ - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4+ with:+ name: js-sdk-dist+ path: ${{ runner.temp }}/dist+ if-no-files-found: error+ retention-days: 7++ publish:+ name: Publish+ needs: build+ if: github.ref == 'refs/heads/main'+ runs-on: ubuntu-latest+ environment: release+ permissions:+ id-token: write+ steps:+ - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4+ with:+ name: js-sdk-dist+ path: dist+ - name: Set up Node.js+ uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4+ with:+ node-version: "24"+ registry-url: "https://registry.npmjs.org"+ - name: Publish+ env:+ VERSION: ${{ needs.build.outputs.version }}+ run: |+ for name in firecrawl @mendable/firecrawl-js @mendable/firecrawl; do+ file="dist/$(echo "$name" | sed 's#^@##; s#/#-#')-$VERSION.tgz"+ if [ "$(npm view "$name@$VERSION" version 2>/dev/null)" = "$VERSION" ]; then+ echo "$name@$VERSION is already on npm; skipping"+ continue+ fi+ npm publish "$file" --provenance --access public+ donediff --git a/.github/workflows/publish-php-sdk.yml b/.github/workflows/publish-php-sdk.ymlindex 8b203781c..0ec477213 100644--- a/.github/workflows/publish-php-sdk.yml+++ b/.github/workflows/publish-php-sdk.yml@@ -9,15 +9,22 @@ on:- 'apps/php-sdk/**'- '.github/workflows/publish-php-sdk.yml'+permissions: {}+jobs:publish:name: Publish to Packagist+ if: github.ref == 'refs/heads/main'runs-on: blacksmith-2vcpu-ubuntu-2404+ environment: release+ permissions:+ contents: readsteps:- name: Checkout repositoryuses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5with:fetch-depth: 0+ persist-credentials: false- name: Set up Pythonuses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5diff --git a/.github/workflows/publish-python-sdk.yml b/.github/workflows/publish-python-sdk.ymlindex 6ea20c2c8..0821fe894 100644--- a/.github/workflows/publish-python-sdk.yml+++ b/.github/workflows/publish-python-sdk.yml@@ -1,4 +1,4 @@-name: Publish Python SDK+name: Publish Python SDKon:push:@@ -7,45 +7,77 @@ on:paths:- 'apps/python-sdk/**'-env:- PYPI_USERNAME: ${{ secrets.PYPI_USERNAME }}- PYPI_PASSWORD: ${{ secrets.PYPI_PASSWORD }}+permissions: {}jobs:- build-and-publish:+ build:+ name: Buildruns-on: blacksmith-2vcpu-ubuntu-2404-+ permissions:+ contents: read+ outputs:+ publish: ${{ steps.version_check.outputs.publish }}steps:- name: Checkout repositoryuses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5+ with:+ persist-credentials: false- name: Set up Pythonuses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5with:python-version: '3.x'- - name: Install dependencies- run: |- python -m pip install --upgrade pip- pip install setuptools wheel twine build requests packaging+ - name: Install build tools+ run: python -m pip install build==1.6.1 requests==2.34.2 packaging==26.3- name: Run version check script- id: version_check_script+ id: version_checkrun: |VERSION_INCREMENTED=$(python .github/scripts/check_version_has_incremented.py python ./apps/python-sdk/firecrawl firecrawl-py)- echo "VERSION_INCREMENTED=$VERSION_INCREMENTED" >> $GITHUB_ENV+ echo "publish=$VERSION_INCREMENTED" >> "$GITHUB_OUTPUT"- - name: Build and publish to PyPI- if: ${{ env.VERSION_INCREMENTED == 'true' }}- env:- TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }}- TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}+ - name: Build firecrawl-py and firecrawl+ if: steps.version_check.outputs.publish == 'true'+ working-directory: ./apps/python-sdkrun: |- python -m build- twine upload dist/*+ python -m build --outdir "$RUNNER_TEMP/dist/firecrawl-py"sed -i 's/name = "firecrawl-py"/name = "firecrawl"/g' pyproject.tomlsed -i 's/name="firecrawl-py"/name="firecrawl"/g' setup.py- python -m build- twine upload dist/*- working-directory: ./apps/python-sdk+ rm -rf build *.egg-info+ python -m build --outdir "$RUNNER_TEMP/dist/firecrawl"+ ls -lR "$RUNNER_TEMP/dist"++ - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4+ if: steps.version_check.outputs.publish == 'true'+ with:+ name: python-sdk-dist+ path: ${{ runner.temp }}/dist+ if-no-files-found: error+ retention-days: 7+ publish:+ name: Publish to PyPI+ needs: build+ if: needs.build.outputs.publish == 'true' && github.ref == 'refs/heads/main'+ runs-on: ubuntu-latest+ environment: release+ permissions:+ id-token: write+ steps:+ - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4+ with:+ name: python-sdk-dist+ path: dist++ - name: Publish firecrawl-py+ uses: pypa/gh-action-pypi-publish@7245906fb61fa33a13554f9d5fb856fe61758086 # v1.14.3+ with:+ packages-dir: dist/firecrawl-py/+ attestations: true++ - name: Publish firecrawl+ uses: pypa/gh-action-pypi-publish@7245906fb61fa33a13554f9d5fb856fe61758086 # v1.14.3+ with:+ packages-dir: dist/firecrawl/+ attestations: truediff --git a/.github/workflows/publish-ruby-sdk.yml b/.github/workflows/publish-ruby-sdk.ymlindex d56d01253..cbaa6deab 100644--- a/.github/workflows/publish-ruby-sdk.yml+++ b/.github/workflows/publish-ruby-sdk.yml@@ -8,13 +8,21 @@ on:paths:- 'apps/ruby-sdk/**'+permissions: {}+jobs:publish:name: Publish to RubyGems+ if: github.ref == 'refs/heads/main'runs-on: blacksmith-2vcpu-ubuntu-2404+ environment: release+ permissions:+ contents: readsteps:- name: Checkout repositoryuses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5+ with:+ persist-credentials: false- name: Set up Pythonuses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5diff --git a/.github/workflows/publish-rust-sdk.yml b/.github/workflows/publish-rust-sdk.ymlindex 8491f26cd..25da2e8b2 100644--- a/.github/workflows/publish-rust-sdk.yml+++ b/.github/workflows/publish-rust-sdk.yml@@ -8,13 +8,22 @@ on:paths:- 'apps/rust-sdk/**'+permissions: {}+jobs:- publish:- name: Publish to crates.io+ build:+ name: Build and packageruns-on: blacksmith-2vcpu-ubuntu-2404+ permissions:+ contents: read+ outputs:+ publish: ${{ steps.version_check.outputs.publish }}+ crate_sha256: ${{ steps.package.outputs.sha256 }}steps:- name: Checkout repositoryuses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4+ with:+ persist-credentials: false- name: Set up Pythonuses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5@@ -22,43 +31,90 @@ jobs:python-version: '3.x'- name: Install version check dependencies- run: pip install requests packaging toml+ run: pip install requests==2.34.2 packaging==26.3 toml==0.10.2- name: Run version check scriptid: version_checkrun: |VERSION_INCREMENTED=$(python .github/scripts/check_version_has_incremented.py rust ./apps/rust-sdk firecrawl)- echo "VERSION_INCREMENTED=$VERSION_INCREMENTED" >> $GITHUB_ENV+ echo "publish=$VERSION_INCREMENTED" >> "$GITHUB_OUTPUT"- name: Install Rust toolchain- if: ${{ env.VERSION_INCREMENTED == 'true' }}+ if: steps.version_check.outputs.publish == 'true'uses: dtolnay/rust-toolchain@631a55b12751854ce901bb631d5902ceb48146f7 # stable- - name: Cache cargo registry- if: ${{ env.VERSION_INCREMENTED == 'true' }}- uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4- with:- path: |- ~/.cargo/registry- ~/.cargo/git- apps/rust-sdk/target- key: ${{ runner.os }}-cargo-${{ hashFiles('apps/rust-sdk/Cargo.lock') }}- restore-keys: |- ${{ runner.os }}-cargo-+ - name: Package+ id: package+ if: steps.version_check.outputs.publish == 'true'+ working-directory: ./apps/rust-sdk+ run: |+ cargo package --allow-dirty+ crate=$(ls target/package/*.crate)+ echo "sha256=$(sha256sum "$crate" | cut -d' ' -f1)" >> "$GITHUB_OUTPUT"+ mkdir -p "$RUNNER_TEMP/dist" && cp "$crate" "$RUNNER_TEMP/dist/"- name: Build- if: ${{ env.VERSION_INCREMENTED == 'true' }}+ if: steps.version_check.outputs.publish == 'true'working-directory: ./apps/rust-sdkrun: cargo build --release- name: Run tests- if: ${{ env.VERSION_INCREMENTED == 'true' }}+ if: steps.version_check.outputs.publish == 'true'working-directory: ./apps/rust-sdkrun: cargo test --lib+ - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4+ if: steps.version_check.outputs.publish == 'true'+ with:+ name: rust-sdk-crate+ path: ${{ runner.temp }}/dist+ if-no-files-found: error+ retention-days: 7++ publish:+ name: Publish to crates.io+ needs: build+ if: needs.build.outputs.publish == 'true' && github.ref == 'refs/heads/main'+ runs-on: ubuntu-latest+ environment: release+ permissions:+ contents: read+ id-token: write+ steps:+ - name: Checkout repository+ uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4+ with:+ persist-credentials: false++ - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4+ with:+ name: rust-sdk-crate+ path: ${{ runner.temp }}/dist++ - name: Install Rust toolchain+ uses: dtolnay/rust-toolchain@631a55b12751854ce901bb631d5902ceb48146f7 # stable++ # cargo publish uploads what cargo package produces from this commit; refuse+ # unless that is byte-identical to the crate the build job tested.+ - name: Check the crate matches the build+ working-directory: ./apps/rust-sdk+ env:+ EXPECTED: ${{ needs.build.outputs.crate_sha256 }}+ run: |+ cargo package --no-verify --allow-dirty+ actual=$(sha256sum target/package/*.crate | cut -d' ' -f1)+ artifact=$(sha256sum "$RUNNER_TEMP"/dist/*.crate | cut -d' ' -f1)+ if [ "$actual" != "$EXPECTED" ] || [ "$artifact" != "$EXPECTED" ]; then+ echo "::error::Crate mismatch: build $EXPECTED, artifact $artifact, publish $actual"+ exit 1+ fi++ - name: Authenticate to crates.io+ id: auth+ uses: rust-lang/crates-io-auth-action@3268eb413133e2960fa4ae0e963ca76380c54cd9 # v1.0.6+- name: Publish to crates.io- if: ${{ env.VERSION_INCREMENTED == 'true' }}working-directory: ./apps/rust-sdkenv:- CARGO_REGISTRY_TOKEN: ${{ secrets.CRATES_IO_TOKEN }}- run: cargo publish --allow-dirty+ CARGO_REGISTRY_TOKEN: ${{ steps.auth.outputs.token }}+ run: cargo publish --no-verify --allow-dirtydiff --git a/.github/workflows/test-js-sdk.yml b/.github/workflows/test-js-sdk.ymlindex 9227a7ed5..fe7611be7 100644--- a/.github/workflows/test-js-sdk.yml+++ b/.github/workflows/test-js-sdk.yml@@ -61,7 +61,7 @@ jobs:tags: tag:ciuse-cache: 'true'- name: Install dependencies- run: pnpm install+ run: pnpm install --frozen-lockfileworking-directory: ./apps/js-sdk/firecrawl- name: Buildrun: pnpm run builddiff --git a/apps/js-sdk/firecrawl/pnpm-workspace.yaml b/apps/js-sdk/firecrawl/pnpm-workspace.yamlindex 1bf703b8e..9fc00710d 100644--- a/apps/js-sdk/firecrawl/pnpm-workspace.yaml+++ b/apps/js-sdk/firecrawl/pnpm-workspace.yaml@@ -1,5 +1,8 @@allowBuilds:esbuild: true+minimumReleaseAge: 4320 # 3 days+minimumReleaseAgeExclude:+ - firecrawloverrides:"@isaacs/brace-expansion@<=5.0.0": ">=5.0.1""minimatch@<10.2.3": ">=10.2.3"