Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

ci(api): publish semver image tags (#3781)

Gergő Móriczeeb72c5c2852
diff --git a/.github/scripts/resolve_api_image_version.py b/.github/scripts/resolve_api_image_version.py
new file mode 100644
index 000000000..df5638af9
--- /dev/null
+++ b/.github/scripts/resolve_api_image_version.py
@@ -0,0 +1,129 @@
+#!/usr/bin/env python3
+"""Resolve the next deployable API image version.
+
+Deploy image versions are full SemVer Git tags: vX.Y.Z. Existing two-part
+marketing release tags such as v2.10 are treated as v2.10.0 bases.
+"""
+
+from __future__ import annotations
+
+import argparse
+import os
+import re
+import subprocess
+import sys
+from dataclasses import dataclass
+
+
+VERSION_TAG_RE = re.compile(r"^v(0|[1-9]\d*)\.(0|[1-9]\d*)(?:\.(0|[1-9]\d*))?$")
+
+
+@dataclass(frozen=True)
+class SemverTag:
+ name: str
+ version: tuple[int, int, int]
+ commit: str
+
+
+def git(*args: str) -> str:
+ return subprocess.check_output(["git", *args], text=True).strip()
+
+
+def list_semver_tags() -> list[SemverTag]:
+ raw_tags = git("tag", "--list", "v*").splitlines()
+ tags: list[SemverTag] = []
+
+ for tag in raw_tags:
+ match = VERSION_TAG_RE.match(tag)
+ if not match:
+ continue
+
+ major, minor, patch = match.groups()
+ commit = git("rev-list", "-n", "1", tag)
+ tags.append(
+ SemverTag(
+ name=tag,
+ version=(int(major), int(minor), int(patch or 0)),
+ commit=commit,
+ )
+ )
+
+ return sorted(tags, key=lambda item: item.version)
+
+
+def bump_version(base: tuple[int, int, int] | None, bump: str) -> tuple[int, int, int]:
+ if base is None:
+ return (0, 1, 0) if bump == "minor" else (0, 0, 1)
+
+ major, minor, patch = base
+ if bump == "minor":
+ return (major, minor + 1, 0)
+
+ return (major, minor, patch + 1)
+
+
+def write_output(name: str, value: str) -> None:
+ print(f"{name}={value}")
+
+ github_output = os.environ.get("GITHUB_OUTPUT")
+ if github_output:
+ with open(github_output, "a", encoding="utf-8") as output:
+ output.write(f"{name}={value}\n")
+
+
+def main() -> int:
+ parser = argparse.ArgumentParser()
+ parser.add_argument("--bump", choices=["patch", "minor"], default="patch")
+ parser.add_argument("--sha", default=os.environ.get("GITHUB_SHA", "HEAD"))
+ parser.add_argument("--image-name", default="firecrawl")
+ parser.add_argument(
+ "--include-current-commit-tags",
+ action="store_true",
+ help="Use the highest SemVer tag even when it already points at --sha.",
+ )
+ args = parser.parse_args()
+
+ current_sha = git("rev-parse", args.sha)
+ short_sha = git("rev-parse", "--short=12", current_sha)
+ semver_tags = list_semver_tags()
+
+ if args.include_current_commit_tags:
+ base_tags = semver_tags
+ else:
+ # Excluding tags already on this commit makes production retries
+ # idempotent. If a prior attempt created vX.Y.Z but failed before
+ # pushing every manifest, the same target version is reused instead of
+ # burning vX.Y.(Z+1).
+ base_tags = [tag for tag in semver_tags if tag.commit != current_sha]
+
+ base = base_tags[-1].version if base_tags else None
+ version = bump_version(base, args.bump)
+ tag_name = f"v{version[0]}.{version[1]}.{version[2]}"
+
+ existing_target = next((tag for tag in semver_tags if tag.name == tag_name), None)
+ if existing_target and existing_target.commit != current_sha:
+ print(
+ f"Resolved tag {tag_name} already exists on {existing_target.commit}, "
+ f"not {current_sha}",
+ file=sys.stderr,
+ )
+ return 1
+
+ repo_owner = os.environ.get("GITHUB_REPOSITORY_OWNER", "firecrawl").lower()
+ image = f"ghcr.io/{repo_owner}/{args.image_name}"
+ version_text = ".".join(str(part) for part in version)
+
+ write_output("version", version_text)
+ write_output("tag", tag_name)
+ write_output("tag_exists", "true" if existing_target else "false")
+ write_output("major", str(version[0]))
+ write_output("minor", str(version[1]))
+ write_output("major_minor", f"{version[0]}.{version[1]}")
+ write_output("short_sha", short_sha)
+ write_output("image", image)
+
+ return 0
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/.github/workflows/deploy-image-staging.yml b/.github/workflows/deploy-image-staging.yml
index 9256b1fd4..7d90bc735 100644
--- a/.github/workflows/deploy-image-staging.yml
+++ b/.github/workflows/deploy-image-staging.yml
@@ -1,20 +1,62 @@
name: STAGING Deploy Images to GHCR
-env:
- DOTNET_VERSION: '6.0.x'
-
on:
workflow_dispatch:
+permissions:
+ contents: read
+ packages: write
+
+concurrency:
+ group: api-image-staging-release
+ cancel-in-progress: false
+
jobs:
- push-app-image:
- runs-on: blacksmith-4vcpu-ubuntu-2404
+ version:
+ runs-on: blacksmith-2vcpu-ubuntu-2404
+ outputs:
+ version: ${{ steps.version.outputs.version }}
+ short_sha: ${{ steps.version.outputs.short_sha }}
+ image: ${{ steps.version.outputs.image }}
+ staging_image: ${{ steps.staging_image.outputs.image }}
+ steps:
+ - name: Checkout repository
+ uses: actions/checkout@v4
+ with:
+ fetch-depth: 0
+
+ - name: Fetch tags
+ run: git fetch --force --tags origin
+
+ - name: Resolve staging image version
+ id: version
+ run: python .github/scripts/resolve_api_image_version.py --bump patch --sha "${GITHUB_SHA}" --include-current-commit-tags
+
+ - name: Resolve staging image repository
+ id: staging_image
+ run: echo "image=ghcr.io/${GITHUB_REPOSITORY_OWNER,,}/firecrawl-staging" >> "$GITHUB_OUTPUT"
+
+ build:
+ needs: version
+ runs-on: ${{ matrix.runner }}
+ strategy:
+ matrix:
+ include:
+ - platform: linux/amd64
+ platform_suffix: linux-amd64
+ runner: blacksmith-4vcpu-ubuntu-2404
+ - platform: linux/arm64
+ platform_suffix: linux-arm64
+ runner: blacksmith-4vcpu-ubuntu-2404-arm
defaults:
run:
working-directory: './apps/api'
steps:
- - name: 'Checkout GitHub Action'
- uses: actions/checkout@main
+ - name: Checkout repository
+ uses: actions/checkout@v4
+
+ - name: Setup Blacksmith Builder
+ uses: useblacksmith/setup-docker-builder@ef12d5b165b596e3aa44ea8198d8fde563eab402 # v1
- name: 'Login to GitHub Container Registry'
uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
@@ -23,7 +65,45 @@ jobs:
username: ${{github.actor}}
password: ${{secrets.GITHUB_TOKEN}}
- - name: 'Build Inventory Image'
+ - name: 'Build and Push Image'
+ uses: useblacksmith/build-push-action@30c71162f16ea2c27c3e21523255d209b8b538c1 # v2
+ with:
+ context: ./apps/api
+ push: true
+ tags: ${{ needs.version.outputs.image }}:sha-${{ needs.version.outputs.short_sha }}-staging-${{ matrix.platform_suffix }}
+ platforms: ${{ matrix.platform }}
+ provenance: false
+
+ publish:
+ runs-on: blacksmith-2vcpu-ubuntu-2404
+ needs:
+ - version
+ - build
+ steps:
+ - name: 'Login to GitHub Container Registry'
+ uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
+ with:
+ registry: ghcr.io
+ username: ${{github.actor}}
+ password: ${{secrets.GITHUB_TOKEN}}
+
+ - name: 'Create and Push Staging Multi-Arch Manifest'
+ env:
+ IMAGE: ${{ needs.version.outputs.image }}
+ STAGING_IMAGE: ${{ needs.version.outputs.staging_image }}
+ VERSION: ${{ needs.version.outputs.version }}
+ SHORT_SHA: ${{ needs.version.outputs.short_sha }}
run: |
- docker build . --tag ghcr.io/firecrawl/firecrawl-staging:latest
- docker push ghcr.io/firecrawl/firecrawl-staging:latest
+ amd64_image="${IMAGE}:sha-${SHORT_SHA}-staging-linux-amd64"
+ arm64_image="${IMAGE}:sha-${SHORT_SHA}-staging-linux-arm64"
+ manifest_tags=(
+ "${IMAGE}:${VERSION}-staging"
+ "${STAGING_IMAGE}:${VERSION}-staging"
+ "${STAGING_IMAGE}:latest"
+ )
+
+ for target in "${manifest_tags[@]}"; do
+ docker manifest rm "${target}" >/dev/null 2>&1 || true
+ docker manifest create "${target}" "${amd64_image}" "${arm64_image}"
+ docker manifest push "${target}"
+ done
diff --git a/.github/workflows/deploy-image.yml b/.github/workflows/deploy-image.yml
index b3e3889aa..027f55e8b 100644
--- a/.github/workflows/deploy-image.yml
+++ b/.github/workflows/deploy-image.yml
@@ -7,17 +7,71 @@ on:
paths:
- apps/api/**
- .github/workflows/deploy-image.yml
+ - .github/scripts/resolve_api_image_version.py
workflow_dispatch:
+ inputs:
+ bump:
+ description: Version bump to apply when publishing a new deploy tag
+ required: false
+ default: patch
+ type: choice
+ options:
+ - patch
+ - minor
+
+permissions:
+ contents: write
+ packages: write
+
+concurrency:
+ group: api-image-release
+ cancel-in-progress: false
jobs:
+ version:
+ runs-on: blacksmith-2vcpu-ubuntu-2404
+ outputs:
+ version: ${{ steps.version.outputs.version }}
+ tag: ${{ steps.version.outputs.tag }}
+ tag_exists: ${{ steps.version.outputs.tag_exists }}
+ major: ${{ steps.version.outputs.major }}
+ minor: ${{ steps.version.outputs.minor }}
+ major_minor: ${{ steps.version.outputs.major_minor }}
+ short_sha: ${{ steps.version.outputs.short_sha }}
+ image: ${{ steps.version.outputs.image }}
+ steps:
+ - name: Checkout repository
+ uses: actions/checkout@v4
+ with:
+ fetch-depth: 0
+
+ - name: Ensure main ref
+ run: |
+ if [ "${GITHUB_REF}" != "refs/heads/main" ]; then
+ echo "::error::API image releases must run from refs/heads/main, got ${GITHUB_REF}"
+ exit 1
+ fi
+
+ - name: Fetch tags
+ run: git fetch --force --tags origin
+
+ - name: Resolve image version
+ id: version
+ env:
+ BUMP: ${{ github.event.inputs.bump || 'patch' }}
+ run: python .github/scripts/resolve_api_image_version.py --bump "${BUMP}" --sha "${GITHUB_SHA}"
+
build:
+ needs: version
runs-on: ${{ matrix.runner }}
strategy:
matrix:
include:
- platform: linux/amd64
+ platform_suffix: linux-amd64
runner: blacksmith-4vcpu-ubuntu-2404
- platform: linux/arm64
+ platform_suffix: linux-arm64
runner: blacksmith-4vcpu-ubuntu-2404-arm
defaults:
run:
@@ -36,30 +90,26 @@ jobs:
username: ${{github.actor}}
password: ${{secrets.GITHUB_TOKEN}}
- - name: Lowercase Repo Owner
- run: |
- echo "REPO_OWNER=${GITHUB_REPOSITORY_OWNER,,}" >>${GITHUB_ENV}
- env:
- GITHUB_REPOSITORY_OWNER: '${{ github.repository_owner }}'
-
- - name: Extract platform suffix
- run: |
- platform=${{ matrix.platform }}
- echo "PLATFORM_SUFFIX=${platform//\//-}" >> $GITHUB_ENV
-
- name: 'Build and Push Image'
uses: useblacksmith/build-push-action@30c71162f16ea2c27c3e21523255d209b8b538c1 # v2
with:
context: ./apps/api
push: true
- tags: ghcr.io/${{ env.REPO_OWNER }}/firecrawl:${{ env.PLATFORM_SUFFIX }}
+ tags: ${{ needs.version.outputs.image }}:sha-${{ needs.version.outputs.short_sha }}-${{ matrix.platform_suffix }}
platforms: ${{ matrix.platform }}
provenance: false
- manifest:
+ publish:
runs-on: blacksmith-2vcpu-ubuntu-2404
- needs: build
+ needs:
+ - version
+ - build
steps:
+ - name: Checkout repository
+ uses: actions/checkout@v4
+ with:
+ fetch-depth: 0
+
- name: 'Login to GitHub Container Registry'
uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
with:
@@ -67,15 +117,48 @@ jobs:
username: ${{github.actor}}
password: ${{secrets.GITHUB_TOKEN}}
- - name: Lowercase Repo Owner
- run: |
- echo "REPO_OWNER=${GITHUB_REPOSITORY_OWNER,,}" >>${GITHUB_ENV}
+ - name: Create Git tag
env:
- GITHUB_REPOSITORY_OWNER: '${{ github.repository_owner }}'
+ RELEASE_TAG: ${{ needs.version.outputs.tag }}
+ run: |
+ git fetch --force --tags origin
+
+ if git rev-parse -q --verify "refs/tags/${RELEASE_TAG}" >/dev/null; then
+ tag_commit="$(git rev-list -n 1 "${RELEASE_TAG}")"
+ if [ "${tag_commit}" != "${GITHUB_SHA}" ]; then
+ echo "::error::${RELEASE_TAG} already points at ${tag_commit}, not ${GITHUB_SHA}"
+ exit 1
+ fi
+
+ echo "${RELEASE_TAG} already exists on this commit; reusing it."
+ exit 0
+ fi
+
+ git config user.name "github-actions[bot]"
+ git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
+ git tag -a "${RELEASE_TAG}" "${GITHUB_SHA}" -m "Release API image ${RELEASE_TAG}"
+ git push origin "${RELEASE_TAG}"
- name: 'Create and Push Multi-Arch Manifest'
+ env:
+ IMAGE: ${{ needs.version.outputs.image }}
+ VERSION: ${{ needs.version.outputs.version }}
+ MAJOR: ${{ needs.version.outputs.major }}
+ MAJOR_MINOR: ${{ needs.version.outputs.major_minor }}
+ SHORT_SHA: ${{ needs.version.outputs.short_sha }}
run: |
- docker manifest create ghcr.io/${{ env.REPO_OWNER }}/firecrawl:latest \
- ghcr.io/${{ env.REPO_OWNER }}/firecrawl:linux-amd64 \
- ghcr.io/${{ env.REPO_OWNER }}/firecrawl:linux-arm64
- docker manifest push ghcr.io/${{ env.REPO_OWNER }}/firecrawl:latest
+ amd64_image="${IMAGE}:sha-${SHORT_SHA}-linux-amd64"
+ arm64_image="${IMAGE}:sha-${SHORT_SHA}-linux-arm64"
+ manifest_tags=(
+ "${IMAGE}:${VERSION}"
+ "${IMAGE}:${VERSION}-production"
+ "${IMAGE}:${MAJOR_MINOR}"
+ "${IMAGE}:${MAJOR}"
+ "${IMAGE}:latest"
+ )
+
+ for target in "${manifest_tags[@]}"; do
+ docker manifest rm "${target}" >/dev/null 2>&1 || true
+ docker manifest create "${target}" "${amd64_image}" "${arm64_image}"
+ docker manifest push "${target}"
+ done