Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
ci(api): publish semver image tags (#3781)
Gergő Móriczeeb72c5c2852
diff --git a/.github/scripts/resolve_api_image_version.py b/.github/scripts/resolve_api_image_version.pynew file mode 100644index 000000000..df5638af9--- /dev/null+++ b/.github/scripts/resolve_api_image_version.py@@ -0,0 +1,129 @@+#!/usr/bin/env python3+"""Resolve the next deployable API image version.++Deploy image versions are full SemVer Git tags: vX.Y.Z. Existing two-part+marketing release tags such as v2.10 are treated as v2.10.0 bases.+"""++from __future__ import annotations++import argparse+import os+import re+import subprocess+import sys+from dataclasses import dataclass+++VERSION_TAG_RE = re.compile(r"^v(0|[1-9]\d*)\.(0|[1-9]\d*)(?:\.(0|[1-9]\d*))?$")+++@dataclass(frozen=True)+class SemverTag:+ name: str+ version: tuple[int, int, int]+ commit: str+++def git(*args: str) -> str:+ return subprocess.check_output(["git", *args], text=True).strip()+++def list_semver_tags() -> list[SemverTag]:+ raw_tags = git("tag", "--list", "v*").splitlines()+ tags: list[SemverTag] = []++ for tag in raw_tags:+ match = VERSION_TAG_RE.match(tag)+ if not match:+ continue++ major, minor, patch = match.groups()+ commit = git("rev-list", "-n", "1", tag)+ tags.append(+ SemverTag(+ name=tag,+ version=(int(major), int(minor), int(patch or 0)),+ commit=commit,+ )+ )++ return sorted(tags, key=lambda item: item.version)+++def bump_version(base: tuple[int, int, int] | None, bump: str) -> tuple[int, int, int]:+ if base is None:+ return (0, 1, 0) if bump == "minor" else (0, 0, 1)++ major, minor, patch = base+ if bump == "minor":+ return (major, minor + 1, 0)++ return (major, minor, patch + 1)+++def write_output(name: str, value: str) -> None:+ print(f"{name}={value}")++ github_output = os.environ.get("GITHUB_OUTPUT")+ if github_output:+ with open(github_output, "a", encoding="utf-8") as output:+ output.write(f"{name}={value}\n")+++def main() -> int:+ parser = argparse.ArgumentParser()+ parser.add_argument("--bump", choices=["patch", "minor"], default="patch")+ parser.add_argument("--sha", default=os.environ.get("GITHUB_SHA", "HEAD"))+ parser.add_argument("--image-name", default="firecrawl")+ parser.add_argument(+ "--include-current-commit-tags",+ action="store_true",+ help="Use the highest SemVer tag even when it already points at --sha.",+ )+ args = parser.parse_args()++ current_sha = git("rev-parse", args.sha)+ short_sha = git("rev-parse", "--short=12", current_sha)+ semver_tags = list_semver_tags()++ if args.include_current_commit_tags:+ base_tags = semver_tags+ else:+ # Excluding tags already on this commit makes production retries+ # idempotent. If a prior attempt created vX.Y.Z but failed before+ # pushing every manifest, the same target version is reused instead of+ # burning vX.Y.(Z+1).+ base_tags = [tag for tag in semver_tags if tag.commit != current_sha]++ base = base_tags[-1].version if base_tags else None+ version = bump_version(base, args.bump)+ tag_name = f"v{version[0]}.{version[1]}.{version[2]}"++ existing_target = next((tag for tag in semver_tags if tag.name == tag_name), None)+ if existing_target and existing_target.commit != current_sha:+ print(+ f"Resolved tag {tag_name} already exists on {existing_target.commit}, "+ f"not {current_sha}",+ file=sys.stderr,+ )+ return 1++ repo_owner = os.environ.get("GITHUB_REPOSITORY_OWNER", "firecrawl").lower()+ image = f"ghcr.io/{repo_owner}/{args.image_name}"+ version_text = ".".join(str(part) for part in version)++ write_output("version", version_text)+ write_output("tag", tag_name)+ write_output("tag_exists", "true" if existing_target else "false")+ write_output("major", str(version[0]))+ write_output("minor", str(version[1]))+ write_output("major_minor", f"{version[0]}.{version[1]}")+ write_output("short_sha", short_sha)+ write_output("image", image)++ return 0+++if __name__ == "__main__":+ raise SystemExit(main())diff --git a/.github/workflows/deploy-image-staging.yml b/.github/workflows/deploy-image-staging.ymlindex 9256b1fd4..7d90bc735 100644--- a/.github/workflows/deploy-image-staging.yml+++ b/.github/workflows/deploy-image-staging.yml@@ -1,20 +1,62 @@name: STAGING Deploy Images to GHCR-env:- DOTNET_VERSION: '6.0.x'-on:workflow_dispatch:+permissions:+ contents: read+ packages: write++concurrency:+ group: api-image-staging-release+ cancel-in-progress: false+jobs:- push-app-image:- runs-on: blacksmith-4vcpu-ubuntu-2404+ version:+ runs-on: blacksmith-2vcpu-ubuntu-2404+ outputs:+ version: ${{ steps.version.outputs.version }}+ short_sha: ${{ steps.version.outputs.short_sha }}+ image: ${{ steps.version.outputs.image }}+ staging_image: ${{ steps.staging_image.outputs.image }}+ steps:+ - name: Checkout repository+ uses: actions/checkout@v4+ with:+ fetch-depth: 0++ - name: Fetch tags+ run: git fetch --force --tags origin++ - name: Resolve staging image version+ id: version+ run: python .github/scripts/resolve_api_image_version.py --bump patch --sha "${GITHUB_SHA}" --include-current-commit-tags++ - name: Resolve staging image repository+ id: staging_image+ run: echo "image=ghcr.io/${GITHUB_REPOSITORY_OWNER,,}/firecrawl-staging" >> "$GITHUB_OUTPUT"++ build:+ needs: version+ runs-on: ${{ matrix.runner }}+ strategy:+ matrix:+ include:+ - platform: linux/amd64+ platform_suffix: linux-amd64+ runner: blacksmith-4vcpu-ubuntu-2404+ - platform: linux/arm64+ platform_suffix: linux-arm64+ runner: blacksmith-4vcpu-ubuntu-2404-armdefaults:run:working-directory: './apps/api'steps:- - name: 'Checkout GitHub Action'- uses: actions/checkout@main+ - name: Checkout repository+ uses: actions/checkout@v4++ - name: Setup Blacksmith Builder+ uses: useblacksmith/setup-docker-builder@ef12d5b165b596e3aa44ea8198d8fde563eab402 # v1- name: 'Login to GitHub Container Registry'uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3@@ -23,7 +65,45 @@ jobs:username: ${{github.actor}}password: ${{secrets.GITHUB_TOKEN}}- - name: 'Build Inventory Image'+ - name: 'Build and Push Image'+ uses: useblacksmith/build-push-action@30c71162f16ea2c27c3e21523255d209b8b538c1 # v2+ with:+ context: ./apps/api+ push: true+ tags: ${{ needs.version.outputs.image }}:sha-${{ needs.version.outputs.short_sha }}-staging-${{ matrix.platform_suffix }}+ platforms: ${{ matrix.platform }}+ provenance: false++ publish:+ runs-on: blacksmith-2vcpu-ubuntu-2404+ needs:+ - version+ - build+ steps:+ - name: 'Login to GitHub Container Registry'+ uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3+ with:+ registry: ghcr.io+ username: ${{github.actor}}+ password: ${{secrets.GITHUB_TOKEN}}++ - name: 'Create and Push Staging Multi-Arch Manifest'+ env:+ IMAGE: ${{ needs.version.outputs.image }}+ STAGING_IMAGE: ${{ needs.version.outputs.staging_image }}+ VERSION: ${{ needs.version.outputs.version }}+ SHORT_SHA: ${{ needs.version.outputs.short_sha }}run: |- docker build . --tag ghcr.io/firecrawl/firecrawl-staging:latest- docker push ghcr.io/firecrawl/firecrawl-staging:latest+ amd64_image="${IMAGE}:sha-${SHORT_SHA}-staging-linux-amd64"+ arm64_image="${IMAGE}:sha-${SHORT_SHA}-staging-linux-arm64"+ manifest_tags=(+ "${IMAGE}:${VERSION}-staging"+ "${STAGING_IMAGE}:${VERSION}-staging"+ "${STAGING_IMAGE}:latest"+ )++ for target in "${manifest_tags[@]}"; do+ docker manifest rm "${target}" >/dev/null 2>&1 || true+ docker manifest create "${target}" "${amd64_image}" "${arm64_image}"+ docker manifest push "${target}"+ donediff --git a/.github/workflows/deploy-image.yml b/.github/workflows/deploy-image.ymlindex b3e3889aa..027f55e8b 100644--- a/.github/workflows/deploy-image.yml+++ b/.github/workflows/deploy-image.yml@@ -7,17 +7,71 @@ on:paths:- apps/api/**- .github/workflows/deploy-image.yml+ - .github/scripts/resolve_api_image_version.pyworkflow_dispatch:+ inputs:+ bump:+ description: Version bump to apply when publishing a new deploy tag+ required: false+ default: patch+ type: choice+ options:+ - patch+ - minor++permissions:+ contents: write+ packages: write++concurrency:+ group: api-image-release+ cancel-in-progress: falsejobs:+ version:+ runs-on: blacksmith-2vcpu-ubuntu-2404+ outputs:+ version: ${{ steps.version.outputs.version }}+ tag: ${{ steps.version.outputs.tag }}+ tag_exists: ${{ steps.version.outputs.tag_exists }}+ major: ${{ steps.version.outputs.major }}+ minor: ${{ steps.version.outputs.minor }}+ major_minor: ${{ steps.version.outputs.major_minor }}+ short_sha: ${{ steps.version.outputs.short_sha }}+ image: ${{ steps.version.outputs.image }}+ steps:+ - name: Checkout repository+ uses: actions/checkout@v4+ with:+ fetch-depth: 0++ - name: Ensure main ref+ run: |+ if [ "${GITHUB_REF}" != "refs/heads/main" ]; then+ echo "::error::API image releases must run from refs/heads/main, got ${GITHUB_REF}"+ exit 1+ fi++ - name: Fetch tags+ run: git fetch --force --tags origin++ - name: Resolve image version+ id: version+ env:+ BUMP: ${{ github.event.inputs.bump || 'patch' }}+ run: python .github/scripts/resolve_api_image_version.py --bump "${BUMP}" --sha "${GITHUB_SHA}"+build:+ needs: versionruns-on: ${{ matrix.runner }}strategy:matrix:include:- platform: linux/amd64+ platform_suffix: linux-amd64runner: blacksmith-4vcpu-ubuntu-2404- platform: linux/arm64+ platform_suffix: linux-arm64runner: blacksmith-4vcpu-ubuntu-2404-armdefaults:run:@@ -36,30 +90,26 @@ jobs:username: ${{github.actor}}password: ${{secrets.GITHUB_TOKEN}}- - name: Lowercase Repo Owner- run: |- echo "REPO_OWNER=${GITHUB_REPOSITORY_OWNER,,}" >>${GITHUB_ENV}- env:- GITHUB_REPOSITORY_OWNER: '${{ github.repository_owner }}'-- - name: Extract platform suffix- run: |- platform=${{ matrix.platform }}- echo "PLATFORM_SUFFIX=${platform//\//-}" >> $GITHUB_ENV-- name: 'Build and Push Image'uses: useblacksmith/build-push-action@30c71162f16ea2c27c3e21523255d209b8b538c1 # v2with:context: ./apps/apipush: true- tags: ghcr.io/${{ env.REPO_OWNER }}/firecrawl:${{ env.PLATFORM_SUFFIX }}+ tags: ${{ needs.version.outputs.image }}:sha-${{ needs.version.outputs.short_sha }}-${{ matrix.platform_suffix }}platforms: ${{ matrix.platform }}provenance: false- manifest:+ publish:runs-on: blacksmith-2vcpu-ubuntu-2404- needs: build+ needs:+ - version+ - buildsteps:+ - name: Checkout repository+ uses: actions/checkout@v4+ with:+ fetch-depth: 0+- name: 'Login to GitHub Container Registry'uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3with:@@ -67,15 +117,48 @@ jobs:username: ${{github.actor}}password: ${{secrets.GITHUB_TOKEN}}- - name: Lowercase Repo Owner- run: |- echo "REPO_OWNER=${GITHUB_REPOSITORY_OWNER,,}" >>${GITHUB_ENV}+ - name: Create Git tagenv:- GITHUB_REPOSITORY_OWNER: '${{ github.repository_owner }}'+ RELEASE_TAG: ${{ needs.version.outputs.tag }}+ run: |+ git fetch --force --tags origin++ if git rev-parse -q --verify "refs/tags/${RELEASE_TAG}" >/dev/null; then+ tag_commit="$(git rev-list -n 1 "${RELEASE_TAG}")"+ if [ "${tag_commit}" != "${GITHUB_SHA}" ]; then+ echo "::error::${RELEASE_TAG} already points at ${tag_commit}, not ${GITHUB_SHA}"+ exit 1+ fi++ echo "${RELEASE_TAG} already exists on this commit; reusing it."+ exit 0+ fi++ git config user.name "github-actions[bot]"+ git config user.email "41898282+github-actions[bot]@users.noreply.github.com"+ git tag -a "${RELEASE_TAG}" "${GITHUB_SHA}" -m "Release API image ${RELEASE_TAG}"+ git push origin "${RELEASE_TAG}"- name: 'Create and Push Multi-Arch Manifest'+ env:+ IMAGE: ${{ needs.version.outputs.image }}+ VERSION: ${{ needs.version.outputs.version }}+ MAJOR: ${{ needs.version.outputs.major }}+ MAJOR_MINOR: ${{ needs.version.outputs.major_minor }}+ SHORT_SHA: ${{ needs.version.outputs.short_sha }}run: |- docker manifest create ghcr.io/${{ env.REPO_OWNER }}/firecrawl:latest \- ghcr.io/${{ env.REPO_OWNER }}/firecrawl:linux-amd64 \- ghcr.io/${{ env.REPO_OWNER }}/firecrawl:linux-arm64- docker manifest push ghcr.io/${{ env.REPO_OWNER }}/firecrawl:latest+ amd64_image="${IMAGE}:sha-${SHORT_SHA}-linux-amd64"+ arm64_image="${IMAGE}:sha-${SHORT_SHA}-linux-arm64"+ manifest_tags=(+ "${IMAGE}:${VERSION}"+ "${IMAGE}:${VERSION}-production"+ "${IMAGE}:${MAJOR_MINOR}"+ "${IMAGE}:${MAJOR}"+ "${IMAGE}:latest"+ )++ for target in "${manifest_tags[@]}"; do+ docker manifest rm "${target}" >/dev/null 2>&1 || true+ docker manifest create "${target}" "${amd64_image}" "${arm64_image}"+ docker manifest push "${target}"+ done