Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

[codex] restore safe API image build caching (#3852)

Gergő Móricz2e053a85c46d
diff --git a/.github/workflows/deploy-image-staging.yml b/.github/workflows/deploy-image-staging.yml
index 7d90bc735..5becfe50f 100644
--- a/.github/workflows/deploy-image-staging.yml
+++ b/.github/workflows/deploy-image-staging.yml
@@ -73,6 +73,31 @@ jobs:
tags: ${{ needs.version.outputs.image }}:sha-${{ needs.version.outputs.short_sha }}-staging-${{ matrix.platform_suffix }}
platforms: ${{ matrix.platform }}
provenance: false
+ build-args: |
+ GIT_SHA=${{ github.sha }}
+ cache-from: type=registry,ref=${{ needs.version.outputs.image }}:buildcache-staging-${{ matrix.platform_suffix }}
+ cache-to: type=registry,ref=${{ needs.version.outputs.image }}:buildcache-staging-${{ matrix.platform_suffix }},mode=max
+
+ - name: Verify pushed image contents
+ env:
+ IMAGE_TAG: ${{ needs.version.outputs.image }}:sha-${{ needs.version.outputs.short_sha }}-staging-${{ matrix.platform_suffix }}
+ EXPECTED_SHA: ${{ github.sha }}
+ run: |
+ docker pull "${IMAGE_TAG}"
+ actual_sha="$(docker run --rm --entrypoint node "${IMAGE_TAG}" -p 'process.env.FIRECRAWL_BUILD_SHA')"
+ if [ "${actual_sha}" != "${EXPECTED_SHA}" ]; then
+ echo "::error::Image build SHA mismatch: expected ${EXPECTED_SHA}, got ${actual_sha}"
+ exit 1
+ fi
+
+ docker run --rm --entrypoint node "${IMAGE_TAG}" -e '
+ const fs = require("fs");
+ for (const path of ["dist/src/harness.js", "dist/src/controllers/v0/crawl-status.js", "BUILD_SHA"]) {
+ if (!fs.statSync(path).size) {
+ throw new Error(`${path} missing or empty`);
+ }
+ }
+ '
publish:
runs-on: blacksmith-2vcpu-ubuntu-2404
@@ -88,12 +113,15 @@ jobs:
password: ${{secrets.GITHUB_TOKEN}}
- name: 'Create and Push Staging Multi-Arch Manifest'
+ timeout-minutes: 10
env:
IMAGE: ${{ needs.version.outputs.image }}
STAGING_IMAGE: ${{ needs.version.outputs.staging_image }}
VERSION: ${{ needs.version.outputs.version }}
SHORT_SHA: ${{ needs.version.outputs.short_sha }}
run: |
+ set -euo pipefail
+
amd64_image="${IMAGE}:sha-${SHORT_SHA}-staging-linux-amd64"
arm64_image="${IMAGE}:sha-${SHORT_SHA}-staging-linux-arm64"
manifest_tags=(
@@ -103,7 +131,9 @@ jobs:
)
for target in "${manifest_tags[@]}"; do
- docker manifest rm "${target}" >/dev/null 2>&1 || true
- docker manifest create "${target}" "${amd64_image}" "${arm64_image}"
- docker manifest push "${target}"
+ echo "Publishing multi-arch manifest ${target}"
+ docker buildx imagetools create \
+ --tag "${target}" \
+ "${amd64_image}" \
+ "${arm64_image}"
done
diff --git a/.github/workflows/deploy-image.yml b/.github/workflows/deploy-image.yml
index 34c61a61c..9cc8e80b9 100644
--- a/.github/workflows/deploy-image.yml
+++ b/.github/workflows/deploy-image.yml
@@ -98,7 +98,31 @@ jobs:
tags: ${{ needs.version.outputs.image }}:sha-${{ needs.version.outputs.short_sha }}-${{ matrix.platform_suffix }}
platforms: ${{ matrix.platform }}
provenance: false
- no-cache: true
+ build-args: |
+ GIT_SHA=${{ github.sha }}
+ cache-from: type=registry,ref=${{ needs.version.outputs.image }}:buildcache-${{ matrix.platform_suffix }}
+ cache-to: type=registry,ref=${{ needs.version.outputs.image }}:buildcache-${{ matrix.platform_suffix }},mode=max
+
+ - name: Verify pushed image contents
+ env:
+ IMAGE_TAG: ${{ needs.version.outputs.image }}:sha-${{ needs.version.outputs.short_sha }}-${{ matrix.platform_suffix }}
+ EXPECTED_SHA: ${{ github.sha }}
+ run: |
+ docker pull "${IMAGE_TAG}"
+ actual_sha="$(docker run --rm --entrypoint node "${IMAGE_TAG}" -p 'process.env.FIRECRAWL_BUILD_SHA')"
+ if [ "${actual_sha}" != "${EXPECTED_SHA}" ]; then
+ echo "::error::Image build SHA mismatch: expected ${EXPECTED_SHA}, got ${actual_sha}"
+ exit 1
+ fi
+
+ docker run --rm --entrypoint node "${IMAGE_TAG}" -e '
+ const fs = require("fs");
+ for (const path of ["dist/src/harness.js", "dist/src/controllers/v0/crawl-status.js", "BUILD_SHA"]) {
+ if (!fs.statSync(path).size) {
+ throw new Error(`${path} missing or empty`);
+ }
+ }
+ '
publish:
runs-on: blacksmith-2vcpu-ubuntu-2404
@@ -141,6 +165,7 @@ jobs:
git push origin "${RELEASE_TAG}"
- name: 'Create and Push Multi-Arch Manifest'
+ timeout-minutes: 10
env:
IMAGE: ${{ needs.version.outputs.image }}
VERSION: ${{ needs.version.outputs.version }}
@@ -148,6 +173,8 @@ jobs:
MAJOR_MINOR: ${{ needs.version.outputs.major_minor }}
SHORT_SHA: ${{ needs.version.outputs.short_sha }}
run: |
+ set -euo pipefail
+
amd64_image="${IMAGE}:sha-${SHORT_SHA}-linux-amd64"
arm64_image="${IMAGE}:sha-${SHORT_SHA}-linux-arm64"
manifest_tags=(
@@ -159,7 +186,9 @@ jobs:
)
for target in "${manifest_tags[@]}"; do
- docker manifest rm "${target}" >/dev/null 2>&1 || true
- docker manifest create "${target}" "${amd64_image}" "${arm64_image}"
- docker manifest push "${target}"
+ echo "Publishing multi-arch manifest ${target}"
+ docker buildx imagetools create \
+ --tag "${target}" \
+ "${amd64_image}" \
+ "${arm64_image}"
done
diff --git a/apps/api/Dockerfile b/apps/api/Dockerfile
index 1896312cc..6d58102ae 100644
--- a/apps/api/Dockerfile
+++ b/apps/api/Dockerfile
@@ -44,9 +44,12 @@ ENV RUSTUP_HOME=/usr/local/rustup \
RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --no-modify-path \
&& chmod -R a+w $RUSTUP_HOME $CARGO_HOME
-# Copy source files
+# Copy dependency inputs. The native workspace is included here because its
+# install script builds the Rust package; API TypeScript source is copied after
+# dependency install so source-only changes do not invalidate this layer.
COPY pnpm-lock.yaml pnpm-workspace.yaml package.json ./
-COPY . .
+COPY patches ./patches
+COPY native ./native
# Install dependencies
RUN --mount=type=cache,id=pnpm,target=/pnpm/store \
@@ -54,8 +57,14 @@ RUN --mount=type=cache,id=pnpm,target=/pnpm/store \
--mount=type=cache,target=/app/native/target \
pnpm install --frozen-lockfile
+COPY . .
+
# Build the application from a clean output directory.
RUN rm -rf dist && pnpm run build
+ARG GIT_SHA=unknown
+RUN test -s dist/src/harness.js && \
+ test -s dist/src/controllers/v0/crawl-status.js && \
+ printf '%s\n' "$GIT_SHA" > BUILD_SHA
# Remove dev dependencies
RUN pnpm prune --prod --ignore-scripts
@@ -80,13 +89,18 @@ RUN ARCH=$(dpkg --print-architecture) && \
EXPOSE 8080
WORKDIR /app
+ARG GIT_SHA=unknown
+ENV FIRECRAWL_BUILD_SHA=$GIT_SHA
# Copy built application
COPY --from=build /app/node_modules ./node_modules
COPY --from=build /app/dist ./dist
COPY --from=build /app/native ./native
+COPY --from=build /app/BUILD_SHA ./BUILD_SHA
# Copy Go shared library
COPY --from=go-build /app/sharedLibs/go-html-to-md/libhtml-to-markdown.so ./sharedLibs/go-html-to-md/
+RUN node -e 'const fs = require("fs"); for (const p of ["dist/src/harness.js", "dist/src/controllers/v0/crawl-status.js", "BUILD_SHA"]) { if (!fs.statSync(p).size) throw new Error(`${p} missing or empty`); }'
+
CMD ["node", "dist/src/harness.js", "--start-docker"]