Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
[codex] restore safe API image build caching (#3852)
Gergő Móricz2e053a85c46d
diff --git a/.github/workflows/deploy-image-staging.yml b/.github/workflows/deploy-image-staging.ymlindex 7d90bc735..5becfe50f 100644--- a/.github/workflows/deploy-image-staging.yml+++ b/.github/workflows/deploy-image-staging.yml@@ -73,6 +73,31 @@ jobs:tags: ${{ needs.version.outputs.image }}:sha-${{ needs.version.outputs.short_sha }}-staging-${{ matrix.platform_suffix }}platforms: ${{ matrix.platform }}provenance: false+ build-args: |+ GIT_SHA=${{ github.sha }}+ cache-from: type=registry,ref=${{ needs.version.outputs.image }}:buildcache-staging-${{ matrix.platform_suffix }}+ cache-to: type=registry,ref=${{ needs.version.outputs.image }}:buildcache-staging-${{ matrix.platform_suffix }},mode=max++ - name: Verify pushed image contents+ env:+ IMAGE_TAG: ${{ needs.version.outputs.image }}:sha-${{ needs.version.outputs.short_sha }}-staging-${{ matrix.platform_suffix }}+ EXPECTED_SHA: ${{ github.sha }}+ run: |+ docker pull "${IMAGE_TAG}"+ actual_sha="$(docker run --rm --entrypoint node "${IMAGE_TAG}" -p 'process.env.FIRECRAWL_BUILD_SHA')"+ if [ "${actual_sha}" != "${EXPECTED_SHA}" ]; then+ echo "::error::Image build SHA mismatch: expected ${EXPECTED_SHA}, got ${actual_sha}"+ exit 1+ fi++ docker run --rm --entrypoint node "${IMAGE_TAG}" -e '+ const fs = require("fs");+ for (const path of ["dist/src/harness.js", "dist/src/controllers/v0/crawl-status.js", "BUILD_SHA"]) {+ if (!fs.statSync(path).size) {+ throw new Error(`${path} missing or empty`);+ }+ }+ 'publish:runs-on: blacksmith-2vcpu-ubuntu-2404@@ -88,12 +113,15 @@ jobs:password: ${{secrets.GITHUB_TOKEN}}- name: 'Create and Push Staging Multi-Arch Manifest'+ timeout-minutes: 10env:IMAGE: ${{ needs.version.outputs.image }}STAGING_IMAGE: ${{ needs.version.outputs.staging_image }}VERSION: ${{ needs.version.outputs.version }}SHORT_SHA: ${{ needs.version.outputs.short_sha }}run: |+ set -euo pipefail+amd64_image="${IMAGE}:sha-${SHORT_SHA}-staging-linux-amd64"arm64_image="${IMAGE}:sha-${SHORT_SHA}-staging-linux-arm64"manifest_tags=(@@ -103,7 +131,9 @@ jobs:)for target in "${manifest_tags[@]}"; do- docker manifest rm "${target}" >/dev/null 2>&1 || true- docker manifest create "${target}" "${amd64_image}" "${arm64_image}"- docker manifest push "${target}"+ echo "Publishing multi-arch manifest ${target}"+ docker buildx imagetools create \+ --tag "${target}" \+ "${amd64_image}" \+ "${arm64_image}"donediff --git a/.github/workflows/deploy-image.yml b/.github/workflows/deploy-image.ymlindex 34c61a61c..9cc8e80b9 100644--- a/.github/workflows/deploy-image.yml+++ b/.github/workflows/deploy-image.yml@@ -98,7 +98,31 @@ jobs:tags: ${{ needs.version.outputs.image }}:sha-${{ needs.version.outputs.short_sha }}-${{ matrix.platform_suffix }}platforms: ${{ matrix.platform }}provenance: false- no-cache: true+ build-args: |+ GIT_SHA=${{ github.sha }}+ cache-from: type=registry,ref=${{ needs.version.outputs.image }}:buildcache-${{ matrix.platform_suffix }}+ cache-to: type=registry,ref=${{ needs.version.outputs.image }}:buildcache-${{ matrix.platform_suffix }},mode=max++ - name: Verify pushed image contents+ env:+ IMAGE_TAG: ${{ needs.version.outputs.image }}:sha-${{ needs.version.outputs.short_sha }}-${{ matrix.platform_suffix }}+ EXPECTED_SHA: ${{ github.sha }}+ run: |+ docker pull "${IMAGE_TAG}"+ actual_sha="$(docker run --rm --entrypoint node "${IMAGE_TAG}" -p 'process.env.FIRECRAWL_BUILD_SHA')"+ if [ "${actual_sha}" != "${EXPECTED_SHA}" ]; then+ echo "::error::Image build SHA mismatch: expected ${EXPECTED_SHA}, got ${actual_sha}"+ exit 1+ fi++ docker run --rm --entrypoint node "${IMAGE_TAG}" -e '+ const fs = require("fs");+ for (const path of ["dist/src/harness.js", "dist/src/controllers/v0/crawl-status.js", "BUILD_SHA"]) {+ if (!fs.statSync(path).size) {+ throw new Error(`${path} missing or empty`);+ }+ }+ 'publish:runs-on: blacksmith-2vcpu-ubuntu-2404@@ -141,6 +165,7 @@ jobs:git push origin "${RELEASE_TAG}"- name: 'Create and Push Multi-Arch Manifest'+ timeout-minutes: 10env:IMAGE: ${{ needs.version.outputs.image }}VERSION: ${{ needs.version.outputs.version }}@@ -148,6 +173,8 @@ jobs:MAJOR_MINOR: ${{ needs.version.outputs.major_minor }}SHORT_SHA: ${{ needs.version.outputs.short_sha }}run: |+ set -euo pipefail+amd64_image="${IMAGE}:sha-${SHORT_SHA}-linux-amd64"arm64_image="${IMAGE}:sha-${SHORT_SHA}-linux-arm64"manifest_tags=(@@ -159,7 +186,9 @@ jobs:)for target in "${manifest_tags[@]}"; do- docker manifest rm "${target}" >/dev/null 2>&1 || true- docker manifest create "${target}" "${amd64_image}" "${arm64_image}"- docker manifest push "${target}"+ echo "Publishing multi-arch manifest ${target}"+ docker buildx imagetools create \+ --tag "${target}" \+ "${amd64_image}" \+ "${arm64_image}"donediff --git a/apps/api/Dockerfile b/apps/api/Dockerfileindex 1896312cc..6d58102ae 100644--- a/apps/api/Dockerfile+++ b/apps/api/Dockerfile@@ -44,9 +44,12 @@ ENV RUSTUP_HOME=/usr/local/rustup \RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --no-modify-path \&& chmod -R a+w $RUSTUP_HOME $CARGO_HOME-# Copy source files+# Copy dependency inputs. The native workspace is included here because its+# install script builds the Rust package; API TypeScript source is copied after+# dependency install so source-only changes do not invalidate this layer.COPY pnpm-lock.yaml pnpm-workspace.yaml package.json ./-COPY . .+COPY patches ./patches+COPY native ./native# Install dependenciesRUN --mount=type=cache,id=pnpm,target=/pnpm/store \@@ -54,8 +57,14 @@ RUN --mount=type=cache,id=pnpm,target=/pnpm/store \--mount=type=cache,target=/app/native/target \pnpm install --frozen-lockfile+COPY . .+# Build the application from a clean output directory.RUN rm -rf dist && pnpm run build+ARG GIT_SHA=unknown+RUN test -s dist/src/harness.js && \+ test -s dist/src/controllers/v0/crawl-status.js && \+ printf '%s\n' "$GIT_SHA" > BUILD_SHA# Remove dev dependenciesRUN pnpm prune --prod --ignore-scripts@@ -80,13 +89,18 @@ RUN ARCH=$(dpkg --print-architecture) && \EXPOSE 8080WORKDIR /app+ARG GIT_SHA=unknown+ENV FIRECRAWL_BUILD_SHA=$GIT_SHA# Copy built applicationCOPY --from=build /app/node_modules ./node_modulesCOPY --from=build /app/dist ./distCOPY --from=build /app/native ./native+COPY --from=build /app/BUILD_SHA ./BUILD_SHA# Copy Go shared libraryCOPY --from=go-build /app/sharedLibs/go-html-to-md/libhtml-to-markdown.so ./sharedLibs/go-html-to-md/+RUN node -e 'const fs = require("fs"); for (const p of ["dist/src/harness.js", "dist/src/controllers/v0/crawl-status.js", "BUILD_SHA"]) { if (!fs.statSync(p).size) throw new Error(`${p} missing or empty`); }'+CMD ["node", "dist/src/harness.js", "--start-docker"]