Skip to content

A self-hostable bookmark-everything app (links, notes and images) with AI-based automatic tagging and full text search

ci: optimize docker image building for caching (#3125)

Mohamed Bassem36afd0552a92
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index b0bc5835..8e1d1795 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -84,6 +84,11 @@ jobs:
working-directory: packages/e2e_tests
run: pnpm exec playwright install --with-deps chromium
+ # The default docker driver can't import the registry build cache that
+ # the e2e docker-compose file uses, so build with a buildx builder.
+ - name: Set up Docker Buildx
+ uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3
+
- name: E2E Tests
working-directory: packages/e2e_tests
run: pnpm test
diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml
index c5ebe8e7..f34ab1ad 100644
--- a/.github/workflows/docker.yml
+++ b/.github/workflows/docker.yml
@@ -12,40 +12,11 @@ jobs:
strategy:
fail-fast: false
matrix:
- platform: [linux/amd64, linux/arm64]
- image: [web, workers, cli, mcp, aio]
include:
- - platform: linux/amd64
- suffix: amd64
+ - arch: amd64
os: ubuntu-latest
- - platform: linux/arm64
- suffix: arm64
+ - arch: arm64
os: ubuntu-24.04-arm
- - image: web
- name: karakeep-web
- target: web
- tags_latest: ghcr.io/hoarder-app/hoarder-web:latest,ghcr.io/karakeep-app/karakeep-web:latest
- tags_release: ghcr.io/hoarder-app/hoarder-web:${{ github.event.release.name }},ghcr.io/karakeep-app/karakeep-web:${{ github.event.release.name }},ghcr.io/hoarder-app/hoarder-web:release,ghcr.io/karakeep-app/karakeep-web:release
- - image: workers
- name: karakeep-workers
- target: workers
- tags_latest: ghcr.io/hoarder-app/hoarder-workers:latest,ghcr.io/karakeep-app/karakeep-workers:latest
- tags_release: ghcr.io/hoarder-app/hoarder-workers:${{ github.event.release.name }},ghcr.io/karakeep-app/karakeep-workers:${{ github.event.release.name }},ghcr.io/hoarder-app/hoarder-workers:release,ghcr.io/karakeep-app/karakeep-workers:release
- - image: cli
- name: karakeep-cli
- target: cli
- tags_latest: ghcr.io/hoarder-app/hoarder-cli:latest,ghcr.io/karakeep-app/karakeep-cli:latest
- tags_release: ghcr.io/hoarder-app/hoarder-cli:${{ github.event.release.name }},ghcr.io/karakeep-app/karakeep-cli:${{ github.event.release.name }},ghcr.io/hoarder-app/hoarder-cli:release,ghcr.io/karakeep-app/karakeep-cli:release
- - image: mcp
- name: karakeep-mcp
- target: mcp
- tags_latest: ghcr.io/karakeep-app/karakeep-mcp:latest
- tags_release: ghcr.io/karakeep-app/karakeep-mcp:${{ github.event.release.name }},ghcr.io/karakeep-app/karakeep-mcp:release
- - image: aio
- name: karakeep-aio
- target: aio
- tags_latest: ghcr.io/hoarder-app/hoarder:latest,ghcr.io/karakeep-app/karakeep:latest
- tags_release: ghcr.io/hoarder-app/hoarder:${{ github.event.release.name }},ghcr.io/karakeep-app/karakeep:${{ github.event.release.name }},ghcr.io/hoarder-app/hoarder:release,ghcr.io/karakeep-app/karakeep:release
runs-on: ${{ matrix.os }}
permissions:
packages: write
@@ -65,50 +36,20 @@ jobs:
username: ${{ github.actor }}
password: ${{ secrets.GHCR_GITHUB_PAT }}
- - name: Prepare tags
- id: tags
- run: |
- set -euo pipefail
- add_suffix() {
- local value="$1"
- local out=""
- IFS=',' read -ra items <<< "$value"
- for item in "${items[@]}"; do
- out+="${item}-${{ matrix.suffix }},"
- done
- echo "${out%,}"
- }
-
- all_tags=""
-
- # Only push 'latest' tags on pushes to main, not on releases
- if [[ "${{ github.event_name }}" == "push" ]]; then
- latest_with_suffix=$(add_suffix "${{ matrix.tags_latest }}")
- all_tags="${latest_with_suffix}"
- echo "latest=${latest_with_suffix}" >> "$GITHUB_OUTPUT"
- fi
-
- # Only push release-specific tags on releases
- if [[ "${{ github.event_name }}" == "release" ]]; then
- release_with_suffix=$(add_suffix "${{ matrix.tags_release }}")
- all_tags="${release_with_suffix}"
- echo "release=${release_with_suffix}" >> "$GITHUB_OUTPUT"
- fi
-
- echo "all=${all_tags}" >> "$GITHUB_OUTPUT"
-
- - name: Build ${{ matrix.name }}
- uses: docker/build-push-action@ca052bb54ab0790a636c9b5f226502c73d547a25 # v5
+ # Builds all the images in a single BuildKit session so that the stages
+ # they share are only built once. Tags and caches are defined in the bake
+ # file: pushes to main get the 'latest' tags, releases get the
+ # release-specific ones.
+ - name: Build images (${{ matrix.arch }})
+ uses: docker/bake-action@3acf805d94d93a86cce4ca44798a76464a75b88c # v6.9.0
+ env:
+ ARCH: ${{ matrix.arch }}
+ RELEASE: ${{ github.event_name == 'release' && github.event.release.name || '' }}
+ SERVER_VERSION: ${{ github.event_name == 'release' && github.event.release.name || 'nightly' }}
with:
- context: .
- build-args: SERVER_VERSION=${{ github.event_name == 'release' && github.event.release.name || 'nightly' }}
- file: docker/Dockerfile
- target: ${{ matrix.target }}
- platforms: ${{ matrix.platform }}
+ source: .
+ files: docker/docker-bake.hcl
push: true
- tags: ${{ steps.tags.outputs.all }}
- cache-from: type=registry,ref=ghcr.io/karakeep-app/karakeep-build-cache:${{ matrix.target }}-${{ matrix.suffix }}
- cache-to: type=registry,mode=max,ref=ghcr.io/karakeep-app/karakeep-build-cache:${{ matrix.target }}-${{ matrix.suffix }}
manifest:
needs: build
diff --git a/docker/Dockerfile b/docker/Dockerfile
index 1d6ae273..2def9749 100644
--- a/docker/Dockerfile
+++ b/docker/Dockerfile
@@ -55,23 +55,36 @@ RUN pnpm install --frozen-lockfile
COPY . .
-# Build the db migration script
+# Each app is built in its own stage so that images only build the apps they
+# ship (e.g. the cli image doesn't compile the web app) and BuildKit can build
+# them in parallel.
+
+################# DB Migrations Builder ##############
+FROM base AS db_migrations_builder
+
RUN cd packages/db && \
pnpm exec ncc build migrate.ts -o /db_migrations && \
cp -R drizzle /db_migrations
+################# Web Builder ##############
+FROM base AS web_builder
-# Compile the web app
RUN (cd apps/web && pnpm exec next build --experimental-build-mode compile)
-# Build the worker code
+################# Workers Builder ##############
+FROM base AS workers_builder
+
RUN (cd apps/workers && pnpm --config.verify-deps-before-run=false build)
RUN CI=true pnpm deploy --node-linker=isolated --legacy --filter @karakeep/workers --prod /prod/workers
-# Build the cli
+################# CLI Builder ##############
+FROM base AS cli_builder
+
RUN (cd apps/cli && pnpm --config.verify-deps-before-run=false build)
-# Build the mcp server
+################# MCP Builder ##############
+FROM base AS mcp_builder
+
RUN (cd apps/mcp && pnpm --config.verify-deps-before-run=false build)
################# The All-in-one builder ##############
@@ -154,21 +167,21 @@ ENV NODE_ENV production
ENV NEXT_TELEMETRY_DISABLED=1
ENV NEXTAUTH_URL_INTERNAL=http://localhost:${PORT}
-COPY --from=base --chown=node:node /app/apps/web/.next/standalone ./
-COPY --from=base /app/apps/web/public ./apps/web/public
-COPY --from=base /db_migrations /db_migrations
+COPY --from=web_builder --chown=node:node /app/apps/web/.next/standalone ./
+COPY --from=web_builder /app/apps/web/public ./apps/web/public
+COPY --from=db_migrations_builder /db_migrations /db_migrations
# Set the correct permission for prerender cache
RUN mkdir -p ./apps/web/.next && chown node:node ./apps/web/.next
# Automatically leverage output traces to reduce image size
# https://nextjs.org/docs/advanced-features/output-file-tracing
-COPY --from=base --chown=node:node /app/apps/web/.next/static ./apps/web/.next/static
+COPY --from=web_builder --chown=node:node /app/apps/web/.next/static ./apps/web/.next/static
######################
# Prepare the workers app
######################
-COPY --from=base /prod/workers /app/apps/workers
+COPY --from=workers_builder /prod/workers /app/apps/workers
ENTRYPOINT ["/init"]
@@ -209,7 +222,7 @@ FROM node:24.18.1-slim AS cli
LABEL org.opencontainers.image.source="https://github.com/karakeep-app/karakeep"
WORKDIR /app
-COPY --from=base /app/apps/cli/dist/index.mjs apps/cli/index.mjs
+COPY --from=cli_builder /app/apps/cli/dist/index.mjs apps/cli/index.mjs
WORKDIR /app/apps/cli
@@ -225,7 +238,7 @@ FROM node:24.18.1-slim AS mcp
LABEL org.opencontainers.image.source="https://github.com/karakeep-app/karakeep"
WORKDIR /app
-COPY --from=base /app/apps/mcp/dist/index.js apps/mcp/index.js
+COPY --from=mcp_builder /app/apps/mcp/dist/index.js apps/mcp/index.js
WORKDIR /app/apps/mcp
diff --git a/docker/docker-bake.hcl b/docker/docker-bake.hcl
new file mode 100644
index 00000000..51b04eae
--- /dev/null
+++ b/docker/docker-bake.hcl
@@ -0,0 +1,100 @@
+# Builds all the images for a single architecture in one BuildKit session so
+# that the stages they share (deps install, app builds, monolith, ...) are only
+# built once.
+#
+# Usage: ARCH=amd64 docker buildx bake -f docker/docker-bake.hcl --push
+
+# The architecture to build for (amd64 or arm64). Tags and caches are suffixed
+# with it; the manifest job in CI stitches the architectures together.
+variable "ARCH" {
+ default = "amd64"
+}
+
+# The release name. When empty, the images are tagged as `latest`, otherwise
+# they're tagged with the release name and `release`.
+variable "RELEASE" {
+ default = ""
+}
+
+variable "SERVER_VERSION" {
+ default = "nightly"
+}
+
+variable "CACHE_REPO" {
+ default = "ghcr.io/karakeep-app/karakeep-build-cache"
+}
+
+function "tags" {
+ params = [repos]
+ result = flatten([
+ for repo in repos : RELEASE == ""
+ ? ["${repo}:latest-${ARCH}"]
+ : ["${repo}:${RELEASE}-${ARCH}", "${repo}:release-${ARCH}"]
+ ])
+}
+
+# Each image exports its full (mode=max) build graph to its own cache ref.
+# packages/e2e_tests/docker-compose.yml imports the aio-amd64 cache, so keep it
+# in sync when renaming the refs.
+function "cache_from" {
+ params = [name]
+ result = ["type=registry,ref=${CACHE_REPO}:${name}-${ARCH}"]
+}
+
+function "cache_to" {
+ params = [name]
+ result = ["type=registry,mode=max,ref=${CACHE_REPO}:${name}-${ARCH}"]
+}
+
+group "default" {
+ targets = ["web", "workers", "cli", "mcp", "aio"]
+}
+
+target "_common" {
+ context = "."
+ dockerfile = "docker/Dockerfile"
+ platforms = ["linux/${ARCH}"]
+ args = {
+ SERVER_VERSION = SERVER_VERSION
+ }
+}
+
+target "web" {
+ inherits = ["_common"]
+ target = "web"
+ tags = tags(["ghcr.io/hoarder-app/hoarder-web", "ghcr.io/karakeep-app/karakeep-web"])
+ cache-from = cache_from("web")
+ cache-to = cache_to("web")
+}
+
+target "workers" {
+ inherits = ["_common"]
+ target = "workers"
+ tags = tags(["ghcr.io/hoarder-app/hoarder-workers", "ghcr.io/karakeep-app/karakeep-workers"])
+ cache-from = cache_from("workers")
+ cache-to = cache_to("workers")
+}
+
+target "cli" {
+ inherits = ["_common"]
+ target = "cli"
+ tags = tags(["ghcr.io/hoarder-app/hoarder-cli", "ghcr.io/karakeep-app/karakeep-cli"])
+ cache-from = cache_from("cli")
+ cache-to = cache_to("cli")
+}
+
+target "mcp" {
+ inherits = ["_common"]
+ target = "mcp"
+ tags = tags(["ghcr.io/karakeep-app/karakeep-mcp"])
+ cache-from = cache_from("mcp")
+ cache-to = cache_to("mcp")
+}
+
+target "aio" {
+ inherits = ["_common"]
+ target = "aio"
+ tags = tags(["ghcr.io/hoarder-app/hoarder", "ghcr.io/karakeep-app/karakeep"])
+ cache-from = cache_from("aio")
+ cache-to = cache_to("aio")
+}
diff --git a/packages/e2e_tests/docker-compose.yml b/packages/e2e_tests/docker-compose.yml
index dc95a462..f6757e39 100644
--- a/packages/e2e_tests/docker-compose.yml
+++ b/packages/e2e_tests/docker-compose.yml
@@ -4,6 +4,13 @@ services:
dockerfile: docker/Dockerfile
context: ../../
target: aio
+ # Reuse the layers (deps install, monolith, ...) that the docker workflow
+ # pushes from main. The checked-out source is still what gets built; only
+ # unchanged layers are reused. The cache is public so that it's readable
+ # without credentials (e.g. from fork PRs). If it's unavailable, buildkit
+ # just warns and builds from scratch.
+ cache_from:
+ - type=registry,ref=ghcr.io/karakeep-app/karakeep-build-cache:aio-amd64
restart: unless-stopped
ports:
- "${KARAKEEP_PORT:-3000}:3000"