A self-hostable bookmark-everything app (links, notes and images) with AI-based automatic tagging and full text search
ci: optimize docker image building for caching (#3125)
Mohamed Bassem36afd0552a92
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.ymlindex b0bc5835..8e1d1795 100644--- a/.github/workflows/ci.yml+++ b/.github/workflows/ci.yml@@ -84,6 +84,11 @@ jobs:working-directory: packages/e2e_testsrun: pnpm exec playwright install --with-deps chromium+ # The default docker driver can't import the registry build cache that+ # the e2e docker-compose file uses, so build with a buildx builder.+ - name: Set up Docker Buildx+ uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3+- name: E2E Testsworking-directory: packages/e2e_testsrun: pnpm testdiff --git a/.github/workflows/docker.yml b/.github/workflows/docker.ymlindex c5ebe8e7..f34ab1ad 100644--- a/.github/workflows/docker.yml+++ b/.github/workflows/docker.yml@@ -12,40 +12,11 @@ jobs:strategy:fail-fast: falsematrix:- platform: [linux/amd64, linux/arm64]- image: [web, workers, cli, mcp, aio]include:- - platform: linux/amd64- suffix: amd64+ - arch: amd64os: ubuntu-latest- - platform: linux/arm64- suffix: arm64+ - arch: arm64os: ubuntu-24.04-arm- - image: web- name: karakeep-web- target: web- tags_latest: ghcr.io/hoarder-app/hoarder-web:latest,ghcr.io/karakeep-app/karakeep-web:latest- tags_release: ghcr.io/hoarder-app/hoarder-web:${{ github.event.release.name }},ghcr.io/karakeep-app/karakeep-web:${{ github.event.release.name }},ghcr.io/hoarder-app/hoarder-web:release,ghcr.io/karakeep-app/karakeep-web:release- - image: workers- name: karakeep-workers- target: workers- tags_latest: ghcr.io/hoarder-app/hoarder-workers:latest,ghcr.io/karakeep-app/karakeep-workers:latest- tags_release: ghcr.io/hoarder-app/hoarder-workers:${{ github.event.release.name }},ghcr.io/karakeep-app/karakeep-workers:${{ github.event.release.name }},ghcr.io/hoarder-app/hoarder-workers:release,ghcr.io/karakeep-app/karakeep-workers:release- - image: cli- name: karakeep-cli- target: cli- tags_latest: ghcr.io/hoarder-app/hoarder-cli:latest,ghcr.io/karakeep-app/karakeep-cli:latest- tags_release: ghcr.io/hoarder-app/hoarder-cli:${{ github.event.release.name }},ghcr.io/karakeep-app/karakeep-cli:${{ github.event.release.name }},ghcr.io/hoarder-app/hoarder-cli:release,ghcr.io/karakeep-app/karakeep-cli:release- - image: mcp- name: karakeep-mcp- target: mcp- tags_latest: ghcr.io/karakeep-app/karakeep-mcp:latest- tags_release: ghcr.io/karakeep-app/karakeep-mcp:${{ github.event.release.name }},ghcr.io/karakeep-app/karakeep-mcp:release- - image: aio- name: karakeep-aio- target: aio- tags_latest: ghcr.io/hoarder-app/hoarder:latest,ghcr.io/karakeep-app/karakeep:latest- tags_release: ghcr.io/hoarder-app/hoarder:${{ github.event.release.name }},ghcr.io/karakeep-app/karakeep:${{ github.event.release.name }},ghcr.io/hoarder-app/hoarder:release,ghcr.io/karakeep-app/karakeep:releaseruns-on: ${{ matrix.os }}permissions:packages: write@@ -65,50 +36,20 @@ jobs:username: ${{ github.actor }}password: ${{ secrets.GHCR_GITHUB_PAT }}- - name: Prepare tags- id: tags- run: |- set -euo pipefail- add_suffix() {- local value="$1"- local out=""- IFS=',' read -ra items <<< "$value"- for item in "${items[@]}"; do- out+="${item}-${{ matrix.suffix }},"- done- echo "${out%,}"- }-- all_tags=""-- # Only push 'latest' tags on pushes to main, not on releases- if [[ "${{ github.event_name }}" == "push" ]]; then- latest_with_suffix=$(add_suffix "${{ matrix.tags_latest }}")- all_tags="${latest_with_suffix}"- echo "latest=${latest_with_suffix}" >> "$GITHUB_OUTPUT"- fi-- # Only push release-specific tags on releases- if [[ "${{ github.event_name }}" == "release" ]]; then- release_with_suffix=$(add_suffix "${{ matrix.tags_release }}")- all_tags="${release_with_suffix}"- echo "release=${release_with_suffix}" >> "$GITHUB_OUTPUT"- fi-- echo "all=${all_tags}" >> "$GITHUB_OUTPUT"-- - name: Build ${{ matrix.name }}- uses: docker/build-push-action@ca052bb54ab0790a636c9b5f226502c73d547a25 # v5+ # Builds all the images in a single BuildKit session so that the stages+ # they share are only built once. Tags and caches are defined in the bake+ # file: pushes to main get the 'latest' tags, releases get the+ # release-specific ones.+ - name: Build images (${{ matrix.arch }})+ uses: docker/bake-action@3acf805d94d93a86cce4ca44798a76464a75b88c # v6.9.0+ env:+ ARCH: ${{ matrix.arch }}+ RELEASE: ${{ github.event_name == 'release' && github.event.release.name || '' }}+ SERVER_VERSION: ${{ github.event_name == 'release' && github.event.release.name || 'nightly' }}with:- context: .- build-args: SERVER_VERSION=${{ github.event_name == 'release' && github.event.release.name || 'nightly' }}- file: docker/Dockerfile- target: ${{ matrix.target }}- platforms: ${{ matrix.platform }}+ source: .+ files: docker/docker-bake.hclpush: true- tags: ${{ steps.tags.outputs.all }}- cache-from: type=registry,ref=ghcr.io/karakeep-app/karakeep-build-cache:${{ matrix.target }}-${{ matrix.suffix }}- cache-to: type=registry,mode=max,ref=ghcr.io/karakeep-app/karakeep-build-cache:${{ matrix.target }}-${{ matrix.suffix }}manifest:needs: builddiff --git a/docker/Dockerfile b/docker/Dockerfileindex 1d6ae273..2def9749 100644--- a/docker/Dockerfile+++ b/docker/Dockerfile@@ -55,23 +55,36 @@ RUN pnpm install --frozen-lockfileCOPY . .-# Build the db migration script+# Each app is built in its own stage so that images only build the apps they+# ship (e.g. the cli image doesn't compile the web app) and BuildKit can build+# them in parallel.++################# DB Migrations Builder ##############+FROM base AS db_migrations_builder+RUN cd packages/db && \pnpm exec ncc build migrate.ts -o /db_migrations && \cp -R drizzle /db_migrations+################# Web Builder ##############+FROM base AS web_builder-# Compile the web appRUN (cd apps/web && pnpm exec next build --experimental-build-mode compile)-# Build the worker code+################# Workers Builder ##############+FROM base AS workers_builder+RUN (cd apps/workers && pnpm --config.verify-deps-before-run=false build)RUN CI=true pnpm deploy --node-linker=isolated --legacy --filter @karakeep/workers --prod /prod/workers-# Build the cli+################# CLI Builder ##############+FROM base AS cli_builder+RUN (cd apps/cli && pnpm --config.verify-deps-before-run=false build)-# Build the mcp server+################# MCP Builder ##############+FROM base AS mcp_builder+RUN (cd apps/mcp && pnpm --config.verify-deps-before-run=false build)################# The All-in-one builder ##############@@ -154,21 +167,21 @@ ENV NODE_ENV productionENV NEXT_TELEMETRY_DISABLED=1ENV NEXTAUTH_URL_INTERNAL=http://localhost:${PORT}-COPY --from=base --chown=node:node /app/apps/web/.next/standalone ./-COPY --from=base /app/apps/web/public ./apps/web/public-COPY --from=base /db_migrations /db_migrations+COPY --from=web_builder --chown=node:node /app/apps/web/.next/standalone ./+COPY --from=web_builder /app/apps/web/public ./apps/web/public+COPY --from=db_migrations_builder /db_migrations /db_migrations# Set the correct permission for prerender cacheRUN mkdir -p ./apps/web/.next && chown node:node ./apps/web/.next# Automatically leverage output traces to reduce image size# https://nextjs.org/docs/advanced-features/output-file-tracing-COPY --from=base --chown=node:node /app/apps/web/.next/static ./apps/web/.next/static+COPY --from=web_builder --chown=node:node /app/apps/web/.next/static ./apps/web/.next/static####################### Prepare the workers app######################-COPY --from=base /prod/workers /app/apps/workers+COPY --from=workers_builder /prod/workers /app/apps/workersENTRYPOINT ["/init"]@@ -209,7 +222,7 @@ FROM node:24.18.1-slim AS cliLABEL org.opencontainers.image.source="https://github.com/karakeep-app/karakeep"WORKDIR /app-COPY --from=base /app/apps/cli/dist/index.mjs apps/cli/index.mjs+COPY --from=cli_builder /app/apps/cli/dist/index.mjs apps/cli/index.mjsWORKDIR /app/apps/cli@@ -225,7 +238,7 @@ FROM node:24.18.1-slim AS mcpLABEL org.opencontainers.image.source="https://github.com/karakeep-app/karakeep"WORKDIR /app-COPY --from=base /app/apps/mcp/dist/index.js apps/mcp/index.js+COPY --from=mcp_builder /app/apps/mcp/dist/index.js apps/mcp/index.jsWORKDIR /app/apps/mcpdiff --git a/docker/docker-bake.hcl b/docker/docker-bake.hclnew file mode 100644index 00000000..51b04eae--- /dev/null+++ b/docker/docker-bake.hcl@@ -0,0 +1,100 @@+# Builds all the images for a single architecture in one BuildKit session so+# that the stages they share (deps install, app builds, monolith, ...) are only+# built once.+#+# Usage: ARCH=amd64 docker buildx bake -f docker/docker-bake.hcl --push++# The architecture to build for (amd64 or arm64). Tags and caches are suffixed+# with it; the manifest job in CI stitches the architectures together.+variable "ARCH" {+ default = "amd64"+}++# The release name. When empty, the images are tagged as `latest`, otherwise+# they're tagged with the release name and `release`.+variable "RELEASE" {+ default = ""+}++variable "SERVER_VERSION" {+ default = "nightly"+}++variable "CACHE_REPO" {+ default = "ghcr.io/karakeep-app/karakeep-build-cache"+}++function "tags" {+ params = [repos]+ result = flatten([+ for repo in repos : RELEASE == ""+ ? ["${repo}:latest-${ARCH}"]+ : ["${repo}:${RELEASE}-${ARCH}", "${repo}:release-${ARCH}"]+ ])+}++# Each image exports its full (mode=max) build graph to its own cache ref.+# packages/e2e_tests/docker-compose.yml imports the aio-amd64 cache, so keep it+# in sync when renaming the refs.+function "cache_from" {+ params = [name]+ result = ["type=registry,ref=${CACHE_REPO}:${name}-${ARCH}"]+}++function "cache_to" {+ params = [name]+ result = ["type=registry,mode=max,ref=${CACHE_REPO}:${name}-${ARCH}"]+}++group "default" {+ targets = ["web", "workers", "cli", "mcp", "aio"]+}++target "_common" {+ context = "."+ dockerfile = "docker/Dockerfile"+ platforms = ["linux/${ARCH}"]+ args = {+ SERVER_VERSION = SERVER_VERSION+ }+}++target "web" {+ inherits = ["_common"]+ target = "web"+ tags = tags(["ghcr.io/hoarder-app/hoarder-web", "ghcr.io/karakeep-app/karakeep-web"])+ cache-from = cache_from("web")+ cache-to = cache_to("web")+}++target "workers" {+ inherits = ["_common"]+ target = "workers"+ tags = tags(["ghcr.io/hoarder-app/hoarder-workers", "ghcr.io/karakeep-app/karakeep-workers"])+ cache-from = cache_from("workers")+ cache-to = cache_to("workers")+}++target "cli" {+ inherits = ["_common"]+ target = "cli"+ tags = tags(["ghcr.io/hoarder-app/hoarder-cli", "ghcr.io/karakeep-app/karakeep-cli"])+ cache-from = cache_from("cli")+ cache-to = cache_to("cli")+}++target "mcp" {+ inherits = ["_common"]+ target = "mcp"+ tags = tags(["ghcr.io/karakeep-app/karakeep-mcp"])+ cache-from = cache_from("mcp")+ cache-to = cache_to("mcp")+}++target "aio" {+ inherits = ["_common"]+ target = "aio"+ tags = tags(["ghcr.io/hoarder-app/hoarder", "ghcr.io/karakeep-app/karakeep"])+ cache-from = cache_from("aio")+ cache-to = cache_to("aio")+}diff --git a/packages/e2e_tests/docker-compose.yml b/packages/e2e_tests/docker-compose.ymlindex dc95a462..f6757e39 100644--- a/packages/e2e_tests/docker-compose.yml+++ b/packages/e2e_tests/docker-compose.yml@@ -4,6 +4,13 @@ services:dockerfile: docker/Dockerfilecontext: ../../target: aio+ # Reuse the layers (deps install, monolith, ...) that the docker workflow+ # pushes from main. The checked-out source is still what gets built; only+ # unchanged layers are reused. The cache is public so that it's readable+ # without credentials (e.g. from fork PRs). If it's unavailable, buildkit+ # just warns and builds from scratch.+ cache_from:+ - type=registry,ref=ghcr.io/karakeep-app/karakeep-build-cache:aio-amd64restart: unless-stoppedports:- "${KARAKEEP_PORT:-3000}:3000"