Skip to content

OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.

ci: shorter pipeline with parallel checks, sharded tests and lean branch builds (#583)

Carl-Gerhard Lindesvärdf019500cdf49
diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml
new file mode 100644
index 000000000..19489eef4
--- /dev/null
+++ b/.github/actions/setup/action.yml
@@ -0,0 +1,37 @@
+name: Set up the workspace
+description: Node, bun, the bun install cache, dependencies and codegen — what every job that runs our code needs.
+
+runs:
+ using: composite
+ steps:
+ - name: Setup Node.js
+ uses: actions/setup-node@v7
+ with:
+ # .nvmrc is the source of truth; keep ARG NODE_VERSION in
+ # apps/*/Dockerfile in sync with it. Must stay >=22.19.0:
+ # undici@8 fails at import on older Node with
+ # "webidl.util.markAsUncloneable is not a function".
+ node-version-file: ".nvmrc"
+
+ - name: Install bun
+ uses: oven-sh/setup-bun@v2
+ with:
+ # .bun-version is the pin's home and apps/api/src/config/bun-version.test.ts asserts an exact match, so read it
+ # rather than keeping a second copy here.
+ bun-version-file: ".bun-version"
+
+ - name: Setup bun cache
+ uses: actions/cache@v6
+ with:
+ path: ~/.bun/install/cache
+ key: ${{ runner.os }}-bun-${{ hashFiles('**/bun.lock') }}
+ restore-keys: |
+ ${{ runner.os }}-bun-
+
+ - name: Install dependencies
+ shell: bash
+ run: bun install --frozen-lockfile
+
+ - name: Codegen
+ shell: bash
+ run: bun run codegen
diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml
index dd42504a3..9fcb35982 100644
--- a/.github/workflows/docker-build.yml
+++ b/.github/workflows/docker-build.yml
@@ -28,12 +28,79 @@ on:
- "**/.eslintignore"
- "**/.prettierignore"
+# Read-only by default; the jobs that push images or tags widen it themselves.
+permissions:
+ contents: read
+
+# A newer push to a branch supersedes the run still in flight. Main is left
+# out: its runs must overlap so `settle` below can debounce them.
+concurrency:
+ group: ${{ github.ref == 'refs/heads/main' && github.run_id || format('build-{0}', github.ref) }}
+ cancel-in-progress: true
+
env:
repo_owner: "openpanel-dev"
+# Shape of a run: `check`, `test` and `build` start together; `smoke` boots
+# what `build` produced. On a branch that is the whole run. On main, `merge`
+# tags the images once everything is green and `version` turns the run into a
+# release build.
jobs:
- lint-and-test:
+ check:
+ runs-on: ubuntu-latest
+ timeout-minutes: 15
+ steps:
+ - uses: actions/checkout@v7
+ with:
+ persist-credentials: false
+
+ - uses: ./.github/actions/setup
+
+ # Formatting and import order only. `bun run lint` (the full rule set) still reports errors that need a human
+ # decision, so it is not a gate yet; this half is green and catches drift today.
+ - name: Check formatting
+ run: bun run lint:format
+
+ - name: Run TypeScript checks
+ run: bun run typecheck
+
+ # dependency-cruiser keeps the dissolved validation/constants/common/json packages from growing back. Run standalone,
+ # not via `bun run check`, so a lint error cannot mask a dependency-boundary break.
+ - name: Check dependency boundaries
+ run: bun run check:deps
+
+ # The self-hosting CLI keeps its own lockfile (it ships as a binary, not
+ # as a workspace package), so it installs and tests on its own.
+ - name: Test the self-hosting CLI
+ working-directory: self-hosting/cli
+ run: |
+ bun install --frozen-lockfile
+ bun test
+
+ # packages/core is 200+ files run one process each (`--isolate`), which is
+ # most of the suite's time, so it is split across runners with `--shard`;
+ # every other workspace fits in one job. Each job has its own databases.
+ test:
+ name: test (${{ matrix.suite }})
runs-on: ubuntu-latest
+ timeout-minutes: 15
+ strategy:
+ fail-fast: false
+ matrix:
+ include:
+ - suite: core 1/3
+ directory: packages/core
+ command: bun run test --shard=1/3
+ - suite: core 2/3
+ directory: packages/core
+ command: bun run test --shard=2/3
+ - suite: core 3/3
+ directory: packages/core
+ command: bun run test --shard=3/3
+ - suite: everything else
+ directory: .
+ command: bun run test:rest
+ migrate: true
services:
postgres:
image: postgres:16-alpine
@@ -73,56 +140,18 @@ jobs:
--health-timeout 3s
--health-retries 20
steps:
- - uses: actions/checkout@v4
-
- - name: Setup Node.js
- uses: actions/setup-node@v4
- with:
- # .nvmrc is the source of truth; keep ARG NODE_VERSION in
- # apps/*/Dockerfile in sync with it. Must stay >=22.19.0:
- # undici@8 fails at import on older Node with
- # "webidl.util.markAsUncloneable is not a function".
- node-version-file: ".nvmrc"
-
- - name: Install bun
- uses: oven-sh/setup-bun@v2
+ - uses: actions/checkout@v7
with:
- # .bun-version is the pin's home and apps/api/src/config/bun-version.test.ts asserts an exact match, so read it
- # rather than keeping a second copy here.
- bun-version-file: ".bun-version"
-
- - name: Setup bun cache
- uses: actions/cache@v6
- with:
- path: ~/.bun/install/cache
- key: ${{ runner.os }}-bun-${{ hashFiles('**/bun.lock') }}
- restore-keys: |
- ${{ runner.os }}-bun-
-
- - name: Install dependencies
- run: bun install --frozen-lockfile
-
- - name: Codegen
- run: bun run codegen
+ persist-credentials: false
- # Formatting and import order only. `bun run lint` (the full rule set) still reports errors that need a human
- # decision, so it is not a gate yet; this half is green and catches drift today.
- - name: Check formatting
- run: bun run lint:format
-
- - name: Run TypeScript checks
- run: bun run typecheck
-
- # dependency-cruiser keeps the dissolved validation/constants/common/json packages from growing back. Run standalone,
- # not via `bun run check`, so a lint error cannot mask a dependency-boundary break.
- - name: Check dependency boundaries
- run: bun run check:deps
+ - uses: ./.github/actions/setup
# Exercises the real migration runners, which the suite does not: tests apply
# the flat test/clickhouse-schema.sql, so `migrate:deploy` is the only place
# Prisma's migrate and the ClickHouse code-migrations actually run. Uses its
# own databases, never the suite's `openpanel_test`.
- name: Migrate database
+ if: matrix.migrate
run: bun run migrate:deploy
env:
DATABASE_URL: postgresql://postgres:postgres@localhost:23432/postgres?schema=public
@@ -134,15 +163,8 @@ jobs:
# creates and migrates the isolated `openpanel_test` databases (test/bun-preload.ts, packages/core/test/preload.ts).
# Env set here would be silently overwritten.
- name: Run tests
- run: bun run test
-
- # The self-hosting CLI keeps its own lockfile (it ships as a binary, not
- # as a workspace package), so it installs and tests on its own.
- - name: Test the self-hosting CLI
- working-directory: self-hosting/cli
- run: |
- bun install --frozen-lockfile
- bun test
+ working-directory: ${{ matrix.directory }}
+ run: ${{ matrix.command }}
# Debounce for main: a push within a minute of the previous one cancels its
# wait, so a burst of merges is built and versioned once, as its newest
@@ -150,6 +172,7 @@ jobs:
settle:
if: github.ref == 'refs/heads/main'
runs-on: ubuntu-latest
+ timeout-minutes: 5
concurrency:
group: main-settle
cancel-in-progress: true
@@ -158,22 +181,30 @@ jobs:
if: github.event_name == 'push' && !contains(github.event.head_commit.message, '[release now]')
run: sleep 60
- build-api:
+ # Pushes by digest only; nothing gets a pullable tag until `merge`. That is
+ # why it can start alongside `check` and `test` instead of after them.
+ # Branches build amd64 alone, which is all `smoke` boots; main builds both
+ # platforms because its images are the ones that ship.
+ build:
permissions:
packages: write
contents: read
- needs: [lint-and-test, settle]
+ needs: [settle]
# settle is skipped on other branches; a cancelled settle (debounced) stops the run.
- if: ${{ !cancelled() && needs.lint-and-test.result == 'success' && contains(fromJSON('["success","skipped"]'), needs.settle.result) }}
+ if: ${{ !cancelled() && contains(fromJSON('["success","skipped"]'), needs.settle.result) }}
strategy:
fail-fast: false
matrix:
+ app: [api, dashboard]
+ platform: ${{ github.ref == 'refs/heads/main' && fromJSON('["linux/amd64", "linux/arm64"]') || fromJSON('["linux/amd64"]') }}
include:
- - platform: linux/amd64
- runner: ubuntu-latest
- - platform: linux/arm64
- runner: ubuntu-24.04-arm
- runs-on: ${{ matrix.runner }}
+ - app: api
+ dockerfile: apps/api/Dockerfile
+ build_args: DATABASE_URL=postgresql://dummy:dummy@localhost:5432/dummy
+ - app: dashboard
+ dockerfile: apps/start/Dockerfile
+ runs-on: ${{ matrix.platform == 'linux/arm64' && 'ubuntu-24.04-arm' || 'ubuntu-latest' }}
+ timeout-minutes: 30
steps:
- name: Prepare platform pair
run: |
@@ -181,30 +212,38 @@ jobs:
echo "PLATFORM_PAIR=${platform//\//-}" >> $GITHUB_ENV
- name: Checkout repository
- uses: actions/checkout@v4
+ uses: actions/checkout@v7
+ with:
+ persist-credentials: false
- name: Set up Docker Buildx
- uses: docker/setup-buildx-action@v3
+ uses: docker/setup-buildx-action@v4
- name: Log in to GitHub Container Registry
- uses: docker/login-action@v3
+ uses: docker/login-action@v4
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
+ # The layer cache lives in the registry, next to the image, not in the
+ # Actions cache: the install layers alone are ~1 GB per build, which
+ # kept the 10 GB Actions cache full and evicting, and exporting them
+ # there took longer than the build itself. zstd because that 1 GB is
+ # compressed on every export and decompressed on every import, and
+ # with gzip that took over a minute each way; the image itself keeps
+ # its default layers.
- name: Build and push by digest
id: build
- uses: docker/build-push-action@v6
+ uses: docker/build-push-action@v7
with:
context: .
- file: apps/api/Dockerfile
+ file: ${{ matrix.dockerfile }}
platforms: ${{ matrix.platform }}
- cache-from: type=gha,scope=api-${{ env.PLATFORM_PAIR }}
- cache-to: type=gha,mode=max,scope=api-${{ env.PLATFORM_PAIR }}
- outputs: type=image,name=ghcr.io/${{ env.repo_owner }}/api,push-by-digest=true,name-canonical=true,push=true
- build-args: |
- DATABASE_URL=postgresql://dummy:dummy@localhost:5432/dummy
+ cache-from: type=registry,ref=ghcr.io/${{ env.repo_owner }}/${{ matrix.app }}:buildcache-${{ env.PLATFORM_PAIR }}
+ cache-to: type=registry,ref=ghcr.io/${{ env.repo_owner }}/${{ matrix.app }}:buildcache-${{ env.PLATFORM_PAIR }},mode=max,image-manifest=true,oci-mediatypes=true,compression=zstd
+ outputs: type=image,name=ghcr.io/${{ env.repo_owner }}/${{ matrix.app }},push-by-digest=true,name-canonical=true,push=true
+ build-args: ${{ matrix.build_args }}
- name: Export digest
run: |
@@ -213,38 +252,41 @@ jobs:
touch "/tmp/digests/${digest#sha256:}"
- name: Upload digest
- uses: actions/upload-artifact@v4
+ uses: actions/upload-artifact@v7
with:
- name: digests-api-${{ env.PLATFORM_PAIR }}
+ name: digests-${{ matrix.app }}-${{ env.PLATFORM_PAIR }}
path: /tmp/digests/*
if-no-files-found: error
retention-days: 1
# Boots the images this run produced and asserts they actually serve traffic.
- # Gates every merge-* job below, so a broken image never gets a pullable tag.
+ # Gates `merge`, so a broken image never gets a pullable tag.
smoke:
permissions:
packages: read
contents: read
- needs: [build-api, build-dashboard]
+ needs: [build]
# Explicit, because a skipped settle earlier in the chain would otherwise skip this.
- if: ${{ !cancelled() && needs.build-api.result == 'success' && needs.build-dashboard.result == 'success' }}
+ if: ${{ !cancelled() && needs.build.result == 'success' }}
runs-on: ubuntu-latest
+ timeout-minutes: 15
steps:
- name: Checkout repository
- uses: actions/checkout@v4
+ uses: actions/checkout@v7
+ with:
+ persist-credentials: false
- name: Log in to GitHub Container Registry
- uses: docker/login-action@v3
+ uses: docker/login-action@v4
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- # build-* pushed by digest but has not tagged anything yet, so pull the
+ # build pushed by digest but has not tagged anything yet, so pull the
# exact amd64 digests this run produced.
- name: Download amd64 digests
- uses: actions/download-artifact@v4
+ uses: actions/download-artifact@v8
with:
path: /tmp/smoke-digests
pattern: digests-*-linux-amd64
@@ -271,126 +313,25 @@ jobs:
if: always()
run: docker compose -f .github/smoke/docker-compose.yml down -v || true
- merge-api:
- permissions:
- packages: write
- contents: read
- needs: [build-api, smoke]
- if: ${{ !cancelled() && needs.build-api.result == 'success' && needs.smoke.result == 'success' }}
- runs-on: ubuntu-latest
- steps:
- - name: Checkout repository
- uses: actions/checkout@v4
-
- - name: Download digests
- uses: actions/download-artifact@v4
- with:
- path: /tmp/digests
- pattern: digests-api-*
- merge-multiple: true
-
- - name: Generate tags
- id: tags
- run: |
- BRANCH_NAME=$(echo "${{ github.ref_name }}" | sed 's/\//-/g')
- SHORT_SHA=$(echo "${{ github.sha }}" | cut -c1-4)
- echo "branch_name=$BRANCH_NAME" >> $GITHUB_OUTPUT
- echo "short_sha=$SHORT_SHA" >> $GITHUB_OUTPUT
-
- - name: Set up Docker Buildx
- uses: docker/setup-buildx-action@v3
-
- - name: Log in to GitHub Container Registry
- uses: docker/login-action@v3
- with:
- registry: ghcr.io
- username: ${{ github.actor }}
- password: ${{ secrets.GITHUB_TOKEN }}
-
- - name: Create manifest list and push
- working-directory: /tmp/digests
- run: |
- docker buildx imagetools create \
- -t ghcr.io/${{ env.repo_owner }}/api:${{ steps.tags.outputs.branch_name }}-${{ steps.tags.outputs.short_sha }} \
- $(printf 'ghcr.io/${{ env.repo_owner }}/api@sha256:%s ' *)
-
- build-dashboard:
+ # Main only: a branch's images are smoke-tested and then left untagged.
+ merge:
permissions:
packages: write
contents: read
- needs: [lint-and-test, settle]
- if: ${{ !cancelled() && needs.lint-and-test.result == 'success' && contains(fromJSON('["success","skipped"]'), needs.settle.result) }}
+ needs: [check, test, build, smoke]
+ if: ${{ github.ref == 'refs/heads/main' && !cancelled() && needs.check.result == 'success' && needs.test.result == 'success' && needs.smoke.result == 'success' }}
strategy:
fail-fast: false
matrix:
- include:
- - platform: linux/amd64
- runner: ubuntu-latest
- - platform: linux/arm64
- runner: ubuntu-24.04-arm
- runs-on: ${{ matrix.runner }}
- steps:
- - name: Prepare platform pair
- run: |
- platform=${{ matrix.platform }}
- echo "PLATFORM_PAIR=${platform//\//-}" >> $GITHUB_ENV
-
- - name: Checkout repository
- uses: actions/checkout@v4
-
- - name: Set up Docker Buildx
- uses: docker/setup-buildx-action@v3
-
- - name: Log in to GitHub Container Registry
- uses: docker/login-action@v3
- with:
- registry: ghcr.io
- username: ${{ github.actor }}
- password: ${{ secrets.GITHUB_TOKEN }}
-
- - name: Build and push by digest
- id: build
- uses: docker/build-push-action@v6
- with:
- context: .
- file: apps/start/Dockerfile
- platforms: ${{ matrix.platform }}
- cache-from: type=gha,scope=dashboard-${{ env.PLATFORM_PAIR }}
- cache-to: type=gha,mode=max,scope=dashboard-${{ env.PLATFORM_PAIR }}
- outputs: type=image,name=ghcr.io/${{ env.repo_owner }}/dashboard,push-by-digest=true,name-canonical=true,push=true
- build-args: |
- NO_CLOUDFLARE=1
-
- - name: Export digest
- run: |
- mkdir -p /tmp/digests
- digest="${{ steps.build.outputs.digest }}"
- touch "/tmp/digests/${digest#sha256:}"
-
- - name: Upload digest
- uses: actions/upload-artifact@v4
- with:
- name: digests-dashboard-${{ env.PLATFORM_PAIR }}
- path: /tmp/digests/*
- if-no-files-found: error
- retention-days: 1
-
- merge-dashboard:
- permissions:
- packages: write
- contents: read
- needs: [build-dashboard, smoke]
- if: ${{ !cancelled() && needs.build-dashboard.result == 'success' && needs.smoke.result == 'success' }}
+ app: [api, dashboard]
runs-on: ubuntu-latest
+ timeout-minutes: 10
steps:
- - name: Checkout repository
- uses: actions/checkout@v4
-
- name: Download digests
- uses: actions/download-artifact@v4
+ uses: actions/download-artifact@v8
with:
path: /tmp/digests
- pattern: digests-dashboard-*
+ pattern: digests-${{ matrix.app }}-*
merge-multiple: true
- name: Generate tags
@@ -402,10 +343,10 @@ jobs:
echo "short_sha=$SHORT_SHA" >> $GITHUB_OUTPUT
- name: Set up Docker Buildx
- uses: docker/setup-buildx-action@v3
+ uses: docker/setup-buildx-action@v4
- name: Log in to GitHub Container Registry
- uses: docker/login-action@v3
+ uses: docker/login-action@v4
with:
registry: ghcr.io
username: ${{ github.actor }}
@@ -415,16 +356,17 @@ jobs:
working-directory: /tmp/digests
run: |
docker buildx imagetools create \
- -t ghcr.io/${{ env.repo_owner }}/dashboard:${{ steps.tags.outputs.branch_name }}-${{ steps.tags.outputs.short_sha }} \
- $(printf 'ghcr.io/${{ env.repo_owner }}/dashboard@sha256:%s ' *)
+ -t ghcr.io/${{ env.repo_owner }}/${{ matrix.app }}:${{ steps.tags.outputs.branch_name }}-${{ steps.tags.outputs.short_sha }} \
+ $(printf 'ghcr.io/${{ env.repo_owner }}/${{ matrix.app }}@sha256:%s ' *)
# Every build on main gets the next version: the git tag vX.Y.Z and the image
# tags X.Y.Z, X.Y and X, taken from this run's own digests. One run at a time,
# so two versions are never computed from the same last tag.
version:
- if: ${{ github.ref == 'refs/heads/main' && !cancelled() && needs.merge-api.result == 'success' && needs.merge-dashboard.result == 'success' }}
- needs: [merge-api, merge-dashboard]
+ if: ${{ github.ref == 'refs/heads/main' && !cancelled() && needs.merge.result == 'success' }}
+ needs: [merge]
runs-on: ubuntu-latest
+ timeout-minutes: 15
concurrency:
group: main-version
cancel-in-progress: false
@@ -432,7 +374,7 @@ jobs:
contents: write
packages: write
steps:
- - uses: actions/checkout@v4
+ - uses: actions/checkout@v7
with:
fetch-depth: 0
fetch-tags: true
@@ -449,15 +391,15 @@ jobs:
- name: Download digests
if: steps.next.outputs.skip != 'true'
- uses: actions/download-artifact@v4
+ uses: actions/download-artifact@v8
with:
path: /tmp/digests
pattern: digests-*
- - uses: docker/setup-buildx-action@v3
+ - uses: docker/setup-buildx-action@v4
if: steps.next.outputs.skip != 'true'
- - uses: docker/login-action@v3
+ - uses: docker/login-action@v4
if: steps.next.outputs.skip != 'true'
with:
registry: ghcr.io
diff --git a/.github/workflows/self-hosting-release.yml b/.github/workflows/self-hosting-release.yml
index 0778333ce..55d6c089f 100644
--- a/.github/workflows/self-hosting-release.yml
+++ b/.github/workflows/self-hosting-release.yml
@@ -34,7 +34,7 @@ jobs:
major: ${{ steps.version.outputs.major }}
previous: ${{ steps.version.outputs.previous }}
steps:
- - uses: actions/checkout@v4
+ - uses: actions/checkout@v7
with:
fetch-depth: 0
fetch-tags: true
@@ -83,15 +83,15 @@ jobs:
contents: read
packages: read
steps:
- - uses: docker/setup-buildx-action@v3
+ - uses: docker/setup-buildx-action@v4
- - uses: docker/login-action@v3
+ - uses: docker/login-action@v4
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- - uses: docker/login-action@v3
+ - uses: docker/login-action@v4
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
@@ -117,7 +117,7 @@ jobs:
permissions:
contents: read
steps:
- - uses: actions/checkout@v4
+ - uses: actions/checkout@v7
with:
fetch-depth: 0
fetch-tags: true
@@ -135,7 +135,7 @@ jobs:
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
run: bun tooling/release/src/release-notes.ts --from "${{ needs.resolve.outputs.previous }}" --to "v${{ needs.resolve.outputs.version }}" --out release-notes.md
- - uses: actions/upload-artifact@v4
+ - uses: actions/upload-artifact@v7
with:
name: release-notes
path: release-notes.md
@@ -153,9 +153,9 @@ jobs:
GH_TOKEN: ${{ github.token }}
VERSION: ${{ needs.resolve.outputs.version }}
steps:
- - uses: actions/checkout@v4
+ - uses: actions/checkout@v7
- - uses: actions/download-artifact@v4
+ - uses: actions/download-artifact@v8
with:
name: release-notes
diff --git a/apps/api/Dockerfile b/apps/api/Dockerfile
index 5acdb2966..ef9a84a1c 100644
--- a/apps/api/Dockerfile
+++ b/apps/api/Dockerfile
@@ -21,14 +21,26 @@ RUN apt-get update && \
&& apt-get clean && \
rm -rf /var/lib/apt/lists/*
+# MANIFESTS — only what `bun install` reads: every `package.json` (a workspace install has to see every member's),
+# the lockfile, `bunfig.toml` and the patches. Cut out of the whole tree with `find` rather than COPYed from a list,
+# because Docker's COPY cannot glob a directory level and a hand-maintained list failed SILENTLY when it drifted (a
+# missing manifest surfaced as a runtime "Cannot find module").
+#
+# The stage itself re-runs on every build, but its output is byte-identical until a manifest changes, so the install
+# layer in `toolchain` stays cached across source-only changes instead of re-downloading the workspace each build.
+FROM base AS manifests
+
+WORKDIR /app
+
+COPY . .
+
+RUN find . \( -type f -o -type l \) ! -name package.json ! -name bun.lock ! -name bunfig.toml ! -path './patches/*' -delete && \
+ find . -type d -empty -delete
+
# TOOLCHAIN — the full workspace, installed once WITH devDependencies, because codegen needs them (core's `tar` for
# scripts/download-geo.ts, db's `prisma` CLI for schema generation). Neither this stage nor `build` ships.
#
-# The whole tree is copied rather than a list of manifests: a workspace install has to see every member's
-# `package.json`, Docker's COPY cannot glob a directory level, and a hand-maintained list failed SILENTLY when it
-# drifted (a missing manifest surfaced as a runtime "Cannot find module").
-#
-# `bunfig.toml` is copied with the tree on purpose: it carries `[install] ignoreScripts = true` and
+# `bunfig.toml` rides along with the manifests on purpose: it carries `[install] ignoreScripts = true` and
# `linker = "isolated"`. Both install lines name `--linker=isolated` too, so the layout is a property of the command.
#
# No python3/make/g++: with `ignoreScripts` nothing compiles at install time, and the native addons this image needs
@@ -37,10 +49,12 @@ FROM base AS toolchain
WORKDIR /app
-COPY . .
+COPY --from=manifests /app ./
RUN bun install --frozen-lockfile --linker=isolated
+COPY . .
+
# BUILD — the Prisma client and core's geo/ASN data. There is no JS build step: the only output is generated
# *source* under packages/, which `prod-deps` inherits.
FROM toolchain AS build
diff --git a/apps/start/Dockerfile b/apps/start/Dockerfile
index f66c3a406..1d67f9985 100644
--- a/apps/start/Dockerfile
+++ b/apps/start/Dockerfile
@@ -9,9 +9,21 @@
ARG BUN_VERSION=1.4.0
ARG NODE_VERSION=24.19.0
+# MANIFESTS — only what `bun install` reads (every `package.json`, the lockfile, `bunfig.toml`, the patches), cut out
+# of the whole tree with `find` so the list cannot drift. Its output only changes when a manifest does, which keeps
+# the install layer below cached across source-only changes. Same stage as apps/api/Dockerfile's.
+FROM oven/bun:${BUN_VERSION}-slim AS manifests
+
+WORKDIR /app
+
+COPY . .
+
+RUN find . \( -type f -o -type l \) ! -name package.json ! -name bun.lock ! -name bunfig.toml ! -path './patches/*' -delete && \
+ find . -type d -empty -delete
+
# INSTALL — one full workspace install, exactly what bun.lock was resolved against. `bunfig.toml` rides along with the
-# tree and carries `[install] ignoreScripts = true` and `linker = "isolated"`; `--linker=isolated` is named here too so
-# the layout is a property of the command, not only of a file.
+# manifests and carries `[install] ignoreScripts = true` and `linker = "isolated"`; `--linker=isolated` is named here
+# too so the layout is a property of the command, not only of a file.
#
# No apt block: the runner starts `FROM node:${NODE_VERSION}-slim` and copies only `.output`, so an apt block here
# shipped nothing. `bun install` needs TLS and `oven/bun:${BUN_VERSION}-slim` already ships `ca-certificates`.
@@ -19,14 +31,16 @@ FROM oven/bun:${BUN_VERSION}-slim AS install
WORKDIR /app
+COPY --from=manifests /app ./
+
+RUN bun install --frozen-lockfile --linker=isolated
+
COPY . .
# The tracking script the self-hosted dashboard serves.
COPY apps/public/public/op1.js ./apps/start/public/op1.js
COPY apps/public/public/op1-replay.js ./apps/start/public/op1-replay.js
-RUN bun install --frozen-lockfile --linker=isolated
-
# BUILD — Nitro's standalone server output. NITRO=1 selects the node-server
# preset over the Cloudflare one; SELF_HOSTED=1 is read at build time.
#
diff --git a/package.json b/package.json
index 536431719..7363691b3 100644
--- a/package.json
+++ b/package.json
@@ -56,7 +56,9 @@
"vue": "3.5.25"
},
"scripts": {
- "test": "bun run --filter @openpanel/core test && bun run --filter @openpanel/db test && bun run --filter @openpanel/api test && bun run --filter @openpanel/seed test && bun run --filter start test && bun run --filter @openpanel/release test && vitest run && bun test tooling/gates/conformance",
+ "test": "bun run test:core && bun run test:rest",
+ "test:core": "bun run --filter @openpanel/core test",
+ "test:rest": "bun run --filter @openpanel/db test && bun run --filter @openpanel/api test && bun run --filter @openpanel/seed test && bun run --filter start test && bun run --filter @openpanel/release test && vitest run && bun test tooling/gates/conformance",
"gen:bots": "bun run --filter @openpanel/core gen:bots",
"gen:referrers": "bun run --filter @openpanel/core gen:referrers",
"dock:up": "docker compose up -d",
diff --git a/packages/core/src/buffers/clickhouse-failover.test.ts b/packages/core/src/buffers/clickhouse-failover.test.ts
index c9233f0b0..e24c455f9 100644
--- a/packages/core/src/buffers/clickhouse-failover.test.ts
+++ b/packages/core/src/buffers/clickhouse-failover.test.ts
@@ -33,11 +33,11 @@ function liveNodeUrl(): { base: string; withDatabase: string } {
const { base: LIVE_BASE_URL, withDatabase: LIVE_NODE_URL } = liveNodeUrl();
-async function bootstrapTestDatabase(): Promise<boolean> {
+async function liveNodeReachable(): Promise<boolean> {
try {
const response = await fetch(LIVE_BASE_URL, {
method: 'POST',
- body: `CREATE DATABASE IF NOT EXISTS ${CLICKHOUSE_TEST_DATABASE}`,
+ body: 'SELECT 1',
});
return response.ok;
} catch {
@@ -56,9 +56,6 @@ async function deadNodeIsRefused(): Promise<boolean> {
}
}
-const clickhouseReachable = await bootstrapTestDatabase();
-const failoverIsReal = await deadNodeIsRefused();
-
// `bun test` shares one process and one `process.env` across every file in
// the run, unlike vitest's per-file env reset. Whichever file first calls
// `loadClickHouse()` pins `@openpanel/db/src/clickhouse/client`'s picker for
@@ -93,6 +90,21 @@ process.env.CLICKHOUSE_REQUEST_TIMEOUT_MS = String(
// cases never exercise the retry at all.
process.env.CLICKHOUSE_UNHEALTHY_MARK_MS = '0';
+const clickhouseReachable = await liveNodeReachable();
+const failoverIsReal = await deadNodeIsRefused();
+
+// This file may be the first in its shard to touch ClickHouse, and
+// `ch.insert` below needs the schema. Only reachability decides the skip
+// above; a broken bootstrap throws here instead of silently skipping. Runs
+// after the env is pinned because importing the bootstrap evaluates the
+// client module, and that evaluation is what reads the two-node list.
+if (clickhouseReachable) {
+ const { bootstrapTestDatabases } = await import(
+ '../../../../test/bootstrap-databases'
+ );
+ await bootstrapTestDatabases();
+}
+
afterAll(() => {
restoreClickhouseEnv();
});
diff --git a/packages/core/src/modules/chart/funnel.step-breakdown.test.ts b/packages/core/src/modules/chart/funnel.step-breakdown.test.ts
index da55536f8..ccd3f3b2c 100644
--- a/packages/core/src/modules/chart/funnel.step-breakdown.test.ts
+++ b/packages/core/src/modules/chart/funnel.step-breakdown.test.ts
@@ -1,5 +1,6 @@
import { afterAll, beforeAll, describe, expect, it } from 'bun:test';
import { createClient } from '@openpanel/db/src/clickhouse/client';
+import { bootstrapTestDatabases } from '../../../../../test/bootstrap-databases';
import { TEST_CLICKHOUSE_URL } from '../../../../../test/databases';
import { buildEvent } from '../../../../../test/fixtures';
import { testServiceDeps } from '../../../test/service-deps';
@@ -111,6 +112,8 @@ async function withClient(
describe('funnel breakdown by a property of one step', () => {
beforeAll(async () => {
+ // The file may be the first to touch ClickHouse in its shard.
+ await bootstrapTestDatabases();
deps = await testServiceDeps();
await withClient(async (client) => {
await client.command({
@@ -122,7 +125,7 @@ describe('funnel breakdown by a property of one step', () => {
format: 'JSONEachRow',
});
});
- });
+ }, 30_000);
afterAll(async () => {
await withClient((client) =>