The open alternative to Salesforce, designed for AI.
TypeScript crmcrm-systemcustomergood-first-issuegraphqlhacktoberfestjavascriptmarketingmonoreponestjsopen-sourcepostgresqlreactreactjssalestypescriptweb
feat(docker): add ClickHouse to twenty-app-dev image (#26598)
martmull3accdee43fb4
diff --git a/.github/workflows/ci-test-docker-compose.yaml b/.github/workflows/ci-test-docker-compose.yamlindex 3ab57529016..5d5e4a1fb0e 100644--- a/.github/workflows/ci-test-docker-compose.yaml+++ b/.github/workflows/ci-test-docker-compose.yaml@@ -166,6 +166,12 @@ jobs:grep -qi "^content-security-policy: frame-ancestors 'self';" /tmp/twenty-headersgrep -q 'id="twenty-client-config" type="application/json"' /tmp/twenty-documentdone+ - name: Verify ClickHouse migrations+ run: |+ expected=$(ls packages/twenty-server/src/database/clickhouse/migrations/*.sql | wc -l)+ applied=$(docker exec twenty-app-dev clickhouse client --query "SELECT count() FROM twenty._migration")+ echo "Applied ${applied}/${expected} ClickHouse migrations"+ test "$applied" -eq "$expected"ci-test-docker-status-check:if: always() && !cancelled()timeout-minutes: 5diff --git a/packages/twenty-docker/twenty-app-dev/rootfs/etc/clickhouse-server/config.xml b/packages/twenty-docker/twenty-app-dev/rootfs/etc/clickhouse-server/config.xmlnew file mode 100644index 00000000000..7f86b5d114d--- /dev/null+++ b/packages/twenty-docker/twenty-app-dev/rootfs/etc/clickhouse-server/config.xml@@ -0,0 +1,32 @@+<clickhouse>+ <logger>+ <level>warning</level>+ <console>1</console>+ </logger>++ <listen_host>127.0.0.1</listen_host>+ <http_port>8123</http_port>+ <tcp_port>9000</tcp_port>++ <path>/data/clickhouse/</path>+ <timezone>UTC</timezone>++ <users>+ <default>+ <password></password>+ <networks>+ <ip>127.0.0.1</ip>+ </networks>+ <profile>default</profile>+ <quota>default</quota>+ </default>+ </users>++ <profiles>+ <default/>+ </profiles>++ <quotas>+ <default/>+ </quotas>+</clickhouse>diff --git a/packages/twenty-docker/twenty-app-dev/rootfs/etc/s6-overlay/s6-rc.d/clickhouse/run b/packages/twenty-docker/twenty-app-dev/rootfs/etc/s6-overlay/s6-rc.d/clickhouse/runnew file mode 100755index 00000000000..44d93929902--- /dev/null+++ b/packages/twenty-docker/twenty-app-dev/rootfs/etc/s6-overlay/s6-rc.d/clickhouse/run@@ -0,0 +1,4 @@+#!/bin/sh+# s6 supervises the server, no need for ClickHouse's own watchdog process+export CLICKHOUSE_WATCHDOG_ENABLE=0+exec su-exec clickhouse clickhouse server --config-file=/etc/clickhouse-server/config.xmldiff --git a/packages/twenty-docker/twenty-app-dev/rootfs/etc/s6-overlay/s6-rc.d/clickhouse/type b/packages/twenty-docker/twenty-app-dev/rootfs/etc/s6-overlay/s6-rc.d/clickhouse/typenew file mode 100644index 00000000000..5883cff0cd1--- /dev/null+++ b/packages/twenty-docker/twenty-app-dev/rootfs/etc/s6-overlay/s6-rc.d/clickhouse/type@@ -0,0 +1 @@+longrundiff --git a/packages/twenty-docker/twenty-app-dev/rootfs/etc/s6-overlay/s6-rc.d/init-db/dependencies.d/clickhouse b/packages/twenty-docker/twenty-app-dev/rootfs/etc/s6-overlay/s6-rc.d/init-db/dependencies.d/clickhousenew file mode 100644index 00000000000..e69de29bb2ddiff --git a/packages/twenty-docker/twenty-app-dev/rootfs/etc/s6-overlay/s6-rc.d/user/contents.d/clickhouse b/packages/twenty-docker/twenty-app-dev/rootfs/etc/s6-overlay/s6-rc.d/user/contents.d/clickhousenew file mode 100644index 00000000000..e69de29bb2ddiff --git a/packages/twenty-docker/twenty-app-dev/rootfs/etc/s6-overlay/scripts/init-db.sh b/packages/twenty-docker/twenty-app-dev/rootfs/etc/s6-overlay/scripts/init-db.shindex 39339ed53ea..f11e3fd4f14 100755--- a/packages/twenty-docker/twenty-app-dev/rootfs/etc/s6-overlay/scripts/init-db.sh+++ b/packages/twenty-docker/twenty-app-dev/rootfs/etc/s6-overlay/scripts/init-db.sh@@ -30,6 +30,17 @@ su-exec postgres psql -h localhost -tc \# Run Twenty database setup and migrationscd /app/packages/twenty-server+step_start "Running ClickHouse migrations"+TRIES=0+until curl -sf http://localhost:8123/ping > /dev/null 2>&1 || [ "$TRIES" -ge 120 ]; do+ TRIES=$((TRIES + 1))+ sleep 0.5+done+if ! yarn clickhouse:migrate:prod; then+ echo "Warning: ClickHouse migrations failed, but continuing startup..."+fi+step_done+has_schema=$(PGPASSWORD=twenty psql -h localhost -U twenty -d default -tAc \"SELECT EXISTS (SELECT 1 FROM information_schema.schemata WHERE schema_name = 'core')")diff --git a/packages/twenty-docker/twenty/Dockerfile b/packages/twenty-docker/twenty/Dockerfileindex 34e752ab84e..ce33ca0a8c0 100644--- a/packages/twenty-docker/twenty/Dockerfile+++ b/packages/twenty-docker/twenty/Dockerfile@@ -202,7 +202,7 @@ USER 1000# ===========================================================================-# Target: twenty-app-dev (all-in-one with Postgres + Redis)+# Target: twenty-app-dev (all-in-one with Postgres + Redis + ClickHouse)# docker build --target twenty-app-dev -f packages/twenty-docker/twenty/Dockerfile .# ===========================================================================@@ -226,6 +226,8 @@ RUN S6_ARCH=$(cat /tmp/s6arch) && \echo "$NOARCH_SUM s6-overlay-noarch.tar.xz" | sha256sum -c - && \echo "$ARCH_SUM s6-overlay-arch.tar.xz" | sha256sum -c -+FROM clickhouse/clickhouse-server:25.8.8-alpine@sha256:68b9d9eb4cb949a03d1f90b76fd8d396a5999dd57ba1ee914ea36c77716ce937 AS clickhouse+FROM node:24.19.0-alpine3.23@sha256:244cc2b53f46f9e876304391d17682b0ddae9ac33491f4857e25e35a36ba7995 AS twenty-app-dev# s6-overlay@@ -245,6 +247,21 @@ RUN apk add --no-cache \'libexpat>=2.8.5-r0' \'curl>=8.22.0-r0' 'jq>=1.8.2-r0' su-exec+# ClickHouse binaries are linked against glibc: reuse the glibc runtime its+# official alpine image ships alongside musl, and expose the loader at the path+# glibc binaries expect.+ARG TARGETARCH+COPY --from=clickhouse --chown=root:root /usr/bin/clickhouse /usr/bin/clickhouse+COPY --from=clickhouse \+ /lib/ld-2.35.so /lib/libc.so.6 /lib/libdl.so.2 /lib/libm.so.6 \+ /lib/libpthread.so.0 /lib/librt.so.1 /lib/libresolv.so.2 \+ /lib/libnss_dns.so.2 /lib/libnss_files.so.2 \+ /lib/+RUN if [ "$TARGETARCH" = "arm64" ]; then ln -s /lib/ld-2.35.so /lib/ld-linux-aarch64.so.1; \+ else mkdir -p /lib64 && ln -s /lib/ld-2.35.so /lib64/ld-linux-x86-64.so.2; fi \+ && addgroup -S clickhouse \+ && adduser -S -D -H -h /data/clickhouse -G clickhouse clickhouse+# Workspace root configCOPY --from=twenty-server-build /app/package.json /app/yarn.lock /app/.yarnrc.yml /app/COPY --from=twenty-server-build /app/tsconfig.base.json /app/nx.json /app/@@ -275,8 +292,9 @@ RUN find /app/packages/twenty-server/dist -name '*.js.map' -delete \# s6 service definitionsCOPY packages/twenty-docker/twenty-app-dev/rootfs/ /-RUN mkdir -p /data/postgres /data/redis /app/packages/twenty-server/.local-storage \+RUN mkdir -p /data/postgres /data/redis /data/clickhouse /app/packages/twenty-server/.local-storage \&& chown -R postgres:postgres /data/postgres \+ && chown clickhouse:clickhouse /data/clickhouse \&& chown 1000:1000 /data/redis /app/packages/twenty-server/.local-storageARG APP_VERSION=0.0.0@@ -285,6 +303,7 @@ ENV S6_KEEP_ENV=1ENV PG_DATABASE_URL=postgres://twenty:twenty@localhost:5432/default \SERVER_URL=http://localhost:2020 \REDIS_URL=redis://localhost:6379 \+ CLICKHOUSE_URL=http://localhost:8123/twenty \STORAGE_TYPE=local \APP_SECRET=twenty-app-dev-secret-not-for-production \APP_VERSION=$APP_VERSION \@@ -297,9 +316,9 @@ ENV PG_DATABASE_URL=postgres://twenty:twenty@localhost:5432/default \APPLICATION_LOG_DRIVER=CONSOLEEXPOSE 2020-VOLUME ["/data/postgres", "/app/packages/twenty-server/.local-storage"]+VOLUME ["/data/postgres", "/data/clickhouse", "/app/packages/twenty-server/.local-storage"]LABEL org.opencontainers.image.source=https://github.com/twentyhq/twenty-LABEL org.opencontainers.image.description="All-in-one Twenty image for local development and SDK usage. Includes PostgreSQL, Redis, server, and worker."+LABEL org.opencontainers.image.description="All-in-one Twenty image for local development and SDK usage. Includes PostgreSQL, Redis, ClickHouse, server, and worker."ENTRYPOINT ["/init"]diff --git a/packages/twenty-docs/developers/extend/apps/getting-started/local-server.mdx b/packages/twenty-docs/developers/extend/apps/getting-started/local-server.mdxindex 1c9f8f26727..257950b6a53 100644--- a/packages/twenty-docs/developers/extend/apps/getting-started/local-server.mdx+++ b/packages/twenty-docs/developers/extend/apps/getting-started/local-server.mdx@@ -20,7 +20,7 @@ Use `yarn twenty docker:*` to control the local Twenty container:| `yarn twenty docker:upgrade` | Pull the latest `twenty-app-dev` image || `yarn twenty docker:upgrade 2.2.0` | Upgrade to a specific version |-Data persists across restarts in two Docker volumes (`twenty-app-dev-data` for PostgreSQL, `twenty-app-dev-storage` for files). Use `reset` to wipe everything.+Data persists across restarts in three Docker volumes (`twenty-app-dev-data` for PostgreSQL, `twenty-app-dev-clickhouse` for ClickHouse logs and analytics, `twenty-app-dev-storage` for files). Use `reset` to wipe everything.## Pinning the server version@@ -60,7 +60,7 @@ Pass `--test` to any `docker:*` command to manage a second, fully isolated insta| `yarn twenty docker:reset --test` | Wipe its data || `yarn twenty docker:upgrade --test` | Upgrade its image |-The test instance has its own container (`twenty-app-dev-test`), volumes (`twenty-app-dev-test-data`, `twenty-app-dev-test-storage`), and config — it runs alongside your main instance without conflicts. Combine `--test` with `--port` to override 2021.+The test instance has its own container (`twenty-app-dev-test`), volumes (`twenty-app-dev-test-data`, `twenty-app-dev-test-clickhouse`, `twenty-app-dev-test-storage`), and config — it runs alongside your main instance without conflicts. Combine `--test` with `--port` to override 2021.## Manual setup (without the scaffolder)diff --git a/packages/twenty-sdk/src/cli/commands/docker/index.ts b/packages/twenty-sdk/src/cli/commands/docker/index.tsindex 4b00d7bcc31..c8be4610ba0 100644--- a/packages/twenty-sdk/src/cli/commands/docker/index.ts+++ b/packages/twenty-sdk/src/cli/commands/docker/index.ts@@ -120,15 +120,21 @@ const resetAction = (options: { test?: boolean }) => {const volumeStorage = options.test? 'twenty-app-dev-test-storage': 'twenty-app-dev-storage';+ const volumeClickHouse = options.test+ ? 'twenty-app-dev-test-clickhouse'+ : 'twenty-app-dev-clickhouse';if (containerExists(containerName)) {execSync(`docker rm -f ${containerName}`, { stdio: 'ignore' });}try {- execSync(`docker volume rm ${volumeData} ${volumeStorage}`, {- stdio: 'ignore',- });+ execSync(+ `docker volume rm ${volumeData} ${volumeStorage} ${volumeClickHouse}`,+ {+ stdio: 'ignore',+ },+ );} catch {// Volumes may not exist}diff --git a/packages/twenty-sdk/src/cli/operations/server-start.ts b/packages/twenty-sdk/src/cli/operations/server-start.tsindex ec23af60386..b757fb8ceb4 100644--- a/packages/twenty-sdk/src/cli/operations/server-start.ts+++ b/packages/twenty-sdk/src/cli/operations/server-start.ts@@ -135,6 +135,9 @@ const innerServerStart = async (const volumeStorage = isTest? 'twenty-app-dev-test-storage': 'twenty-app-dev-storage';+ const volumeClickHouse = isTest+ ? 'twenty-app-dev-test-clickhouse'+ : 'twenty-app-dev-clickhouse';if (checkDockerRunning() && containerExists(containerName)) {const currentImage = getContainerImageTag(containerName);@@ -262,6 +265,8 @@ const innerServerStart = async ('-v',`${volumeData}:/data/postgres`,'-v',+ `${volumeClickHouse}:/data/clickhouse`,+ '-v',`${volumeStorage}:/app/packages/twenty-server/.local-storage`,image,],diff --git a/packages/twenty-sdk/src/cli/operations/server-upgrade.ts b/packages/twenty-sdk/src/cli/operations/server-upgrade.tsindex 32cc8ead12f..7e1e370d707 100644--- a/packages/twenty-sdk/src/cli/operations/server-upgrade.ts+++ b/packages/twenty-sdk/src/cli/operations/server-upgrade.ts@@ -103,6 +103,9 @@ const innerServerUpgrade = async (const volumeStorage = isTest? 'twenty-app-dev-test-storage': 'twenty-app-dev-storage';+ const volumeClickHouse = isTest+ ? 'twenty-app-dev-test-clickhouse'+ : 'twenty-app-dev-clickhouse';onProgress?.('Starting container with new image...');@@ -122,6 +125,8 @@ const innerServerUpgrade = async ('-v',`${volumeData}:/data/postgres`,'-v',+ `${volumeClickHouse}:/data/clickhouse`,+ '-v',`${volumeStorage}:/app/packages/twenty-server/.local-storage`,image,],