OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.
ralph(P1-005): Self-host ships Redpanda: compose template, setup script, real env names
ralph7645e0bea4d5
diff --git a/.env.example b/.env.exampleindex 639f2d023..69dde0b0d 100644--- a/.env.example+++ b/.env.example@@ -28,18 +28,24 @@ NEXT_PUBLIC_API_URL="http://localhost:3333"WORKER_PORT=9999API_PORT=3333-# REDPANDA (optional, runs in parallel with the groupmq events queue)-# Empty REDPANDA_BROKERS disables both producer and consumer; everything-# falls back to groupmq. Set REDPANDA_PROJECT_IDS=* to route all projects,-# or a comma-separated allow-list to route specific ones.-# REDPANDA_BROKERS=""-# REDPANDA_EVENTS_TOPIC="events"-# REDPANDA_CONSUMER_GROUP="openpanel-events"-# REDPANDA_PROJECT_IDS=""-# REDPANDA_PARTITIONS_CONCURRENT="8"+# KAFKA / REDPANDA (optional, runs in parallel with the groupmq events queue)+# Empty KAFKA_BROKERS disables both producer and consumer; everything falls+# back to groupmq. Enable ENABLED_QUEUES=events_kafka on the worker(s) that+# should consume from Kafka.+# KAFKA_BROKERS=""+# KAFKA_EVENTS_TOPIC="events"+# KAFKA_CONSUMER_GROUP="openpanel-events"+# KAFKA_PARTITIONS_CONCURRENT="8"+# Number of partitions Redpanda/Kafka creates the `events` topic with; cloud+# runs 24. FOOTGUN: the producer partitions by murmur2(key) % numPartitions,+# so raising this once the topic already exists reshuffles every key and+# silently breaks per-key ordering and mutual exclusion for events produced+# before the change. Set it once at install; to raise it later, drain+# ingestion (stop producers, let consumers fully catch up) and recreate the+# topic — see self-hosting/redpanda/bootstrap.template.yaml.+# KAFKA_EVENTS_TOPIC_PARTITIONS="24"# Local dev with docker-compose:-# REDPANDA_BROKERS="localhost:19092"-# REDPANDA_PROJECT_IDS="*"+# KAFKA_BROKERS="localhost:19092"# EMAIL - set one or both; SMTP takes priority over Resend if SMTP_HOST is set# Option A - Resenddiff --git a/.github/smoke/docker-compose.yml b/.github/smoke/docker-compose.ymlindex 6151cd8c3..b3e51b5a2 100644--- a/.github/smoke/docker-compose.yml+++ b/.github/smoke/docker-compose.yml@@ -52,6 +52,31 @@ services:timeout: 5sretries: 24+ # Mirrors self-hosting/docker-compose.template.yml's op-rp: same image, same+ # single-node sizing (not the root docker-compose.yml's 1G dev cap). Bootstrap+ # config is the checked-in dev file (partitions=24, same as cloud) rather than+ # the self-host setup script's generated one, which doesn't exist until+ # ./setup has run — keeps this stack reproducible from this file alone.+ op-rp:+ image: redpandadata/redpanda:v25.1.4+ command:+ - redpanda+ - start+ - --kafka-addr=internal://0.0.0.0:9092+ - --advertise-kafka-addr=internal://op-rp:9092+ - --rpc-addr=0.0.0.0:33145+ - --advertise-rpc-addr=op-rp:33145+ - --smp=1+ - --memory=2G+ - --default-log-level=info+ volumes:+ - ../../docker/redpanda/bootstrap.yaml:/etc/redpanda/.bootstrap.yaml:ro+ healthcheck:+ test: ["CMD", "rpk", "cluster", "health", "--exit-when-healthy"]+ interval: 5s+ timeout: 5s+ retries: 24+op-api:image: ${OP_API_IMAGE:-openpanel-smoke-image-unset}command: >@@ -79,6 +104,8 @@ services:COOKIE_SECRET: smoke-test-cookie-secret-not-a-real-secretEMAIL_SENDER: [email protected]RESEND_API_KEY: re_smoke_test_key+ KAFKA_BROKERS: op-rp:9092+ KAFKA_EVENTS_TOPIC_PARTITIONS: "24"healthcheck:test: ["CMD-SHELL", "curl -f http://localhost:3000/healthcheck || exit 1"]interval: 5s@@ -91,6 +118,8 @@ services:condition: service_healthyop-kv:condition: service_healthy+ op-rp:+ condition: service_healthyop-dashboard:image: ${OP_DASHBOARD_IMAGE:-openpanel-smoke-image-unset}@@ -125,3 +154,5 @@ services:depends_on:op-api:condition: service_healthy+ op-rp:+ condition: service_healthydiff --git a/.github/smoke/smoke.sh b/.github/smoke/smoke.shindex fe950f02c..7b747329a 100755--- a/.github/smoke/smoke.sh+++ b/.github/smoke/smoke.sh@@ -28,7 +28,7 @@ fail() {echo "::error::$*"echo "--- container status ---"docker compose ps || true- for svc in op-api op-worker op-dashboard; do+ for svc in op-api op-worker op-dashboard op-rp; doecho "--- $svc logs ---"docker compose logs --no-color --tail=200 "$svc" || truedonediff --git a/apps/public/content/docs/self-hosting/deploy-docker-compose.mdx b/apps/public/content/docs/self-hosting/deploy-docker-compose.mdxindex 1d3f6c1f5..c515022e1 100644--- a/apps/public/content/docs/self-hosting/deploy-docker-compose.mdx+++ b/apps/public/content/docs/self-hosting/deploy-docker-compose.mdx@@ -11,7 +11,7 @@ This guide will help you deploy OpenPanel using Docker Compose. This method give## Prerequisites- A VPS or server (Docker and Node will be installed automatically if needed)-- At least 2GB RAM (4GB+ recommended)+- At least 4GB RAM (6GB+ recommended) — the stack now includes Redpanda alongside Postgres, Redis and ClickHouse- Domain name pointing to your server (optional but recommended)- Basic knowledge of command line@@ -78,6 +78,7 @@ This will start all required services:- **op-db**: PostgreSQL database- **op-kv**: Redis cache- **op-ch**: ClickHouse analytics database+- **op-rp**: Redpanda (Kafka-compatible event broker, see [Event Streaming](/docs/self-hosting/environment-variables#event-streaming-kafka))- **op-api**: OpenPanel API server- **op-dashboard**: OpenPanel dashboard (frontend)- **op-worker**: Background worker for processing eventsdiff --git a/apps/public/content/docs/self-hosting/environment-variables.mdx b/apps/public/content/docs/self-hosting/environment-variables.mdxindex bb4442ebf..5c8b2b4c5 100644--- a/apps/public/content/docs/self-hosting/environment-variables.mdx+++ b/apps/public/content/docs/self-hosting/environment-variables.mdx@@ -743,7 +743,7 @@ OpenPanel can optionally ingest events through a Kafka-compatible broker (KafkaRouting is all-or-nothing: if `KAFKA_BROKERS` is set, every event is produced to the Kafka topic; otherwise all events stay on the default Redis-backed `events` queue. Set `ENABLED_QUEUES=events_kafka` on the worker(s) that should consume from Kafka.<Callout>-The client speaks the Kafka wire protocol via [kafkajs](https://kafka.js.org/), so any Kafka-compatible broker works. The bundled `docker-compose.yml` uses [Redpanda](https://redpanda.com/) as a lightweight, single-binary option, but you can point `KAFKA_BROKERS` at any Kafka cluster.+The client speaks the Kafka wire protocol via [kafkajs](https://kafka.js.org/), so any Kafka-compatible broker works. `self-hosting/docker-compose.template.yml` ships an `op-rp` service running [Redpanda](https://redpanda.com/) — a lightweight, single-binary broker — by default, with `KAFKA_BROKERS` already pointed at it. You can instead point `KAFKA_BROKERS` at any Kafka cluster of your own.</Callout>### KAFKA_BROKERS@@ -774,6 +774,23 @@ Name of the Kafka topic used for incoming events.KAFKA_EVENTS_TOPIC=openpanel-events```+### KAFKA_EVENTS_TOPIC_PARTITIONS++**Type**: `number`+**Required**: No+**Default**: `24`++Number of partitions the bundled Redpanda service creates the `KAFKA_EVENTS_TOPIC` topic with on first boot (matches the partition count OpenPanel Cloud runs). Only takes effect on an empty data volume — it does not repartition a topic that already exists.++**Example**:+```bash+KAFKA_EVENTS_TOPIC_PARTITIONS=48+```++<Callout type="warn">+Set this once, before the first `./start`. The Kafka/Redpanda partitioner is `murmur2(key) % numPartitions`, so raising the partition count after the topic already has data reshuffles where every key lands — it silently breaks per-key event ordering and the mutual exclusion the session buffer relies on for events produced before the change. To raise it later: stop producers, let consumers fully drain the backlog, delete the `events` topic, then restart so it's recreated with the new count.+</Callout>+### KAFKA_CONSUMER_GROUP**Type**: `string`diff --git a/self-hosting/.env.template b/self-hosting/.env.templateindex 22044261f..615eab83c 100644--- a/self-hosting/.env.template+++ b/self-hosting/.env.template@@ -14,3 +14,11 @@ API_URL="$API_URL"COOKIE_SECRET="$COOKIE_SECRET"EMAIL_SENDER="$EMAIL_SENDER"RESEND_API_KEY="$RESEND_API_KEY"++# Events transport (ADR-004). The op-rp service (Redpanda) is a required part+# of this stack, not optional infra — see docker-compose.template.yml.+KAFKA_BROKERS="op-rp:9092"+# Partition count for the `events` topic, applied once by redpanda/bootstrap.yaml+# on first boot. Set once at install — see the footgun warning in+# redpanda/bootstrap.template.yaml before changing it on a running cluster.+KAFKA_EVENTS_TOPIC_PARTITIONS="$KAFKA_EVENTS_TOPIC_PARTITIONS"diff --git a/self-hosting/.gitignore b/self-hosting/.gitignoreindex 568034cb3..ab97a22ac 100644--- a/self-hosting/.gitignore+++ b/self-hosting/.gitignore@@ -1,3 +1,4 @@.envdocker-compose.ymlcaddy/Caddyfile+redpanda/bootstrap.yamldiff --git a/self-hosting/docker-compose.template.yml b/self-hosting/docker-compose.template.ymlindex e5d36fa86..d8b18ae8b 100644--- a/self-hosting/docker-compose.template.yml+++ b/self-hosting/docker-compose.template.yml@@ -89,6 +89,43 @@ services:max-size: "10m"max-file: "3"+ # Required stateful service (ADR-004): /track always produces to Kafka, no+ # fallback transport. Sizing here is a sane single-node starting point, NOT+ # the --memory=1G/--mode=dev-container cap the root docker-compose.yml uses+ # for local dev — that cap was never measured against real ingest traffic+ # and is not a production baseline. Tune --memory to what the host can+ # spare.+ op-rp:+ image: redpandadata/redpanda:v25.1.4+ restart: always+ command:+ - redpanda+ - start+ - --kafka-addr=internal://0.0.0.0:9092+ - --advertise-kafka-addr=internal://op-rp:9092+ - --rpc-addr=0.0.0.0:33145+ - --advertise-rpc-addr=op-rp:33145+ - --smp=1+ - --memory=2G+ - --default-log-level=info+ volumes:+ - op-rp-data:/var/lib/redpanda/data+ - ./redpanda/bootstrap.yaml:/etc/redpanda/.bootstrap.yaml:ro+ healthcheck:+ test: ["CMD", "rpk", "cluster", "health", "--exit-when-healthy"]+ interval: 10s+ timeout: 5s+ retries: 10+ start_period: 10s+ logging:+ driver: "json-file"+ options:+ max-size: "10m"+ max-file: "3"+ # Uncomment to reach the Kafka API from the host+ # ports:+ # - 19092:9092+op-api:image: lindesvard/openpanel-api:2restart: always@@ -111,6 +148,8 @@ services:condition: service_healthyop-kv:condition: service_healthy+ op-rp:+ condition: service_healthyenv_file:- .envlogging:@@ -145,6 +184,8 @@ services:depends_on:op-api:condition: service_healthy+ op-rp:+ condition: service_healthyenv_file:- .envhealthcheck:@@ -168,6 +209,8 @@ volumes:driver: localop-ch-data:driver: local+ op-rp-data:+ driver: localop-proxy-data:driver: localop-proxy-config:diff --git a/self-hosting/quiz.ts b/self-hosting/quiz.tsindex e7b4ee8a0..85448e201 100644--- a/self-hosting/quiz.ts+++ b/self-hosting/quiz.ts@@ -5,6 +5,15 @@ import bcrypt from 'bcrypt';import inquirer from 'inquirer';import yaml from 'js-yaml';+// Cloud runs 24 (docs/ANSWERS.md §1.1). Override by exporting+// KAFKA_EVENTS_TOPIC_PARTITIONS before running ./setup — see the footgun+// warning in redpanda/bootstrap.template.yaml before changing this after+// first boot.+const KAFKA_EVENTS_TOPIC_PARTITIONS_DEFAULT = 24;+const KAFKA_EVENTS_TOPIC_PARTITIONS =+ process.env.KAFKA_EVENTS_TOPIC_PARTITIONS ||+ String(KAFKA_EVENTS_TOPIC_PARTITIONS_DEFAULT);+let envs = {CLICKHOUSE_URL: '',REDIS_URL: '',@@ -13,6 +22,7 @@ let envs = {COOKIE_SECRET: generatePassword(32),RESEND_API_KEY: '',EMAIL_SENDER: '',+ KAFKA_EVENTS_TOPIC_PARTITIONS,};type EnvVars = typeof envs;@@ -152,7 +162,8 @@ function writeEnvFile(envs: EnvVars) {.replace('$DASHBOARD_URL', stripTrailingSlash(envs.DOMAIN_NAME)).replace('$API_URL', `${stripTrailingSlash(envs.DOMAIN_NAME)}/api`).replace('$RESEND_API_KEY', envs.RESEND_API_KEY)- .replace('$EMAIL_SENDER', envs.EMAIL_SENDER);+ .replace('$EMAIL_SENDER', envs.EMAIL_SENDER)+ .replace('$KAFKA_EVENTS_TOPIC_PARTITIONS', envs.KAFKA_EVENTS_TOPIC_PARTITIONS);fs.writeFileSync(envPath,@@ -165,6 +176,25 @@ function writeEnvFile(envs: EnvVars) {);}+// Mirrors the partition count into the Redpanda bootstrap config so the+// broker's default_topic_partitions can never drift from what .env declares+// — see the footgun warning in redpanda/bootstrap.template.yaml.+function writeRedpandaBootstrap(partitions: string) {+ const templatePath = path.resolve(+ __dirname,+ 'redpanda',+ 'bootstrap.template.yaml',+ );+ const bootstrapPath = path.resolve(__dirname, 'redpanda', 'bootstrap.yaml');++ fs.writeFileSync(+ bootstrapPath,+ fs+ .readFileSync(templatePath, 'utf-8')+ .replaceAll('$KAFKA_EVENTS_TOPIC_PARTITIONS', partitions),+ );+}+async function initiateOnboarding() {const T = ' ';const message = [@@ -384,8 +414,12 @@ async function initiateOnboarding() {COOKIE_SECRET: envs.COOKIE_SECRET,RESEND_API_KEY: envs.RESEND_API_KEY || '',EMAIL_SENDER: envs.EMAIL_SENDER || '',+ KAFKA_EVENTS_TOPIC_PARTITIONS: envs.KAFKA_EVENTS_TOPIC_PARTITIONS,});+ console.log('Writing Redpanda bootstrap config...\n');+ writeRedpandaBootstrap(envs.KAFKA_EVENTS_TOPIC_PARTITIONS);+console.log('Updating docker-compose.yml file...\n');fs.copyFileSync(path.resolve(__dirname, 'docker-compose.template.yml'),diff --git a/self-hosting/redpanda/bootstrap.template.yaml b/self-hosting/redpanda/bootstrap.template.yamlnew file mode 100644index 000000000..df7769a82--- /dev/null+++ b/self-hosting/redpanda/bootstrap.template.yaml@@ -0,0 +1,20 @@+# Redpanda cluster bootstrap config — applied ONCE, when the data volume is+# empty on first boot. `default_topic_partitions` governs topics created+# after that point (including an auto-created `events` topic); it does not+# repartition anything that already exists.+#+# Generated as redpanda/bootstrap.yaml by ./setup from this template — do not+# edit the generated file directly, it is overwritten on the next setup run.+#+# FOOTGUN (ADR-004 risk 6): the producer partitions by+# murmur2(key) % numPartitions, so raising this value after the `events`+# topic already exists reshuffles every key — it silently breaks per-key+# ordering and mutual exclusion for events produced before the change. Set it+# once at install. To change it later: stop producers, let consumers drain+# the backlog fully, delete the `events` topic, then restart so it is+# recreated with the new partition count.+default_topic_partitions: $KAFKA_EVENTS_TOPIC_PARTITIONS+default_topic_replications: 1+auto_create_topics_enabled: true+log_compression_type: lz4+log_retention_ms: 172800000