Skip to content

OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.

ralph(P1-005): Self-host ships Redpanda: compose template, setup script, real env names

ralph7645e0bea4d5
diff --git a/.env.example b/.env.example
index 639f2d023..69dde0b0d 100644
--- a/.env.example
+++ b/.env.example
@@ -28,18 +28,24 @@ NEXT_PUBLIC_API_URL="http://localhost:3333"
WORKER_PORT=9999
API_PORT=3333
-# REDPANDA (optional, runs in parallel with the groupmq events queue)
-# Empty REDPANDA_BROKERS disables both producer and consumer; everything
-# falls back to groupmq. Set REDPANDA_PROJECT_IDS=* to route all projects,
-# or a comma-separated allow-list to route specific ones.
-# REDPANDA_BROKERS=""
-# REDPANDA_EVENTS_TOPIC="events"
-# REDPANDA_CONSUMER_GROUP="openpanel-events"
-# REDPANDA_PROJECT_IDS=""
-# REDPANDA_PARTITIONS_CONCURRENT="8"
+# KAFKA / REDPANDA (optional, runs in parallel with the groupmq events queue)
+# Empty KAFKA_BROKERS disables both producer and consumer; everything falls
+# back to groupmq. Enable ENABLED_QUEUES=events_kafka on the worker(s) that
+# should consume from Kafka.
+# KAFKA_BROKERS=""
+# KAFKA_EVENTS_TOPIC="events"
+# KAFKA_CONSUMER_GROUP="openpanel-events"
+# KAFKA_PARTITIONS_CONCURRENT="8"
+# Number of partitions Redpanda/Kafka creates the `events` topic with; cloud
+# runs 24. FOOTGUN: the producer partitions by murmur2(key) % numPartitions,
+# so raising this once the topic already exists reshuffles every key and
+# silently breaks per-key ordering and mutual exclusion for events produced
+# before the change. Set it once at install; to raise it later, drain
+# ingestion (stop producers, let consumers fully catch up) and recreate the
+# topic — see self-hosting/redpanda/bootstrap.template.yaml.
+# KAFKA_EVENTS_TOPIC_PARTITIONS="24"
# Local dev with docker-compose:
-# REDPANDA_BROKERS="localhost:19092"
-# REDPANDA_PROJECT_IDS="*"
+# KAFKA_BROKERS="localhost:19092"
# EMAIL - set one or both; SMTP takes priority over Resend if SMTP_HOST is set
# Option A - Resend
diff --git a/.github/smoke/docker-compose.yml b/.github/smoke/docker-compose.yml
index 6151cd8c3..b3e51b5a2 100644
--- a/.github/smoke/docker-compose.yml
+++ b/.github/smoke/docker-compose.yml
@@ -52,6 +52,31 @@ services:
timeout: 5s
retries: 24
+ # Mirrors self-hosting/docker-compose.template.yml's op-rp: same image, same
+ # single-node sizing (not the root docker-compose.yml's 1G dev cap). Bootstrap
+ # config is the checked-in dev file (partitions=24, same as cloud) rather than
+ # the self-host setup script's generated one, which doesn't exist until
+ # ./setup has run — keeps this stack reproducible from this file alone.
+ op-rp:
+ image: redpandadata/redpanda:v25.1.4
+ command:
+ - redpanda
+ - start
+ - --kafka-addr=internal://0.0.0.0:9092
+ - --advertise-kafka-addr=internal://op-rp:9092
+ - --rpc-addr=0.0.0.0:33145
+ - --advertise-rpc-addr=op-rp:33145
+ - --smp=1
+ - --memory=2G
+ - --default-log-level=info
+ volumes:
+ - ../../docker/redpanda/bootstrap.yaml:/etc/redpanda/.bootstrap.yaml:ro
+ healthcheck:
+ test: ["CMD", "rpk", "cluster", "health", "--exit-when-healthy"]
+ interval: 5s
+ timeout: 5s
+ retries: 24
+
op-api:
image: ${OP_API_IMAGE:-openpanel-smoke-image-unset}
command: >
@@ -79,6 +104,8 @@ services:
COOKIE_SECRET: smoke-test-cookie-secret-not-a-real-secret
EMAIL_SENDER: [email protected]
RESEND_API_KEY: re_smoke_test_key
+ KAFKA_BROKERS: op-rp:9092
+ KAFKA_EVENTS_TOPIC_PARTITIONS: "24"
healthcheck:
test: ["CMD-SHELL", "curl -f http://localhost:3000/healthcheck || exit 1"]
interval: 5s
@@ -91,6 +118,8 @@ services:
condition: service_healthy
op-kv:
condition: service_healthy
+ op-rp:
+ condition: service_healthy
op-dashboard:
image: ${OP_DASHBOARD_IMAGE:-openpanel-smoke-image-unset}
@@ -125,3 +154,5 @@ services:
depends_on:
op-api:
condition: service_healthy
+ op-rp:
+ condition: service_healthy
diff --git a/.github/smoke/smoke.sh b/.github/smoke/smoke.sh
index fe950f02c..7b747329a 100755
--- a/.github/smoke/smoke.sh
+++ b/.github/smoke/smoke.sh
@@ -28,7 +28,7 @@ fail() {
echo "::error::$*"
echo "--- container status ---"
docker compose ps || true
- for svc in op-api op-worker op-dashboard; do
+ for svc in op-api op-worker op-dashboard op-rp; do
echo "--- $svc logs ---"
docker compose logs --no-color --tail=200 "$svc" || true
done
diff --git a/apps/public/content/docs/self-hosting/deploy-docker-compose.mdx b/apps/public/content/docs/self-hosting/deploy-docker-compose.mdx
index 1d3f6c1f5..c515022e1 100644
--- a/apps/public/content/docs/self-hosting/deploy-docker-compose.mdx
+++ b/apps/public/content/docs/self-hosting/deploy-docker-compose.mdx
@@ -11,7 +11,7 @@ This guide will help you deploy OpenPanel using Docker Compose. This method give
## Prerequisites
- A VPS or server (Docker and Node will be installed automatically if needed)
-- At least 2GB RAM (4GB+ recommended)
+- At least 4GB RAM (6GB+ recommended) — the stack now includes Redpanda alongside Postgres, Redis and ClickHouse
- Domain name pointing to your server (optional but recommended)
- Basic knowledge of command line
@@ -78,6 +78,7 @@ This will start all required services:
- **op-db**: PostgreSQL database
- **op-kv**: Redis cache
- **op-ch**: ClickHouse analytics database
+- **op-rp**: Redpanda (Kafka-compatible event broker, see [Event Streaming](/docs/self-hosting/environment-variables#event-streaming-kafka))
- **op-api**: OpenPanel API server
- **op-dashboard**: OpenPanel dashboard (frontend)
- **op-worker**: Background worker for processing events
diff --git a/apps/public/content/docs/self-hosting/environment-variables.mdx b/apps/public/content/docs/self-hosting/environment-variables.mdx
index bb4442ebf..5c8b2b4c5 100644
--- a/apps/public/content/docs/self-hosting/environment-variables.mdx
+++ b/apps/public/content/docs/self-hosting/environment-variables.mdx
@@ -743,7 +743,7 @@ OpenPanel can optionally ingest events through a Kafka-compatible broker (Kafka
Routing is all-or-nothing: if `KAFKA_BROKERS` is set, every event is produced to the Kafka topic; otherwise all events stay on the default Redis-backed `events` queue. Set `ENABLED_QUEUES=events_kafka` on the worker(s) that should consume from Kafka.
<Callout>
-The client speaks the Kafka wire protocol via [kafkajs](https://kafka.js.org/), so any Kafka-compatible broker works. The bundled `docker-compose.yml` uses [Redpanda](https://redpanda.com/) as a lightweight, single-binary option, but you can point `KAFKA_BROKERS` at any Kafka cluster.
+The client speaks the Kafka wire protocol via [kafkajs](https://kafka.js.org/), so any Kafka-compatible broker works. `self-hosting/docker-compose.template.yml` ships an `op-rp` service running [Redpanda](https://redpanda.com/) — a lightweight, single-binary broker — by default, with `KAFKA_BROKERS` already pointed at it. You can instead point `KAFKA_BROKERS` at any Kafka cluster of your own.
</Callout>
### KAFKA_BROKERS
@@ -774,6 +774,23 @@ Name of the Kafka topic used for incoming events.
KAFKA_EVENTS_TOPIC=openpanel-events
```
+### KAFKA_EVENTS_TOPIC_PARTITIONS
+
+**Type**: `number`
+**Required**: No
+**Default**: `24`
+
+Number of partitions the bundled Redpanda service creates the `KAFKA_EVENTS_TOPIC` topic with on first boot (matches the partition count OpenPanel Cloud runs). Only takes effect on an empty data volume — it does not repartition a topic that already exists.
+
+**Example**:
+```bash
+KAFKA_EVENTS_TOPIC_PARTITIONS=48
+```
+
+<Callout type="warn">
+Set this once, before the first `./start`. The Kafka/Redpanda partitioner is `murmur2(key) % numPartitions`, so raising the partition count after the topic already has data reshuffles where every key lands — it silently breaks per-key event ordering and the mutual exclusion the session buffer relies on for events produced before the change. To raise it later: stop producers, let consumers fully drain the backlog, delete the `events` topic, then restart so it's recreated with the new count.
+</Callout>
+
### KAFKA_CONSUMER_GROUP
**Type**: `string`
diff --git a/self-hosting/.env.template b/self-hosting/.env.template
index 22044261f..615eab83c 100644
--- a/self-hosting/.env.template
+++ b/self-hosting/.env.template
@@ -14,3 +14,11 @@ API_URL="$API_URL"
COOKIE_SECRET="$COOKIE_SECRET"
EMAIL_SENDER="$EMAIL_SENDER"
RESEND_API_KEY="$RESEND_API_KEY"
+
+# Events transport (ADR-004). The op-rp service (Redpanda) is a required part
+# of this stack, not optional infra — see docker-compose.template.yml.
+KAFKA_BROKERS="op-rp:9092"
+# Partition count for the `events` topic, applied once by redpanda/bootstrap.yaml
+# on first boot. Set once at install — see the footgun warning in
+# redpanda/bootstrap.template.yaml before changing it on a running cluster.
+KAFKA_EVENTS_TOPIC_PARTITIONS="$KAFKA_EVENTS_TOPIC_PARTITIONS"
diff --git a/self-hosting/.gitignore b/self-hosting/.gitignore
index 568034cb3..ab97a22ac 100644
--- a/self-hosting/.gitignore
+++ b/self-hosting/.gitignore
@@ -1,3 +1,4 @@
.env
docker-compose.yml
caddy/Caddyfile
+redpanda/bootstrap.yaml
diff --git a/self-hosting/docker-compose.template.yml b/self-hosting/docker-compose.template.yml
index e5d36fa86..d8b18ae8b 100644
--- a/self-hosting/docker-compose.template.yml
+++ b/self-hosting/docker-compose.template.yml
@@ -89,6 +89,43 @@ services:
max-size: "10m"
max-file: "3"
+ # Required stateful service (ADR-004): /track always produces to Kafka, no
+ # fallback transport. Sizing here is a sane single-node starting point, NOT
+ # the --memory=1G/--mode=dev-container cap the root docker-compose.yml uses
+ # for local dev — that cap was never measured against real ingest traffic
+ # and is not a production baseline. Tune --memory to what the host can
+ # spare.
+ op-rp:
+ image: redpandadata/redpanda:v25.1.4
+ restart: always
+ command:
+ - redpanda
+ - start
+ - --kafka-addr=internal://0.0.0.0:9092
+ - --advertise-kafka-addr=internal://op-rp:9092
+ - --rpc-addr=0.0.0.0:33145
+ - --advertise-rpc-addr=op-rp:33145
+ - --smp=1
+ - --memory=2G
+ - --default-log-level=info
+ volumes:
+ - op-rp-data:/var/lib/redpanda/data
+ - ./redpanda/bootstrap.yaml:/etc/redpanda/.bootstrap.yaml:ro
+ healthcheck:
+ test: ["CMD", "rpk", "cluster", "health", "--exit-when-healthy"]
+ interval: 10s
+ timeout: 5s
+ retries: 10
+ start_period: 10s
+ logging:
+ driver: "json-file"
+ options:
+ max-size: "10m"
+ max-file: "3"
+ # Uncomment to reach the Kafka API from the host
+ # ports:
+ # - 19092:9092
+
op-api:
image: lindesvard/openpanel-api:2
restart: always
@@ -111,6 +148,8 @@ services:
condition: service_healthy
op-kv:
condition: service_healthy
+ op-rp:
+ condition: service_healthy
env_file:
- .env
logging:
@@ -145,6 +184,8 @@ services:
depends_on:
op-api:
condition: service_healthy
+ op-rp:
+ condition: service_healthy
env_file:
- .env
healthcheck:
@@ -168,6 +209,8 @@ volumes:
driver: local
op-ch-data:
driver: local
+ op-rp-data:
+ driver: local
op-proxy-data:
driver: local
op-proxy-config:
diff --git a/self-hosting/quiz.ts b/self-hosting/quiz.ts
index e7b4ee8a0..85448e201 100644
--- a/self-hosting/quiz.ts
+++ b/self-hosting/quiz.ts
@@ -5,6 +5,15 @@ import bcrypt from 'bcrypt';
import inquirer from 'inquirer';
import yaml from 'js-yaml';
+// Cloud runs 24 (docs/ANSWERS.md §1.1). Override by exporting
+// KAFKA_EVENTS_TOPIC_PARTITIONS before running ./setup — see the footgun
+// warning in redpanda/bootstrap.template.yaml before changing this after
+// first boot.
+const KAFKA_EVENTS_TOPIC_PARTITIONS_DEFAULT = 24;
+const KAFKA_EVENTS_TOPIC_PARTITIONS =
+ process.env.KAFKA_EVENTS_TOPIC_PARTITIONS ||
+ String(KAFKA_EVENTS_TOPIC_PARTITIONS_DEFAULT);
+
let envs = {
CLICKHOUSE_URL: '',
REDIS_URL: '',
@@ -13,6 +22,7 @@ let envs = {
COOKIE_SECRET: generatePassword(32),
RESEND_API_KEY: '',
EMAIL_SENDER: '',
+ KAFKA_EVENTS_TOPIC_PARTITIONS,
};
type EnvVars = typeof envs;
@@ -152,7 +162,8 @@ function writeEnvFile(envs: EnvVars) {
.replace('$DASHBOARD_URL', stripTrailingSlash(envs.DOMAIN_NAME))
.replace('$API_URL', `${stripTrailingSlash(envs.DOMAIN_NAME)}/api`)
.replace('$RESEND_API_KEY', envs.RESEND_API_KEY)
- .replace('$EMAIL_SENDER', envs.EMAIL_SENDER);
+ .replace('$EMAIL_SENDER', envs.EMAIL_SENDER)
+ .replace('$KAFKA_EVENTS_TOPIC_PARTITIONS', envs.KAFKA_EVENTS_TOPIC_PARTITIONS);
fs.writeFileSync(
envPath,
@@ -165,6 +176,25 @@ function writeEnvFile(envs: EnvVars) {
);
}
+// Mirrors the partition count into the Redpanda bootstrap config so the
+// broker's default_topic_partitions can never drift from what .env declares
+// — see the footgun warning in redpanda/bootstrap.template.yaml.
+function writeRedpandaBootstrap(partitions: string) {
+ const templatePath = path.resolve(
+ __dirname,
+ 'redpanda',
+ 'bootstrap.template.yaml',
+ );
+ const bootstrapPath = path.resolve(__dirname, 'redpanda', 'bootstrap.yaml');
+
+ fs.writeFileSync(
+ bootstrapPath,
+ fs
+ .readFileSync(templatePath, 'utf-8')
+ .replaceAll('$KAFKA_EVENTS_TOPIC_PARTITIONS', partitions),
+ );
+}
+
async function initiateOnboarding() {
const T = ' ';
const message = [
@@ -384,8 +414,12 @@ async function initiateOnboarding() {
COOKIE_SECRET: envs.COOKIE_SECRET,
RESEND_API_KEY: envs.RESEND_API_KEY || '',
EMAIL_SENDER: envs.EMAIL_SENDER || '',
+ KAFKA_EVENTS_TOPIC_PARTITIONS: envs.KAFKA_EVENTS_TOPIC_PARTITIONS,
});
+ console.log('Writing Redpanda bootstrap config...\n');
+ writeRedpandaBootstrap(envs.KAFKA_EVENTS_TOPIC_PARTITIONS);
+
console.log('Updating docker-compose.yml file...\n');
fs.copyFileSync(
path.resolve(__dirname, 'docker-compose.template.yml'),
diff --git a/self-hosting/redpanda/bootstrap.template.yaml b/self-hosting/redpanda/bootstrap.template.yaml
new file mode 100644
index 000000000..df7769a82
--- /dev/null
+++ b/self-hosting/redpanda/bootstrap.template.yaml
@@ -0,0 +1,20 @@
+# Redpanda cluster bootstrap config — applied ONCE, when the data volume is
+# empty on first boot. `default_topic_partitions` governs topics created
+# after that point (including an auto-created `events` topic); it does not
+# repartition anything that already exists.
+#
+# Generated as redpanda/bootstrap.yaml by ./setup from this template — do not
+# edit the generated file directly, it is overwritten on the next setup run.
+#
+# FOOTGUN (ADR-004 risk 6): the producer partitions by
+# murmur2(key) % numPartitions, so raising this value after the `events`
+# topic already exists reshuffles every key — it silently breaks per-key
+# ordering and mutual exclusion for events produced before the change. Set it
+# once at install. To change it later: stop producers, let consumers drain
+# the backlog fully, delete the `events` topic, then restart so it is
+# recreated with the new partition count.
+default_topic_partitions: $KAFKA_EVENTS_TOPIC_PARTITIONS
+default_topic_replications: 1
+auto_create_topics_enabled: true
+log_compression_type: lz4
+log_retention_ms: 172800000