Skip to content

OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.

renaming

Carl-Gerhard Lindesvärd57771b6693de
diff --git a/apps/api/src/controllers/track.controller.ts b/apps/api/src/controllers/track.controller.ts
index 2621e28a7..bf6b2d4e3 100644
--- a/apps/api/src/controllers/track.controller.ts
+++ b/apps/api/src/controllers/track.controller.ts
@@ -12,7 +12,7 @@ import {
type EventsQueuePayloadIncomingEvent,
getEventsGroupQueueShard,
produceIncomingEvent,
- shouldUseRedpanda,
+ shouldUseKafka,
} from '@openpanel/queue';
import { getRedisCache } from '@openpanel/redis';
import type {
@@ -218,7 +218,7 @@ async function handleTrack(
const partitionKey = groupId || generateId();
- if (shouldUseRedpanda(projectId)) {
+ if (shouldUseKafka(projectId)) {
promises.push(produceIncomingEvent(queueData, partitionKey));
} else {
promises.push(
diff --git a/apps/api/src/utils/graceful-shutdown.ts b/apps/api/src/utils/graceful-shutdown.ts
index 5b5842eb1..d85075915 100644
--- a/apps/api/src/utils/graceful-shutdown.ts
+++ b/apps/api/src/utils/graceful-shutdown.ts
@@ -1,7 +1,7 @@
import { ch, db } from '@openpanel/db';
import {
cronQueue,
- disconnectRedpanda,
+ disconnectKafka,
eventsGroupQueues,
miscQueue,
notificationQueue,
@@ -92,12 +92,12 @@ export async function shutdown(
logger.error({ err: error }, 'Error closing queue state');
}
- // Step 6.5: Disconnect Redpanda producer (no-op if never initialized)
+ // Step 6.5: Disconnect Kafka producer (no-op if never initialized)
try {
- await disconnectRedpanda();
- logger.info('Redpanda producer disconnected');
+ await disconnectKafka();
+ logger.info('Kafka producer disconnected');
} catch (error) {
- logger.error({ err: error }, 'Error disconnecting Redpanda producer');
+ logger.error({ err: error }, 'Error disconnecting Kafka producer');
}
// Step 7: Close Redis connections
diff --git a/apps/public/content/docs/self-hosting/environment-variables.mdx b/apps/public/content/docs/self-hosting/environment-variables.mdx
index cf69981e1..4994f6b19 100644
--- a/apps/public/content/docs/self-hosting/environment-variables.mdx
+++ b/apps/public/content/docs/self-hosting/environment-variables.mdx
@@ -451,13 +451,17 @@ DISABLE_WORKERS=true
**Required**: No
**Default**: All queues enabled
-Comma-separated list of queue names to enable. Available queues: `events`, `sessions`, `cron`, `notification`, `misc`, `import`.
+Comma-separated list of queue names to enable. Available queues: `events`, `events_kafka`, `sessions`, `cron`, `notification`, `misc`, `import`.
**Example**:
```bash
ENABLED_QUEUES=events,sessions,cron
```
+<Callout>
+Enable `events_kafka` to run the Kafka-based events consumer alongside (or instead of) the default `events` queue. See the [Event Streaming (Kafka)](#event-streaming-kafka) section for the full configuration. A common high-throughput layout is to run one worker pool with `ENABLED_QUEUES=events_kafka` and another pool with everything else.
+</Callout>
+
### EVENT_JOB_CONCURRENCY
**Type**: `number`
@@ -561,6 +565,165 @@ AUTO_BATCH_SIZE=100
⚠️ **Experimental**: This feature is experimental and not used in production. Do not use unless you have a good understanding of the implications and specific performance requirements.
</Callout>
+## Event Streaming (Kafka)
+
+OpenPanel can optionally ingest events through a Kafka-compatible broker (Kafka itself, Redpanda, etc.) instead of (or alongside) the default Redis-backed events queue. This is opt-in per project and useful at high throughput, where many consumer instances can process partitions in parallel.
+
+Routing is decided per event at produce time: if the project's ID matches `KAFKA_PROJECT_IDS` and `KAFKA_BROKERS` is set, the event is produced to the Kafka topic; otherwise it stays on the default Redis-backed `events` queue. Set `ENABLED_QUEUES=events_kafka` on the worker(s) that should consume from Kafka.
+
+<Callout>
+The client speaks the Kafka wire protocol via [kafkajs](https://kafka.js.org/), so any Kafka-compatible broker works. The bundled `docker-compose.yml` uses [Redpanda](https://redpanda.com/) as a lightweight, single-binary option, but you can point `KAFKA_BROKERS` at any Kafka cluster.
+</Callout>
+
+### KAFKA_BROKERS
+
+**Type**: `string` (comma-separated)
+**Required**: Yes (to enable the Kafka path)
+**Default**: None
+
+Comma-separated list of Kafka broker addresses. When unset, the Kafka producer/consumer is disabled and all events fall back to the default Redis-backed queue.
+
+**Example**:
+```bash
+KAFKA_BROKERS=redpanda:9092
+# Multiple brokers
+KAFKA_BROKERS=kafka-1:9092,kafka-2:9092,kafka-3:9092
+```
+
+### KAFKA_PROJECT_IDS
+
+**Type**: `string` (comma-separated or `*`)
+**Required**: No
+**Default**: None (no projects route to Kafka)
+
+Allow-list of project IDs whose events should be produced to Kafka. Use `*` to route every project's events through Kafka. If unset, no project uses Kafka even when `KAFKA_BROKERS` is configured.
+
+**Example**:
+```bash
+# Route every project through Kafka
+KAFKA_PROJECT_IDS=*
+
+# Route only specific projects
+KAFKA_PROJECT_IDS=proj_abc123,proj_def456
+```
+
+### KAFKA_EVENTS_TOPIC
+
+**Type**: `string`
+**Required**: No
+**Default**: `events`
+
+Name of the Kafka topic used for incoming events.
+
+**Example**:
+```bash
+KAFKA_EVENTS_TOPIC=openpanel-events
+```
+
+### KAFKA_CONSUMER_GROUP
+
+**Type**: `string`
+**Required**: No
+**Default**: `openpanel-events`
+
+Kafka consumer group ID used by the worker. All worker replicas with the same group ID share partition assignments cooperatively.
+
+**Example**:
+```bash
+KAFKA_CONSUMER_GROUP=openpanel-events
+```
+
+### KAFKA_CLIENT_ID
+
+**Type**: `string`
+**Required**: No
+**Default**: `openpanel`
+
+Kafka client ID used by both producer and consumer. Shows up in broker logs and metrics.
+
+**Example**:
+```bash
+KAFKA_CLIENT_ID=openpanel-prod
+```
+
+### KAFKA_PARTITIONS_CONCURRENT
+
+**Type**: `number`
+**Required**: No
+**Default**: `8`
+
+Maximum number of partitions a single consumer instance processes concurrently. Has no effect beyond the number of partitions actually assigned to this consumer.
+
+**Example**:
+```bash
+KAFKA_PARTITIONS_CONCURRENT=12
+```
+
+### KAFKA_MIN_MESSAGES
+
+**Type**: `number`
+**Required**: No
+**Default**: `1`
+
+Minimum number of messages the broker should accumulate before responding to a fetch request (each message is assumed to be ~1 KiB internally). Higher values create larger batches at the cost of slightly higher latency.
+
+**Example**:
+```bash
+KAFKA_MIN_MESSAGES=16
+```
+
+### KAFKA_MAX_WAIT_MS
+
+**Type**: `number`
+**Required**: No
+**Default**: `500`
+
+Maximum time in milliseconds the broker waits to fulfil `KAFKA_MIN_MESSAGES` before returning a fetch response. Caps end-to-end latency under low traffic.
+
+**Example**:
+```bash
+KAFKA_MAX_WAIT_MS=200
+```
+
+### KAFKA_MAX_MESSAGES_PER_PARTITION
+
+**Type**: `number`
+**Required**: No
+**Default**: `256`
+
+Maximum number of messages returned per partition per fetch request (each message is assumed to be ~1 KiB internally). Caps in-memory batch size during backlog recovery and bounds the worst-case time the consumer spends on a single batch.
+
+**Example**:
+```bash
+KAFKA_MAX_MESSAGES_PER_PARTITION=512
+```
+
+### KAFKA_SESSION_TIMEOUT_MS
+
+**Type**: `number`
+**Required**: No
+**Default**: `30000`
+
+Consumer group session timeout in milliseconds. The broker considers a consumer dead and rebalances its partitions if it does not heartbeat within this window.
+
+**Example**:
+```bash
+KAFKA_SESSION_TIMEOUT_MS=45000
+```
+
+### KAFKA_HEARTBEAT_INTERVAL_MS
+
+**Type**: `number`
+**Required**: No
+**Default**: `3000`
+
+How often, in milliseconds, the consumer sends background heartbeats to the broker. Should be well below `KAFKA_SESSION_TIMEOUT_MS`.
+
+**Example**:
+```bash
+KAFKA_HEARTBEAT_INTERVAL_MS=3000
+```
+
## Buffers
### SESSION_BUFFER_BATCH_SIZE
diff --git a/apps/worker/src/boot-workers.ts b/apps/worker/src/boot-workers.ts
index aa54541a1..b90b4f793 100644
--- a/apps/worker/src/boot-workers.ts
+++ b/apps/worker/src/boot-workers.ts
@@ -22,9 +22,9 @@ import { cohortComputeJob } from './jobs/cohort.compute';
import { cronJob } from './jobs/cron';
import { incomingEvent } from './jobs/events.incoming-event';
import {
- type RedpandaConsumerHandle,
- startRedpandaEventsConsumer,
-} from './jobs/events.redpanda-consumer';
+ type KafkaConsumerHandle,
+ startKafkaEventsConsumer,
+} from './jobs/events.kafka-consumer';
import { gscJob } from './jobs/gsc';
import { importJob } from './jobs/import';
import { insightsProjectJob } from './jobs/insights';
@@ -64,7 +64,7 @@ function getEnabledQueues(): QueueName[] {
);
return [
'events',
- 'events_redpanda',
+ 'events_kafka',
'sessions',
'cron',
'notification',
@@ -169,20 +169,17 @@ export function bootWorkers() {
logger.info({ concurrency }, `Started worker for ${queueName}`);
}
- // Start Redpanda events consumer (parallel to groupmq events for allow-listed project IDs)
- if (
- enabledQueues.includes('events_redpanda') &&
- process.env.REDPANDA_BROKERS
- ) {
+ // Start Kafka events consumer (parallel to groupmq events for allow-listed project IDs)
+ if (enabledQueues.includes('events_kafka') && process.env.KAFKA_BROKERS) {
enableEventsHeartbeat();
- let handle: RedpandaConsumerHandle | null = null;
- const startPromise = startRedpandaEventsConsumer()
+ let handle: KafkaConsumerHandle | null = null;
+ const startPromise = startKafkaEventsConsumer()
.then((h) => {
handle = h;
- logger.info('Started Redpanda events consumer');
+ logger.info('Started Kafka events consumer');
})
.catch((err) => {
- logger.error({ err }, 'Failed to start Redpanda events consumer');
+ logger.error({ err }, 'Failed to start Kafka events consumer');
});
extraStops.push(async () => {
await startPromise.catch(() => undefined);
diff --git a/apps/worker/src/jobs/events.redpanda-consumer.ts b/apps/worker/src/jobs/events.kafka-consumer.ts
similarity index 82%
rename from apps/worker/src/jobs/events.redpanda-consumer.ts
rename to apps/worker/src/jobs/events.kafka-consumer.ts
index 616836dab..deca022ad 100644
--- a/apps/worker/src/jobs/events.redpanda-consumer.ts
+++ b/apps/worker/src/jobs/events.kafka-consumer.ts
@@ -1,16 +1,16 @@
import {
- createRedpandaEventsConsumer,
+ createKafkaEventsConsumer,
type EventsQueuePayloadIncomingEvent,
+ KAFKA_EVENTS_TOPIC,
+ KAFKA_PARTITIONS_CONCURRENT,
+ kafkaLogger,
type KafkaMessage,
- REDPANDA_EVENTS_TOPIC,
- REDPANDA_PARTITIONS_CONCURRENT,
- redpandaLogger,
} from '@openpanel/queue';
import { logger } from '../utils/logger';
import { markEventsActivity } from '../utils/worker-heartbeat';
import { incomingEvent } from './events.incoming-event';
-export interface RedpandaConsumerHandle {
+export interface KafkaConsumerHandle {
stop: () => Promise<void>;
}
@@ -19,18 +19,18 @@ export interface RedpandaConsumerHandle {
// comfortably under that even for slow handlers.
const HEARTBEAT_EVERY = 16;
-export async function startRedpandaEventsConsumer(): Promise<RedpandaConsumerHandle> {
- const consumer = createRedpandaEventsConsumer();
+export async function startKafkaEventsConsumer(): Promise<KafkaConsumerHandle> {
+ const consumer = createKafkaEventsConsumer();
await consumer.connect();
await consumer.subscribe({
- topic: REDPANDA_EVENTS_TOPIC,
+ topic: KAFKA_EVENTS_TOPIC,
fromBeginning: false,
});
consumer.on(consumer.events.HEARTBEAT, markEventsActivity);
await consumer.run({
- partitionsConsumedConcurrently: REDPANDA_PARTITIONS_CONCURRENT,
+ partitionsConsumedConcurrently: KAFKA_PARTITIONS_CONCURRENT,
eachBatchAutoResolve: false,
eachBatch: async ({
batch,
@@ -76,7 +76,7 @@ export async function startRedpandaEventsConsumer(): Promise<RedpandaConsumerHan
} catch (err) {
logger.error(
{ err, partition: batch.partition, offset: m.offset },
- 'redpanda message parse failed'
+ 'kafka message parse failed'
);
}
if (payload) {
@@ -93,7 +93,7 @@ export async function startRedpandaEventsConsumer(): Promise<RedpandaConsumerHan
offset: m.offset,
projectId: payload.projectId,
},
- 'redpanda incomingEvent handler failed'
+ 'kafka incomingEvent handler failed'
);
}
}
@@ -113,12 +113,12 @@ export async function startRedpandaEventsConsumer(): Promise<RedpandaConsumerHan
},
});
- redpandaLogger.info(
+ kafkaLogger.info(
{
- topic: REDPANDA_EVENTS_TOPIC,
- partitionsConsumedConcurrently: REDPANDA_PARTITIONS_CONCURRENT,
+ topic: KAFKA_EVENTS_TOPIC,
+ partitionsConsumedConcurrently: KAFKA_PARTITIONS_CONCURRENT,
},
- 'redpanda events consumer running'
+ 'kafka events consumer running'
);
return {
diff --git a/packages/queue/index.ts b/packages/queue/index.ts
index 7abcfd528..3dc13f492 100644
--- a/packages/queue/index.ts
+++ b/packages/queue/index.ts
@@ -1,4 +1,4 @@
export * from './src/queues';
export type * from './src/queues';
-export * from './src/redpanda';
+export * from './src/kafka';
export type { JobsOptions } from 'bullmq';
diff --git a/packages/queue/src/redpanda.ts b/packages/queue/src/kafka.ts
similarity index 52%
rename from packages/queue/src/redpanda.ts
rename to packages/queue/src/kafka.ts
index 58cd0596d..5a177f880 100644
--- a/packages/queue/src/redpanda.ts
+++ b/packages/queue/src/kafka.ts
@@ -4,7 +4,7 @@ import type { EventsQueuePayloadIncomingEvent } from './queues';
export type { KafkaMessage } from 'kafkajs';
-export const redpandaLogger = createLogger({ name: 'redpanda' });
+export const kafkaLogger = createLogger({ name: 'kafka' });
const parseBrokers = (raw: string | undefined): string[] => {
if (!raw) {
@@ -16,46 +16,45 @@ const parseBrokers = (raw: string | undefined): string[] => {
.filter(Boolean);
};
-export const REDPANDA_BROKERS = parseBrokers(process.env.REDPANDA_BROKERS);
-export const REDPANDA_EVENTS_TOPIC =
- process.env.REDPANDA_EVENTS_TOPIC || 'events';
-export const REDPANDA_CONSUMER_GROUP =
- process.env.REDPANDA_CONSUMER_GROUP || 'openpanel-events';
-export const REDPANDA_PARTITIONS_CONCURRENT = Number.parseInt(
- process.env.REDPANDA_PARTITIONS_CONCURRENT || '8',
+export const KAFKA_BROKERS = parseBrokers(process.env.KAFKA_BROKERS);
+export const KAFKA_EVENTS_TOPIC = process.env.KAFKA_EVENTS_TOPIC || 'events';
+export const KAFKA_CONSUMER_GROUP =
+ process.env.KAFKA_CONSUMER_GROUP || 'openpanel-events';
+export const KAFKA_PARTITIONS_CONCURRENT = Number.parseInt(
+ process.env.KAFKA_PARTITIONS_CONCURRENT || '8',
10
);
+
// Approx size of one event payload (observed range ~0.9–1.3 KiB).
// We size fetch knobs in messages and convert to bytes via this constant.
-const REDPANDA_BYTES_PER_MESSAGE = 1024;
+const KAFKA_BYTES_PER_MESSAGE = 1024;
-export const REDPANDA_MIN_MESSAGES = Number.parseInt(
- process.env.REDPANDA_MIN_MESSAGES || '1',
+export const KAFKA_MIN_MESSAGES = Number.parseInt(
+ process.env.KAFKA_MIN_MESSAGES || '1',
10
);
-export const REDPANDA_MAX_WAIT_MS = Number.parseInt(
- process.env.REDPANDA_MAX_WAIT_MS || '500',
+export const KAFKA_MAX_WAIT_MS = Number.parseInt(
+ process.env.KAFKA_MAX_WAIT_MS || '500',
10
);
-export const REDPANDA_MAX_MESSAGES_PER_PARTITION = Number.parseInt(
- process.env.REDPANDA_MAX_MESSAGES_PER_PARTITION || '256',
+export const KAFKA_MAX_MESSAGES_PER_PARTITION = Number.parseInt(
+ process.env.KAFKA_MAX_MESSAGES_PER_PARTITION || '256',
10
);
-export const REDPANDA_SESSION_TIMEOUT_MS = Number.parseInt(
- process.env.REDPANDA_SESSION_TIMEOUT_MS || '30000',
+export const KAFKA_SESSION_TIMEOUT_MS = Number.parseInt(
+ process.env.KAFKA_SESSION_TIMEOUT_MS || '30000',
10
);
-export const REDPANDA_HEARTBEAT_INTERVAL_MS = Number.parseInt(
- process.env.REDPANDA_HEARTBEAT_INTERVAL_MS || '3000',
+export const KAFKA_HEARTBEAT_INTERVAL_MS = Number.parseInt(
+ process.env.KAFKA_HEARTBEAT_INTERVAL_MS || '3000',
10
);
-const REDPANDA_MIN_BYTES =
- REDPANDA_MIN_MESSAGES * REDPANDA_BYTES_PER_MESSAGE;
-const REDPANDA_MAX_BYTES_PER_PARTITION =
- REDPANDA_MAX_MESSAGES_PER_PARTITION * REDPANDA_BYTES_PER_MESSAGE;
+const KAFKA_MIN_BYTES = KAFKA_MIN_MESSAGES * KAFKA_BYTES_PER_MESSAGE;
+const KAFKA_MAX_BYTES_PER_PARTITION =
+ KAFKA_MAX_MESSAGES_PER_PARTITION * KAFKA_BYTES_PER_MESSAGE;
-const projectIdsEnv = (process.env.REDPANDA_PROJECT_IDS || '').trim();
+const projectIdsEnv = (process.env.KAFKA_PROJECT_IDS || '').trim();
const allowAllProjects = projectIdsEnv === '*';
const projectIdAllowList = new Set<string>(
projectIdsEnv && !allowAllProjects
@@ -66,10 +65,10 @@ const projectIdAllowList = new Set<string>(
: []
);
-export const isRedpandaConfigured = (): boolean => REDPANDA_BROKERS.length > 0;
+export const isKafkaConfigured = (): boolean => KAFKA_BROKERS.length > 0;
-export const shouldUseRedpanda = (projectId: string): boolean => {
- if (!isRedpandaConfigured()) {
+export const shouldUseKafka = (projectId: string): boolean => {
+ if (!isKafkaConfigured()) {
return false;
}
if (allowAllProjects) {
@@ -80,15 +79,15 @@ export const shouldUseRedpanda = (projectId: string): boolean => {
let kafka: Kafka | null = null;
const getKafka = (): Kafka => {
- if (!isRedpandaConfigured()) {
+ if (!isKafkaConfigured()) {
throw new Error(
- 'REDPANDA_BROKERS env var is not set; cannot create Kafka client'
+ 'KAFKA_BROKERS env var is not set; cannot create Kafka client'
);
}
if (!kafka) {
kafka = new Kafka({
- clientId: process.env.REDPANDA_CLIENT_ID || 'openpanel',
- brokers: REDPANDA_BROKERS,
+ clientId: process.env.KAFKA_CLIENT_ID || 'openpanel',
+ brokers: KAFKA_BROKERS,
logLevel: logLevel.WARN,
});
}
@@ -113,9 +112,9 @@ const getProducer = async (): Promise<Producer> => {
.connect()
.then(() => {
producer = p;
- redpandaLogger.info(
- { brokers: REDPANDA_BROKERS, topic: REDPANDA_EVENTS_TOPIC },
- 'redpanda producer connected'
+ kafkaLogger.info(
+ { brokers: KAFKA_BROKERS, topic: KAFKA_EVENTS_TOPIC },
+ 'kafka producer connected'
);
return p;
})
@@ -133,7 +132,7 @@ export const produceIncomingEvent = async (
): Promise<void> => {
const p = await getProducer();
await p.send({
- topic: REDPANDA_EVENTS_TOPIC,
+ topic: KAFKA_EVENTS_TOPIC,
messages: [
{
key: Buffer.from(partitionKey),
@@ -145,28 +144,28 @@ export const produceIncomingEvent = async (
const consumers = new Set<Consumer>();
-export const createRedpandaEventsConsumer = (options?: {
+export const createKafkaEventsConsumer = (options?: {
groupId?: string;
}): Consumer => {
const client = getKafka();
const consumer = client.consumer({
- groupId: options?.groupId || REDPANDA_CONSUMER_GROUP,
- sessionTimeout: REDPANDA_SESSION_TIMEOUT_MS,
- heartbeatInterval: REDPANDA_HEARTBEAT_INTERVAL_MS,
- minBytes: REDPANDA_MIN_BYTES,
- maxWaitTimeInMs: REDPANDA_MAX_WAIT_MS,
- maxBytesPerPartition: REDPANDA_MAX_BYTES_PER_PARTITION,
+ groupId: options?.groupId || KAFKA_CONSUMER_GROUP,
+ sessionTimeout: KAFKA_SESSION_TIMEOUT_MS,
+ heartbeatInterval: KAFKA_HEARTBEAT_INTERVAL_MS,
+ minBytes: KAFKA_MIN_BYTES,
+ maxWaitTimeInMs: KAFKA_MAX_WAIT_MS,
+ maxBytesPerPartition: KAFKA_MAX_BYTES_PER_PARTITION,
});
consumers.add(consumer);
return consumer;
};
-export const disconnectRedpanda = async (): Promise<void> => {
+export const disconnectKafka = async (): Promise<void> => {
const tasks: Promise<unknown>[] = [];
for (const c of consumers) {
tasks.push(
c.disconnect().catch((err) => {
- redpandaLogger.error({ err }, 'redpanda consumer disconnect error');
+ kafkaLogger.error({ err }, 'kafka consumer disconnect error');
})
);
}
@@ -177,7 +176,7 @@ export const disconnectRedpanda = async (): Promise<void> => {
producerConnectPromise = null;
tasks.push(
p.disconnect().catch((err) => {
- redpandaLogger.error({ err }, 'redpanda producer disconnect error');
+ kafkaLogger.error({ err }, 'kafka producer disconnect error');
})
);
}