OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.
renaming
Carl-Gerhard Lindesvärd57771b6693de
diff --git a/apps/api/src/controllers/track.controller.ts b/apps/api/src/controllers/track.controller.tsindex 2621e28a7..bf6b2d4e3 100644--- a/apps/api/src/controllers/track.controller.ts+++ b/apps/api/src/controllers/track.controller.ts@@ -12,7 +12,7 @@ import {type EventsQueuePayloadIncomingEvent,getEventsGroupQueueShard,produceIncomingEvent,- shouldUseRedpanda,+ shouldUseKafka,} from '@openpanel/queue';import { getRedisCache } from '@openpanel/redis';import type {@@ -218,7 +218,7 @@ async function handleTrack(const partitionKey = groupId || generateId();- if (shouldUseRedpanda(projectId)) {+ if (shouldUseKafka(projectId)) {promises.push(produceIncomingEvent(queueData, partitionKey));} else {promises.push(diff --git a/apps/api/src/utils/graceful-shutdown.ts b/apps/api/src/utils/graceful-shutdown.tsindex 5b5842eb1..d85075915 100644--- a/apps/api/src/utils/graceful-shutdown.ts+++ b/apps/api/src/utils/graceful-shutdown.ts@@ -1,7 +1,7 @@import { ch, db } from '@openpanel/db';import {cronQueue,- disconnectRedpanda,+ disconnectKafka,eventsGroupQueues,miscQueue,notificationQueue,@@ -92,12 +92,12 @@ export async function shutdown(logger.error({ err: error }, 'Error closing queue state');}- // Step 6.5: Disconnect Redpanda producer (no-op if never initialized)+ // Step 6.5: Disconnect Kafka producer (no-op if never initialized)try {- await disconnectRedpanda();- logger.info('Redpanda producer disconnected');+ await disconnectKafka();+ logger.info('Kafka producer disconnected');} catch (error) {- logger.error({ err: error }, 'Error disconnecting Redpanda producer');+ logger.error({ err: error }, 'Error disconnecting Kafka producer');}// Step 7: Close Redis connectionsdiff --git a/apps/public/content/docs/self-hosting/environment-variables.mdx b/apps/public/content/docs/self-hosting/environment-variables.mdxindex cf69981e1..4994f6b19 100644--- a/apps/public/content/docs/self-hosting/environment-variables.mdx+++ b/apps/public/content/docs/self-hosting/environment-variables.mdx@@ -451,13 +451,17 @@ DISABLE_WORKERS=true**Required**: No**Default**: All queues enabled-Comma-separated list of queue names to enable. Available queues: `events`, `sessions`, `cron`, `notification`, `misc`, `import`.+Comma-separated list of queue names to enable. Available queues: `events`, `events_kafka`, `sessions`, `cron`, `notification`, `misc`, `import`.**Example**:```bashENABLED_QUEUES=events,sessions,cron```+<Callout>+Enable `events_kafka` to run the Kafka-based events consumer alongside (or instead of) the default `events` queue. See the [Event Streaming (Kafka)](#event-streaming-kafka) section for the full configuration. A common high-throughput layout is to run one worker pool with `ENABLED_QUEUES=events_kafka` and another pool with everything else.+</Callout>+### EVENT_JOB_CONCURRENCY**Type**: `number`@@ -561,6 +565,165 @@ AUTO_BATCH_SIZE=100⚠️ **Experimental**: This feature is experimental and not used in production. Do not use unless you have a good understanding of the implications and specific performance requirements.</Callout>+## Event Streaming (Kafka)++OpenPanel can optionally ingest events through a Kafka-compatible broker (Kafka itself, Redpanda, etc.) instead of (or alongside) the default Redis-backed events queue. This is opt-in per project and useful at high throughput, where many consumer instances can process partitions in parallel.++Routing is decided per event at produce time: if the project's ID matches `KAFKA_PROJECT_IDS` and `KAFKA_BROKERS` is set, the event is produced to the Kafka topic; otherwise it stays on the default Redis-backed `events` queue. Set `ENABLED_QUEUES=events_kafka` on the worker(s) that should consume from Kafka.++<Callout>+The client speaks the Kafka wire protocol via [kafkajs](https://kafka.js.org/), so any Kafka-compatible broker works. The bundled `docker-compose.yml` uses [Redpanda](https://redpanda.com/) as a lightweight, single-binary option, but you can point `KAFKA_BROKERS` at any Kafka cluster.+</Callout>++### KAFKA_BROKERS++**Type**: `string` (comma-separated)+**Required**: Yes (to enable the Kafka path)+**Default**: None++Comma-separated list of Kafka broker addresses. When unset, the Kafka producer/consumer is disabled and all events fall back to the default Redis-backed queue.++**Example**:+```bash+KAFKA_BROKERS=redpanda:9092+# Multiple brokers+KAFKA_BROKERS=kafka-1:9092,kafka-2:9092,kafka-3:9092+```++### KAFKA_PROJECT_IDS++**Type**: `string` (comma-separated or `*`)+**Required**: No+**Default**: None (no projects route to Kafka)++Allow-list of project IDs whose events should be produced to Kafka. Use `*` to route every project's events through Kafka. If unset, no project uses Kafka even when `KAFKA_BROKERS` is configured.++**Example**:+```bash+# Route every project through Kafka+KAFKA_PROJECT_IDS=*++# Route only specific projects+KAFKA_PROJECT_IDS=proj_abc123,proj_def456+```++### KAFKA_EVENTS_TOPIC++**Type**: `string`+**Required**: No+**Default**: `events`++Name of the Kafka topic used for incoming events.++**Example**:+```bash+KAFKA_EVENTS_TOPIC=openpanel-events+```++### KAFKA_CONSUMER_GROUP++**Type**: `string`+**Required**: No+**Default**: `openpanel-events`++Kafka consumer group ID used by the worker. All worker replicas with the same group ID share partition assignments cooperatively.++**Example**:+```bash+KAFKA_CONSUMER_GROUP=openpanel-events+```++### KAFKA_CLIENT_ID++**Type**: `string`+**Required**: No+**Default**: `openpanel`++Kafka client ID used by both producer and consumer. Shows up in broker logs and metrics.++**Example**:+```bash+KAFKA_CLIENT_ID=openpanel-prod+```++### KAFKA_PARTITIONS_CONCURRENT++**Type**: `number`+**Required**: No+**Default**: `8`++Maximum number of partitions a single consumer instance processes concurrently. Has no effect beyond the number of partitions actually assigned to this consumer.++**Example**:+```bash+KAFKA_PARTITIONS_CONCURRENT=12+```++### KAFKA_MIN_MESSAGES++**Type**: `number`+**Required**: No+**Default**: `1`++Minimum number of messages the broker should accumulate before responding to a fetch request (each message is assumed to be ~1 KiB internally). Higher values create larger batches at the cost of slightly higher latency.++**Example**:+```bash+KAFKA_MIN_MESSAGES=16+```++### KAFKA_MAX_WAIT_MS++**Type**: `number`+**Required**: No+**Default**: `500`++Maximum time in milliseconds the broker waits to fulfil `KAFKA_MIN_MESSAGES` before returning a fetch response. Caps end-to-end latency under low traffic.++**Example**:+```bash+KAFKA_MAX_WAIT_MS=200+```++### KAFKA_MAX_MESSAGES_PER_PARTITION++**Type**: `number`+**Required**: No+**Default**: `256`++Maximum number of messages returned per partition per fetch request (each message is assumed to be ~1 KiB internally). Caps in-memory batch size during backlog recovery and bounds the worst-case time the consumer spends on a single batch.++**Example**:+```bash+KAFKA_MAX_MESSAGES_PER_PARTITION=512+```++### KAFKA_SESSION_TIMEOUT_MS++**Type**: `number`+**Required**: No+**Default**: `30000`++Consumer group session timeout in milliseconds. The broker considers a consumer dead and rebalances its partitions if it does not heartbeat within this window.++**Example**:+```bash+KAFKA_SESSION_TIMEOUT_MS=45000+```++### KAFKA_HEARTBEAT_INTERVAL_MS++**Type**: `number`+**Required**: No+**Default**: `3000`++How often, in milliseconds, the consumer sends background heartbeats to the broker. Should be well below `KAFKA_SESSION_TIMEOUT_MS`.++**Example**:+```bash+KAFKA_HEARTBEAT_INTERVAL_MS=3000+```+## Buffers### SESSION_BUFFER_BATCH_SIZEdiff --git a/apps/worker/src/boot-workers.ts b/apps/worker/src/boot-workers.tsindex aa54541a1..b90b4f793 100644--- a/apps/worker/src/boot-workers.ts+++ b/apps/worker/src/boot-workers.ts@@ -22,9 +22,9 @@ import { cohortComputeJob } from './jobs/cohort.compute';import { cronJob } from './jobs/cron';import { incomingEvent } from './jobs/events.incoming-event';import {- type RedpandaConsumerHandle,- startRedpandaEventsConsumer,-} from './jobs/events.redpanda-consumer';+ type KafkaConsumerHandle,+ startKafkaEventsConsumer,+} from './jobs/events.kafka-consumer';import { gscJob } from './jobs/gsc';import { importJob } from './jobs/import';import { insightsProjectJob } from './jobs/insights';@@ -64,7 +64,7 @@ function getEnabledQueues(): QueueName[] {);return ['events',- 'events_redpanda',+ 'events_kafka','sessions','cron','notification',@@ -169,20 +169,17 @@ export function bootWorkers() {logger.info({ concurrency }, `Started worker for ${queueName}`);}- // Start Redpanda events consumer (parallel to groupmq events for allow-listed project IDs)- if (- enabledQueues.includes('events_redpanda') &&- process.env.REDPANDA_BROKERS- ) {+ // Start Kafka events consumer (parallel to groupmq events for allow-listed project IDs)+ if (enabledQueues.includes('events_kafka') && process.env.KAFKA_BROKERS) {enableEventsHeartbeat();- let handle: RedpandaConsumerHandle | null = null;- const startPromise = startRedpandaEventsConsumer()+ let handle: KafkaConsumerHandle | null = null;+ const startPromise = startKafkaEventsConsumer().then((h) => {handle = h;- logger.info('Started Redpanda events consumer');+ logger.info('Started Kafka events consumer');}).catch((err) => {- logger.error({ err }, 'Failed to start Redpanda events consumer');+ logger.error({ err }, 'Failed to start Kafka events consumer');});extraStops.push(async () => {await startPromise.catch(() => undefined);diff --git a/apps/worker/src/jobs/events.redpanda-consumer.ts b/apps/worker/src/jobs/events.kafka-consumer.tssimilarity index 82%rename from apps/worker/src/jobs/events.redpanda-consumer.tsrename to apps/worker/src/jobs/events.kafka-consumer.tsindex 616836dab..deca022ad 100644--- a/apps/worker/src/jobs/events.redpanda-consumer.ts+++ b/apps/worker/src/jobs/events.kafka-consumer.ts@@ -1,16 +1,16 @@import {- createRedpandaEventsConsumer,+ createKafkaEventsConsumer,type EventsQueuePayloadIncomingEvent,+ KAFKA_EVENTS_TOPIC,+ KAFKA_PARTITIONS_CONCURRENT,+ kafkaLogger,type KafkaMessage,- REDPANDA_EVENTS_TOPIC,- REDPANDA_PARTITIONS_CONCURRENT,- redpandaLogger,} from '@openpanel/queue';import { logger } from '../utils/logger';import { markEventsActivity } from '../utils/worker-heartbeat';import { incomingEvent } from './events.incoming-event';-export interface RedpandaConsumerHandle {+export interface KafkaConsumerHandle {stop: () => Promise<void>;}@@ -19,18 +19,18 @@ export interface RedpandaConsumerHandle {// comfortably under that even for slow handlers.const HEARTBEAT_EVERY = 16;-export async function startRedpandaEventsConsumer(): Promise<RedpandaConsumerHandle> {- const consumer = createRedpandaEventsConsumer();+export async function startKafkaEventsConsumer(): Promise<KafkaConsumerHandle> {+ const consumer = createKafkaEventsConsumer();await consumer.connect();await consumer.subscribe({- topic: REDPANDA_EVENTS_TOPIC,+ topic: KAFKA_EVENTS_TOPIC,fromBeginning: false,});consumer.on(consumer.events.HEARTBEAT, markEventsActivity);await consumer.run({- partitionsConsumedConcurrently: REDPANDA_PARTITIONS_CONCURRENT,+ partitionsConsumedConcurrently: KAFKA_PARTITIONS_CONCURRENT,eachBatchAutoResolve: false,eachBatch: async ({batch,@@ -76,7 +76,7 @@ export async function startRedpandaEventsConsumer(): Promise<RedpandaConsumerHan} catch (err) {logger.error({ err, partition: batch.partition, offset: m.offset },- 'redpanda message parse failed'+ 'kafka message parse failed');}if (payload) {@@ -93,7 +93,7 @@ export async function startRedpandaEventsConsumer(): Promise<RedpandaConsumerHanoffset: m.offset,projectId: payload.projectId,},- 'redpanda incomingEvent handler failed'+ 'kafka incomingEvent handler failed');}}@@ -113,12 +113,12 @@ export async function startRedpandaEventsConsumer(): Promise<RedpandaConsumerHan},});- redpandaLogger.info(+ kafkaLogger.info({- topic: REDPANDA_EVENTS_TOPIC,- partitionsConsumedConcurrently: REDPANDA_PARTITIONS_CONCURRENT,+ topic: KAFKA_EVENTS_TOPIC,+ partitionsConsumedConcurrently: KAFKA_PARTITIONS_CONCURRENT,},- 'redpanda events consumer running'+ 'kafka events consumer running');return {diff --git a/packages/queue/index.ts b/packages/queue/index.tsindex 7abcfd528..3dc13f492 100644--- a/packages/queue/index.ts+++ b/packages/queue/index.ts@@ -1,4 +1,4 @@export * from './src/queues';export type * from './src/queues';-export * from './src/redpanda';+export * from './src/kafka';export type { JobsOptions } from 'bullmq';diff --git a/packages/queue/src/redpanda.ts b/packages/queue/src/kafka.tssimilarity index 52%rename from packages/queue/src/redpanda.tsrename to packages/queue/src/kafka.tsindex 58cd0596d..5a177f880 100644--- a/packages/queue/src/redpanda.ts+++ b/packages/queue/src/kafka.ts@@ -4,7 +4,7 @@ import type { EventsQueuePayloadIncomingEvent } from './queues';export type { KafkaMessage } from 'kafkajs';-export const redpandaLogger = createLogger({ name: 'redpanda' });+export const kafkaLogger = createLogger({ name: 'kafka' });const parseBrokers = (raw: string | undefined): string[] => {if (!raw) {@@ -16,46 +16,45 @@ const parseBrokers = (raw: string | undefined): string[] => {.filter(Boolean);};-export const REDPANDA_BROKERS = parseBrokers(process.env.REDPANDA_BROKERS);-export const REDPANDA_EVENTS_TOPIC =- process.env.REDPANDA_EVENTS_TOPIC || 'events';-export const REDPANDA_CONSUMER_GROUP =- process.env.REDPANDA_CONSUMER_GROUP || 'openpanel-events';-export const REDPANDA_PARTITIONS_CONCURRENT = Number.parseInt(- process.env.REDPANDA_PARTITIONS_CONCURRENT || '8',+export const KAFKA_BROKERS = parseBrokers(process.env.KAFKA_BROKERS);+export const KAFKA_EVENTS_TOPIC = process.env.KAFKA_EVENTS_TOPIC || 'events';+export const KAFKA_CONSUMER_GROUP =+ process.env.KAFKA_CONSUMER_GROUP || 'openpanel-events';+export const KAFKA_PARTITIONS_CONCURRENT = Number.parseInt(+ process.env.KAFKA_PARTITIONS_CONCURRENT || '8',10);+// Approx size of one event payload (observed range ~0.9–1.3 KiB).// We size fetch knobs in messages and convert to bytes via this constant.-const REDPANDA_BYTES_PER_MESSAGE = 1024;+const KAFKA_BYTES_PER_MESSAGE = 1024;-export const REDPANDA_MIN_MESSAGES = Number.parseInt(- process.env.REDPANDA_MIN_MESSAGES || '1',+export const KAFKA_MIN_MESSAGES = Number.parseInt(+ process.env.KAFKA_MIN_MESSAGES || '1',10);-export const REDPANDA_MAX_WAIT_MS = Number.parseInt(- process.env.REDPANDA_MAX_WAIT_MS || '500',+export const KAFKA_MAX_WAIT_MS = Number.parseInt(+ process.env.KAFKA_MAX_WAIT_MS || '500',10);-export const REDPANDA_MAX_MESSAGES_PER_PARTITION = Number.parseInt(- process.env.REDPANDA_MAX_MESSAGES_PER_PARTITION || '256',+export const KAFKA_MAX_MESSAGES_PER_PARTITION = Number.parseInt(+ process.env.KAFKA_MAX_MESSAGES_PER_PARTITION || '256',10);-export const REDPANDA_SESSION_TIMEOUT_MS = Number.parseInt(- process.env.REDPANDA_SESSION_TIMEOUT_MS || '30000',+export const KAFKA_SESSION_TIMEOUT_MS = Number.parseInt(+ process.env.KAFKA_SESSION_TIMEOUT_MS || '30000',10);-export const REDPANDA_HEARTBEAT_INTERVAL_MS = Number.parseInt(- process.env.REDPANDA_HEARTBEAT_INTERVAL_MS || '3000',+export const KAFKA_HEARTBEAT_INTERVAL_MS = Number.parseInt(+ process.env.KAFKA_HEARTBEAT_INTERVAL_MS || '3000',10);-const REDPANDA_MIN_BYTES =- REDPANDA_MIN_MESSAGES * REDPANDA_BYTES_PER_MESSAGE;-const REDPANDA_MAX_BYTES_PER_PARTITION =- REDPANDA_MAX_MESSAGES_PER_PARTITION * REDPANDA_BYTES_PER_MESSAGE;+const KAFKA_MIN_BYTES = KAFKA_MIN_MESSAGES * KAFKA_BYTES_PER_MESSAGE;+const KAFKA_MAX_BYTES_PER_PARTITION =+ KAFKA_MAX_MESSAGES_PER_PARTITION * KAFKA_BYTES_PER_MESSAGE;-const projectIdsEnv = (process.env.REDPANDA_PROJECT_IDS || '').trim();+const projectIdsEnv = (process.env.KAFKA_PROJECT_IDS || '').trim();const allowAllProjects = projectIdsEnv === '*';const projectIdAllowList = new Set<string>(projectIdsEnv && !allowAllProjects@@ -66,10 +65,10 @@ const projectIdAllowList = new Set<string>(: []);-export const isRedpandaConfigured = (): boolean => REDPANDA_BROKERS.length > 0;+export const isKafkaConfigured = (): boolean => KAFKA_BROKERS.length > 0;-export const shouldUseRedpanda = (projectId: string): boolean => {- if (!isRedpandaConfigured()) {+export const shouldUseKafka = (projectId: string): boolean => {+ if (!isKafkaConfigured()) {return false;}if (allowAllProjects) {@@ -80,15 +79,15 @@ export const shouldUseRedpanda = (projectId: string): boolean => {let kafka: Kafka | null = null;const getKafka = (): Kafka => {- if (!isRedpandaConfigured()) {+ if (!isKafkaConfigured()) {throw new Error(- 'REDPANDA_BROKERS env var is not set; cannot create Kafka client'+ 'KAFKA_BROKERS env var is not set; cannot create Kafka client');}if (!kafka) {kafka = new Kafka({- clientId: process.env.REDPANDA_CLIENT_ID || 'openpanel',- brokers: REDPANDA_BROKERS,+ clientId: process.env.KAFKA_CLIENT_ID || 'openpanel',+ brokers: KAFKA_BROKERS,logLevel: logLevel.WARN,});}@@ -113,9 +112,9 @@ const getProducer = async (): Promise<Producer> => {.connect().then(() => {producer = p;- redpandaLogger.info(- { brokers: REDPANDA_BROKERS, topic: REDPANDA_EVENTS_TOPIC },- 'redpanda producer connected'+ kafkaLogger.info(+ { brokers: KAFKA_BROKERS, topic: KAFKA_EVENTS_TOPIC },+ 'kafka producer connected');return p;})@@ -133,7 +132,7 @@ export const produceIncomingEvent = async (): Promise<void> => {const p = await getProducer();await p.send({- topic: REDPANDA_EVENTS_TOPIC,+ topic: KAFKA_EVENTS_TOPIC,messages: [{key: Buffer.from(partitionKey),@@ -145,28 +144,28 @@ export const produceIncomingEvent = async (const consumers = new Set<Consumer>();-export const createRedpandaEventsConsumer = (options?: {+export const createKafkaEventsConsumer = (options?: {groupId?: string;}): Consumer => {const client = getKafka();const consumer = client.consumer({- groupId: options?.groupId || REDPANDA_CONSUMER_GROUP,- sessionTimeout: REDPANDA_SESSION_TIMEOUT_MS,- heartbeatInterval: REDPANDA_HEARTBEAT_INTERVAL_MS,- minBytes: REDPANDA_MIN_BYTES,- maxWaitTimeInMs: REDPANDA_MAX_WAIT_MS,- maxBytesPerPartition: REDPANDA_MAX_BYTES_PER_PARTITION,+ groupId: options?.groupId || KAFKA_CONSUMER_GROUP,+ sessionTimeout: KAFKA_SESSION_TIMEOUT_MS,+ heartbeatInterval: KAFKA_HEARTBEAT_INTERVAL_MS,+ minBytes: KAFKA_MIN_BYTES,+ maxWaitTimeInMs: KAFKA_MAX_WAIT_MS,+ maxBytesPerPartition: KAFKA_MAX_BYTES_PER_PARTITION,});consumers.add(consumer);return consumer;};-export const disconnectRedpanda = async (): Promise<void> => {+export const disconnectKafka = async (): Promise<void> => {const tasks: Promise<unknown>[] = [];for (const c of consumers) {tasks.push(c.disconnect().catch((err) => {- redpandaLogger.error({ err }, 'redpanda consumer disconnect error');+ kafkaLogger.error({ err }, 'kafka consumer disconnect error');}));}@@ -177,7 +176,7 @@ export const disconnectRedpanda = async (): Promise<void> => {producerConnectPromise = null;tasks.push(p.disconnect().catch((err) => {- redpandaLogger.error({ err }, 'redpanda producer disconnect error');+ kafkaLogger.error({ err }, 'kafka producer disconnect error');}));}