OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.
ralph(P1-003): ENABLED_QUEUES: the token is `events`; unknown values fail boot loudly
ralphaa3f37f06b06
diff --git a/apps/public/content/docs/self-hosting/environment-variables.mdx b/apps/public/content/docs/self-hosting/environment-variables.mdxindex 5c8b2b4c5..cad504930 100644--- a/apps/public/content/docs/self-hosting/environment-variables.mdx+++ b/apps/public/content/docs/self-hosting/environment-variables.mdx@@ -622,7 +622,7 @@ DISABLE_WORKERS=true**Required**: No**Default**: All queues enabled-Comma-separated list of queue names to enable. Available queues: `events`, `events_kafka`, `sessions`, `cron`, `notification`, `misc`, `import`.+Comma-separated list of queue names to enable. Accepted queues: `events`, `sessions`, `cron`, `notification`, `import`, `insights`, `gsc`, `cohortCompute`. An unknown value (including the old `events_kafka` name, see below) fails startup with a non-zero exit rather than being silently ignored.**Example**:```bash@@ -630,7 +630,7 @@ ENABLED_QUEUES=events,sessions,cron```<Callout>-Enable `events_kafka` to run the Kafka-based events consumer alongside (or instead of) the default `events` queue. See the [Event Streaming (Kafka)](#event-streaming-kafka) section for the full configuration. A common high-throughput layout is to run one worker pool with `ENABLED_QUEUES=events_kafka` and another pool with everything else.+`events` is the Kafka-based events consumer — there is only one events transport now, so there is no separate `events_kafka` queue name. If you're upgrading from a version that used `ENABLED_QUEUES=events_kafka`, rename it to `ENABLED_QUEUES=events`; the old name is now rejected at boot. See the [Event Streaming (Kafka)](#event-streaming-kafka) section for the full configuration. A common high-throughput layout is to run one worker pool with `ENABLED_QUEUES=events` and another pool with everything else.</Callout>### EVENT_JOB_CONCURRENCY@@ -740,7 +740,7 @@ AUTO_BATCH_SIZE=100OpenPanel can optionally ingest events through a Kafka-compatible broker (Kafka itself, Redpanda, etc.) instead of the default Redis-backed events queue. This is useful at high throughput, where many consumer instances can process partitions in parallel.-Routing is all-or-nothing: if `KAFKA_BROKERS` is set, every event is produced to the Kafka topic; otherwise all events stay on the default Redis-backed `events` queue. Set `ENABLED_QUEUES=events_kafka` on the worker(s) that should consume from Kafka.+Routing is all-or-nothing: if `KAFKA_BROKERS` is set, every event is produced to the Kafka topic; otherwise all events stay on the default Redis-backed `events` queue. Set `ENABLED_QUEUES=events` on the worker(s) that should consume from Kafka.<Callout>The client speaks the Kafka wire protocol via [kafkajs](https://kafka.js.org/), so any Kafka-compatible broker works. `self-hosting/docker-compose.template.yml` ships an `op-rp` service running [Redpanda](https://redpanda.com/) — a lightweight, single-binary broker — by default, with `KAFKA_BROKERS` already pointed at it. You can instead point `KAFKA_BROKERS` at any Kafka cluster of your own.diff --git a/apps/worker/src/boot-workers.test.ts b/apps/worker/src/boot-workers.test.tsnew file mode 100644index 000000000..f7fb50d64--- /dev/null+++ b/apps/worker/src/boot-workers.test.ts@@ -0,0 +1,134 @@+/**+ * ENABLED_QUEUES validation only (ADR-004 rec 5/6 + docs/ANSWERS.md §1.3+ * ruling). `bootWorkers()` itself isn't exercised here — it starts real+ * BullMQ workers and the Kafka consumer, which is integration territory.+ */++import { beforeEach, describe, expect, it, vi } from 'vitest';++vi.mock('./utils/logger', () => {+ const makeLogger = (): Record<string, ReturnType<typeof vi.fn>> => {+ const log: Record<string, ReturnType<typeof vi.fn>> = {+ info: vi.fn(),+ warn: vi.fn(),+ error: vi.fn(),+ fatal: vi.fn(),+ };+ log.child = vi.fn(() => makeLogger());+ return log;+ };+ return { logger: makeLogger() };+});++import { assertKnownQueue, getEnabledQueues } from './boot-workers';+import { logger } from './utils/logger';++const fatal = vi.mocked(logger.fatal);++// `process.exit`'s two overloads (string | number | null | undefined) don't+// unify with vi.spyOn's generic inference — pin the mock to one signature.+function mockProcessExit() {+ return vi+ .spyOn(process, 'exit')+ .mockImplementation((_code?: string | number | null): never => {+ return undefined as never;+ });+}++describe('getEnabledQueues', () => {+ const originalEnabledQueues = process.env.ENABLED_QUEUES;+ let exitSpy: ReturnType<typeof mockProcessExit>;++ beforeEach(() => {+ vi.clearAllMocks();+ if (originalEnabledQueues === undefined) {+ delete process.env.ENABLED_QUEUES;+ } else {+ process.env.ENABLED_QUEUES = originalEnabledQueues;+ }+ exitSpy = mockProcessExit();+ });++ it('defaults to events plus the seven registry queues when unset', () => {+ delete process.env.ENABLED_QUEUES;+ expect(getEnabledQueues()).toEqual([+ 'events',+ 'sessions',+ 'cron',+ 'notification',+ 'import',+ 'insights',+ 'gsc',+ 'cohortCompute',+ ]);+ expect(exitSpy).not.toHaveBeenCalled();+ });++ it('accepts a comma-separated subset of known queues', () => {+ process.env.ENABLED_QUEUES = 'events, sessions ,cron';+ expect(getEnabledQueues()).toEqual(['events', 'sessions', 'cron']);+ expect(exitSpy).not.toHaveBeenCalled();+ });++ it('rejects the events_kafka fixture with a message naming the rename', () => {+ process.env.ENABLED_QUEUES = 'events_kafka';+ getEnabledQueues();++ expect(exitSpy).toHaveBeenCalledWith(1);+ expect(fatal).toHaveBeenCalledTimes(1);+ const [meta, message] = fatal.mock.calls[0] ?? [];+ expect(message).toContain('events_kafka');+ expect(message).toContain('renamed');+ expect(message).toContain('events');+ expect(meta).toMatchObject({ value: 'events_kafka', renamedTo: 'events' });+ });++ it('rejects an unknown token, non-zero exit naming value and accepted set', () => {+ process.env.ENABLED_QUEUES = 'not_a_real_queue';+ getEnabledQueues();++ expect(exitSpy).toHaveBeenCalledWith(1);+ expect(fatal).toHaveBeenCalledTimes(1);+ const [meta, message] = fatal.mock.calls[0] ?? [];+ expect(message).toContain('not_a_real_queue');+ expect(meta).toMatchObject({+ value: 'not_a_real_queue',+ accepted: [+ 'events',+ 'sessions',+ 'cron',+ 'notification',+ 'import',+ 'insights',+ 'gsc',+ 'cohortCompute',+ ],+ });+ });+});++describe('assertKnownQueue', () => {+ let exitSpy: ReturnType<typeof mockProcessExit>;++ beforeEach(() => {+ vi.clearAllMocks();+ exitSpy = mockProcessExit();+ });++ it('accepts every registry queue and events', () => {+ for (const name of [+ 'events',+ 'sessions',+ 'cron',+ 'notification',+ 'import',+ 'insights',+ 'gsc',+ 'cohortCompute',+ ]) {+ assertKnownQueue(name);+ }+ expect(fatal).not.toHaveBeenCalled();+ expect(exitSpy).not.toHaveBeenCalled();+ });+});diff --git a/apps/worker/src/boot-workers.ts b/apps/worker/src/boot-workers.tsindex 161af456c..81eb876ba 100644--- a/apps/worker/src/boot-workers.ts+++ b/apps/worker/src/boot-workers.ts@@ -33,27 +33,70 @@ const workerOptions: WorkerOptions = {connection: getRedisQueue(),};-type QueueName = string; // Can be: events_kafka, sessions, cron, notification, import, insights, gsc, cohortCompute+// The Kafka events consumer's token. Renamed from `events_kafka` (ADR-004+// rec 5/6 + docs/ANSWERS.md §1.3): there is one events transport now, so+// there is one token for it.+const KAFKA_QUEUE_NAME = 'events';++// The seven BullMQ registry queues (ADR-005). Kept in sync with+// packages/core/src/jobs.registry.ts by hand until the worker merges into+// core at P9.+const REGISTRY_QUEUE_NAMES = [+ 'sessions',+ 'cron',+ 'notification',+ 'import',+ 'insights',+ 'gsc',+ 'cohortCompute',+] as const;++const KNOWN_QUEUE_NAMES = [KAFKA_QUEUE_NAME, ...REGISTRY_QUEUE_NAMES] as const;++type QueueName = (typeof KNOWN_QUEUE_NAMES)[number];++const RENAMED_QUEUE_TOKENS: Record<string, string> = {+ events_kafka: KAFKA_QUEUE_NAME,+};++/**+ * Fails boot loudly on an unknown ENABLED_QUEUES token, instead of V1's+ * silent ignore (docs/ANSWERS.md §1.3 ruling). `events_kafka` gets a message+ * naming the rename; anything else names the offending value and the+ * accepted set.+ */+export function assertKnownQueue(value: string): void {+ if ((KNOWN_QUEUE_NAMES as readonly string[]).includes(value)) {+ return;+ }++ const renamedTo = RENAMED_QUEUE_TOKENS[value];+ if (renamedTo) {+ logger.fatal(+ { value, renamedTo, accepted: KNOWN_QUEUE_NAMES },+ `ENABLED_QUEUES: "${value}" was renamed to "${renamedTo}" — update ENABLED_QUEUES to use the new name.`+ );+ process.exit(1);+ return;+ }++ logger.fatal(+ { value, accepted: KNOWN_QUEUE_NAMES },+ `ENABLED_QUEUES: unknown queue "${value}". Accepted values: ${KNOWN_QUEUE_NAMES.join(', ')}.`+ );+ process.exit(1);+}/*** Parses the ENABLED_QUEUES environment variable and returns an array of queue names to start.* If no env var is provided, returns all queues.*/-function getEnabledQueues(): QueueName[] {+export function getEnabledQueues(): QueueName[] {const enabledQueuesEnv = process.env.ENABLED_QUEUES?.trim();if (!enabledQueuesEnv) {logger.info('No ENABLED_QUEUES specified, starting all queues');- return [- 'events_kafka',- 'sessions',- 'cron',- 'notification',- 'import',- 'insights',- 'gsc',- 'cohortCompute',- ];+ return [...KNOWN_QUEUE_NAMES];}const queues = enabledQueuesEnv@@ -61,8 +104,12 @@ function getEnabledQueues(): QueueName[] {.map((q) => q.trim()).filter(Boolean);+ for (const queue of queues) {+ assertKnownQueue(queue);+ }+logger.info({ queues }, 'Starting queues from ENABLED_QUEUES');- return queues;+ return queues as QueueName[];}/**@@ -90,7 +137,7 @@ export function bootWorkers() {const extraStops: Array<() => Promise<unknown>> = [];// Start Kafka events consumer — the sole events transport (ADR-004).- if (enabledQueues.includes('events_kafka')) {+ if (enabledQueues.includes(KAFKA_QUEUE_NAME)) {enableEventsHeartbeat();let handle: KafkaConsumerHandle | null = null;const startPromise = startKafkaEventsConsumer()