Skip to content

OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.

ralph(P1-003): ENABLED_QUEUES: the token is `events`; unknown values fail boot loudly

ralphaa3f37f06b06
diff --git a/apps/public/content/docs/self-hosting/environment-variables.mdx b/apps/public/content/docs/self-hosting/environment-variables.mdx
index 5c8b2b4c5..cad504930 100644
--- a/apps/public/content/docs/self-hosting/environment-variables.mdx
+++ b/apps/public/content/docs/self-hosting/environment-variables.mdx
@@ -622,7 +622,7 @@ DISABLE_WORKERS=true
**Required**: No
**Default**: All queues enabled
-Comma-separated list of queue names to enable. Available queues: `events`, `events_kafka`, `sessions`, `cron`, `notification`, `misc`, `import`.
+Comma-separated list of queue names to enable. Accepted queues: `events`, `sessions`, `cron`, `notification`, `import`, `insights`, `gsc`, `cohortCompute`. An unknown value (including the old `events_kafka` name, see below) fails startup with a non-zero exit rather than being silently ignored.
**Example**:
```bash
@@ -630,7 +630,7 @@ ENABLED_QUEUES=events,sessions,cron
```
<Callout>
-Enable `events_kafka` to run the Kafka-based events consumer alongside (or instead of) the default `events` queue. See the [Event Streaming (Kafka)](#event-streaming-kafka) section for the full configuration. A common high-throughput layout is to run one worker pool with `ENABLED_QUEUES=events_kafka` and another pool with everything else.
+`events` is the Kafka-based events consumer — there is only one events transport now, so there is no separate `events_kafka` queue name. If you're upgrading from a version that used `ENABLED_QUEUES=events_kafka`, rename it to `ENABLED_QUEUES=events`; the old name is now rejected at boot. See the [Event Streaming (Kafka)](#event-streaming-kafka) section for the full configuration. A common high-throughput layout is to run one worker pool with `ENABLED_QUEUES=events` and another pool with everything else.
</Callout>
### EVENT_JOB_CONCURRENCY
@@ -740,7 +740,7 @@ AUTO_BATCH_SIZE=100
OpenPanel can optionally ingest events through a Kafka-compatible broker (Kafka itself, Redpanda, etc.) instead of the default Redis-backed events queue. This is useful at high throughput, where many consumer instances can process partitions in parallel.
-Routing is all-or-nothing: if `KAFKA_BROKERS` is set, every event is produced to the Kafka topic; otherwise all events stay on the default Redis-backed `events` queue. Set `ENABLED_QUEUES=events_kafka` on the worker(s) that should consume from Kafka.
+Routing is all-or-nothing: if `KAFKA_BROKERS` is set, every event is produced to the Kafka topic; otherwise all events stay on the default Redis-backed `events` queue. Set `ENABLED_QUEUES=events` on the worker(s) that should consume from Kafka.
<Callout>
The client speaks the Kafka wire protocol via [kafkajs](https://kafka.js.org/), so any Kafka-compatible broker works. `self-hosting/docker-compose.template.yml` ships an `op-rp` service running [Redpanda](https://redpanda.com/) — a lightweight, single-binary broker — by default, with `KAFKA_BROKERS` already pointed at it. You can instead point `KAFKA_BROKERS` at any Kafka cluster of your own.
diff --git a/apps/worker/src/boot-workers.test.ts b/apps/worker/src/boot-workers.test.ts
new file mode 100644
index 000000000..f7fb50d64
--- /dev/null
+++ b/apps/worker/src/boot-workers.test.ts
@@ -0,0 +1,134 @@
+/**
+ * ENABLED_QUEUES validation only (ADR-004 rec 5/6 + docs/ANSWERS.md §1.3
+ * ruling). `bootWorkers()` itself isn't exercised here — it starts real
+ * BullMQ workers and the Kafka consumer, which is integration territory.
+ */
+
+import { beforeEach, describe, expect, it, vi } from 'vitest';
+
+vi.mock('./utils/logger', () => {
+ const makeLogger = (): Record<string, ReturnType<typeof vi.fn>> => {
+ const log: Record<string, ReturnType<typeof vi.fn>> = {
+ info: vi.fn(),
+ warn: vi.fn(),
+ error: vi.fn(),
+ fatal: vi.fn(),
+ };
+ log.child = vi.fn(() => makeLogger());
+ return log;
+ };
+ return { logger: makeLogger() };
+});
+
+import { assertKnownQueue, getEnabledQueues } from './boot-workers';
+import { logger } from './utils/logger';
+
+const fatal = vi.mocked(logger.fatal);
+
+// `process.exit`'s two overloads (string | number | null | undefined) don't
+// unify with vi.spyOn's generic inference — pin the mock to one signature.
+function mockProcessExit() {
+ return vi
+ .spyOn(process, 'exit')
+ .mockImplementation((_code?: string | number | null): never => {
+ return undefined as never;
+ });
+}
+
+describe('getEnabledQueues', () => {
+ const originalEnabledQueues = process.env.ENABLED_QUEUES;
+ let exitSpy: ReturnType<typeof mockProcessExit>;
+
+ beforeEach(() => {
+ vi.clearAllMocks();
+ if (originalEnabledQueues === undefined) {
+ delete process.env.ENABLED_QUEUES;
+ } else {
+ process.env.ENABLED_QUEUES = originalEnabledQueues;
+ }
+ exitSpy = mockProcessExit();
+ });
+
+ it('defaults to events plus the seven registry queues when unset', () => {
+ delete process.env.ENABLED_QUEUES;
+ expect(getEnabledQueues()).toEqual([
+ 'events',
+ 'sessions',
+ 'cron',
+ 'notification',
+ 'import',
+ 'insights',
+ 'gsc',
+ 'cohortCompute',
+ ]);
+ expect(exitSpy).not.toHaveBeenCalled();
+ });
+
+ it('accepts a comma-separated subset of known queues', () => {
+ process.env.ENABLED_QUEUES = 'events, sessions ,cron';
+ expect(getEnabledQueues()).toEqual(['events', 'sessions', 'cron']);
+ expect(exitSpy).not.toHaveBeenCalled();
+ });
+
+ it('rejects the events_kafka fixture with a message naming the rename', () => {
+ process.env.ENABLED_QUEUES = 'events_kafka';
+ getEnabledQueues();
+
+ expect(exitSpy).toHaveBeenCalledWith(1);
+ expect(fatal).toHaveBeenCalledTimes(1);
+ const [meta, message] = fatal.mock.calls[0] ?? [];
+ expect(message).toContain('events_kafka');
+ expect(message).toContain('renamed');
+ expect(message).toContain('events');
+ expect(meta).toMatchObject({ value: 'events_kafka', renamedTo: 'events' });
+ });
+
+ it('rejects an unknown token, non-zero exit naming value and accepted set', () => {
+ process.env.ENABLED_QUEUES = 'not_a_real_queue';
+ getEnabledQueues();
+
+ expect(exitSpy).toHaveBeenCalledWith(1);
+ expect(fatal).toHaveBeenCalledTimes(1);
+ const [meta, message] = fatal.mock.calls[0] ?? [];
+ expect(message).toContain('not_a_real_queue');
+ expect(meta).toMatchObject({
+ value: 'not_a_real_queue',
+ accepted: [
+ 'events',
+ 'sessions',
+ 'cron',
+ 'notification',
+ 'import',
+ 'insights',
+ 'gsc',
+ 'cohortCompute',
+ ],
+ });
+ });
+});
+
+describe('assertKnownQueue', () => {
+ let exitSpy: ReturnType<typeof mockProcessExit>;
+
+ beforeEach(() => {
+ vi.clearAllMocks();
+ exitSpy = mockProcessExit();
+ });
+
+ it('accepts every registry queue and events', () => {
+ for (const name of [
+ 'events',
+ 'sessions',
+ 'cron',
+ 'notification',
+ 'import',
+ 'insights',
+ 'gsc',
+ 'cohortCompute',
+ ]) {
+ assertKnownQueue(name);
+ }
+ expect(fatal).not.toHaveBeenCalled();
+ expect(exitSpy).not.toHaveBeenCalled();
+ });
+});
diff --git a/apps/worker/src/boot-workers.ts b/apps/worker/src/boot-workers.ts
index 161af456c..81eb876ba 100644
--- a/apps/worker/src/boot-workers.ts
+++ b/apps/worker/src/boot-workers.ts
@@ -33,27 +33,70 @@ const workerOptions: WorkerOptions = {
connection: getRedisQueue(),
};
-type QueueName = string; // Can be: events_kafka, sessions, cron, notification, import, insights, gsc, cohortCompute
+// The Kafka events consumer's token. Renamed from `events_kafka` (ADR-004
+// rec 5/6 + docs/ANSWERS.md §1.3): there is one events transport now, so
+// there is one token for it.
+const KAFKA_QUEUE_NAME = 'events';
+
+// The seven BullMQ registry queues (ADR-005). Kept in sync with
+// packages/core/src/jobs.registry.ts by hand until the worker merges into
+// core at P9.
+const REGISTRY_QUEUE_NAMES = [
+ 'sessions',
+ 'cron',
+ 'notification',
+ 'import',
+ 'insights',
+ 'gsc',
+ 'cohortCompute',
+] as const;
+
+const KNOWN_QUEUE_NAMES = [KAFKA_QUEUE_NAME, ...REGISTRY_QUEUE_NAMES] as const;
+
+type QueueName = (typeof KNOWN_QUEUE_NAMES)[number];
+
+const RENAMED_QUEUE_TOKENS: Record<string, string> = {
+ events_kafka: KAFKA_QUEUE_NAME,
+};
+
+/**
+ * Fails boot loudly on an unknown ENABLED_QUEUES token, instead of V1's
+ * silent ignore (docs/ANSWERS.md §1.3 ruling). `events_kafka` gets a message
+ * naming the rename; anything else names the offending value and the
+ * accepted set.
+ */
+export function assertKnownQueue(value: string): void {
+ if ((KNOWN_QUEUE_NAMES as readonly string[]).includes(value)) {
+ return;
+ }
+
+ const renamedTo = RENAMED_QUEUE_TOKENS[value];
+ if (renamedTo) {
+ logger.fatal(
+ { value, renamedTo, accepted: KNOWN_QUEUE_NAMES },
+ `ENABLED_QUEUES: "${value}" was renamed to "${renamedTo}" — update ENABLED_QUEUES to use the new name.`
+ );
+ process.exit(1);
+ return;
+ }
+
+ logger.fatal(
+ { value, accepted: KNOWN_QUEUE_NAMES },
+ `ENABLED_QUEUES: unknown queue "${value}". Accepted values: ${KNOWN_QUEUE_NAMES.join(', ')}.`
+ );
+ process.exit(1);
+}
/**
* Parses the ENABLED_QUEUES environment variable and returns an array of queue names to start.
* If no env var is provided, returns all queues.
*/
-function getEnabledQueues(): QueueName[] {
+export function getEnabledQueues(): QueueName[] {
const enabledQueuesEnv = process.env.ENABLED_QUEUES?.trim();
if (!enabledQueuesEnv) {
logger.info('No ENABLED_QUEUES specified, starting all queues');
- return [
- 'events_kafka',
- 'sessions',
- 'cron',
- 'notification',
- 'import',
- 'insights',
- 'gsc',
- 'cohortCompute',
- ];
+ return [...KNOWN_QUEUE_NAMES];
}
const queues = enabledQueuesEnv
@@ -61,8 +104,12 @@ function getEnabledQueues(): QueueName[] {
.map((q) => q.trim())
.filter(Boolean);
+ for (const queue of queues) {
+ assertKnownQueue(queue);
+ }
+
logger.info({ queues }, 'Starting queues from ENABLED_QUEUES');
- return queues;
+ return queues as QueueName[];
}
/**
@@ -90,7 +137,7 @@ export function bootWorkers() {
const extraStops: Array<() => Promise<unknown>> = [];
// Start Kafka events consumer — the sole events transport (ADR-004).
- if (enabledQueues.includes('events_kafka')) {
+ if (enabledQueues.includes(KAFKA_QUEUE_NAME)) {
enableEventsHeartbeat();
let handle: KafkaConsumerHandle | null = null;
const startPromise = startKafkaEventsConsumer()