Skip to content

OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.

ralph(P1-001): Delete the GroupMQ transport: /track always produces to Kafka

ralph9d77b0df28f5
diff --git a/apps/api/package.json b/apps/api/package.json
index 7ed09ec36..c9e2ffd3b 100644
--- a/apps/api/package.json
+++ b/apps/api/package.json
@@ -49,7 +49,6 @@
"fastify-metrics": "^12.1.0",
"fastify-raw-body": "^5.0.0",
"fastify-zod-openapi": "^5.6.1",
- "groupmq": "catalog:",
"pino": "catalog:",
"pino-pretty": "catalog:",
"ramda": "^0.29.1",
diff --git a/apps/api/src/controllers/event.controller.ts b/apps/api/src/controllers/event.controller.ts
index a83ed359a..2fa2902aa 100644
--- a/apps/api/src/controllers/event.controller.ts
+++ b/apps/api/src/controllers/event.controller.ts
@@ -4,9 +4,7 @@ import { getSalts } from '@openpanel/db';
import { getAsnInfo, getGeoLocation } from '@openpanel/geo';
import {
type EventsQueuePayloadIncomingEvent,
- getEventsGroupQueueShard,
produceIncomingEvent,
- shouldUseKafka,
} from '@openpanel/queue';
import type { DeprecatedPostEventPayload } from '@openpanel/validation';
import type { FastifyReply, FastifyRequest } from 'fastify';
@@ -74,15 +72,7 @@ export async function postEvent(
sessionId: sessionId ?? '',
};
- if (shouldUseKafka()) {
- await produceIncomingEvent(queueData, groupId);
- } else {
- await getEventsGroupQueueShard(groupId).add({
- orderMs: new Date(timestamp).getTime(),
- data: queueData,
- groupId,
- });
- }
+ await produceIncomingEvent(queueData, groupId);
reply.status(202).send('ok');
}
diff --git a/apps/api/src/controllers/track.controller.test.ts b/apps/api/src/controllers/track.controller.test.ts
index 5632c63ec..adbcfe37a 100644
--- a/apps/api/src/controllers/track.controller.test.ts
+++ b/apps/api/src/controllers/track.controller.test.ts
@@ -5,19 +5,50 @@
* device_id, so it must be trimmed, non-empty, and length-bounded).
* - handleReplay — replay files a chunk under the session id the SDK echoes back;
* it needs no device resolution and trusts the client-sent session id.
+ * - handler — the single events transport (ADR-004): every tracked event goes to
+ * Kafka via produceIncomingEvent, with no branch and no fallback queue.
+ *
+ * `@openpanel/db` and `@openpanel/queue` are partially mocked (importActual +
+ * overrides) rather than replaced: they import each other, and the replay test
+ * below spies on the real `replayBuffer`.
*/
-import { replayBuffer } from '@openpanel/db';
+import {
+ getSalts,
+ replayBuffer,
+ sessionBuffer,
+ upsertProfile,
+} from '@openpanel/db';
+import { getAsnInfo, getGeoLocation } from '@openpanel/geo';
+import { produceIncomingEvent } from '@openpanel/queue';
import type {
IReplayPayload,
ITrackHandlerPayload,
} from '@openpanel/validation';
+import type { FastifyReply, FastifyRequest } from 'fastify';
import { beforeEach, describe, expect, it, vi } from 'vitest';
-import { getOverrideDeviceId, handleReplay } from './track.controller';
+import { getOverrideDeviceId, handleReplay, handler } from './track.controller';
-const track = (
- properties?: Record<string, unknown>
-): ITrackHandlerPayload =>
+vi.mock('@openpanel/queue', async () => {
+ const actual =
+ await vi.importActual<typeof import('@openpanel/queue')>(
+ '@openpanel/queue'
+ );
+ return { ...actual, produceIncomingEvent: vi.fn() };
+});
+
+vi.mock('@openpanel/geo', () => ({
+ getGeoLocation: vi.fn(),
+ getAsnInfo: vi.fn(),
+}));
+
+vi.mock('@openpanel/db', async () => {
+ const actual =
+ await vi.importActual<typeof import('@openpanel/db')>('@openpanel/db');
+ return { ...actual, getSalts: vi.fn(), upsertProfile: vi.fn() };
+});
+
+const track = (properties?: Record<string, unknown>): ITrackHandlerPayload =>
({
type: 'track',
payload: { name: 'page_view', properties },
@@ -106,3 +137,113 @@ describe('handleReplay', () => {
expect(add).not.toHaveBeenCalled();
});
});
+
+const PROJECT_ID = 'proj-kafka';
+const DEVICE_ID = 'device-kafka-1';
+
+const makeRequest = (body: ITrackHandlerPayload) =>
+ ({
+ body,
+ client: { projectId: PROJECT_ID },
+ clientIp: '1.2.3.4',
+ clientSecretAuth: false,
+ timestamp: new Date('2026-06-08T12:00:00.000Z').getTime(),
+ headers: { 'user-agent': 'Mozilla/5.0 (Macintosh) Chrome/148.0.0.0' },
+ }) as unknown as FastifyRequest<{ Body: ITrackHandlerPayload }>;
+
+const makeReply = () => {
+ const reply = {
+ statusCode: 0,
+ body: undefined as unknown,
+ status(code: number) {
+ reply.statusCode = code;
+ return reply;
+ },
+ send(payload: unknown) {
+ reply.body = payload;
+ return reply;
+ },
+ };
+ return reply as unknown as FastifyReply & typeof reply;
+};
+
+describe('handler — events transport', () => {
+ beforeEach(() => {
+ vi.clearAllMocks();
+ vi.mocked(getSalts).mockResolvedValue({
+ current: 'salt-current',
+ previous: 'salt-previous',
+ });
+ vi.mocked(getGeoLocation).mockResolvedValue({
+ country: 'SE',
+ city: 'Stockholm',
+ region: 'AB',
+ longitude: 18,
+ latitude: 59,
+ });
+ vi.mocked(getAsnInfo).mockResolvedValue({
+ asn: undefined,
+ org: undefined,
+ isHosting: false,
+ });
+ vi.mocked(produceIncomingEvent).mockResolvedValue(undefined);
+ vi.spyOn(sessionBuffer, 'getExistingSession').mockResolvedValue(null);
+ });
+
+ it('produces every tracked event to Kafka — the only transport', async () => {
+ await handler(
+ makeRequest({
+ type: 'track',
+ payload: {
+ name: 'page_view',
+ properties: { __deviceId: DEVICE_ID },
+ },
+ } as ITrackHandlerPayload),
+ makeReply()
+ );
+
+ expect(produceIncomingEvent).toHaveBeenCalledTimes(1);
+ const [queueData, partitionKey] =
+ vi.mocked(produceIncomingEvent).mock.calls[0]!;
+ expect(queueData).toMatchObject({
+ projectId: PROJECT_ID,
+ deviceId: DEVICE_ID,
+ event: { name: 'page_view' },
+ });
+ // Browser events partition on the device id, which is what preserves
+ // per-device mutual exclusion in the consumer.
+ expect(partitionKey).toBe(DEVICE_ID);
+ });
+
+ it('surfaces a produce failure instead of falling back to another queue', async () => {
+ vi.mocked(produceIncomingEvent).mockRejectedValue(
+ new Error('kafka unavailable')
+ );
+
+ await expect(
+ handler(
+ makeRequest({
+ type: 'track',
+ payload: { name: 'page_view', properties: { __deviceId: DEVICE_ID } },
+ } as ITrackHandlerPayload),
+ makeReply()
+ )
+ ).rejects.toThrow('kafka unavailable');
+
+ expect(produceIncomingEvent).toHaveBeenCalledTimes(1);
+ });
+
+ it('does not produce for non-event payload types', async () => {
+ vi.mocked(upsertProfile).mockResolvedValue(undefined as never);
+
+ await handler(
+ makeRequest({
+ type: 'identify',
+ payload: { profileId: 'p1', properties: {} },
+ } as ITrackHandlerPayload),
+ makeReply()
+ );
+
+ expect(produceIncomingEvent).not.toHaveBeenCalled();
+ });
+});
diff --git a/apps/api/src/controllers/track.controller.ts b/apps/api/src/controllers/track.controller.ts
index 3452de4f8..a79bc4ae1 100644
--- a/apps/api/src/controllers/track.controller.ts
+++ b/apps/api/src/controllers/track.controller.ts
@@ -18,9 +18,7 @@ import {
} from '@openpanel/geo';
import {
type EventsQueuePayloadIncomingEvent,
- getEventsGroupQueueShard,
produceIncomingEvent,
- shouldUseKafka,
} from '@openpanel/queue';
import type {
IAssignGroupPayload,
@@ -268,17 +266,7 @@ async function handleTrack(
const partitionKey = groupId || generateId();
- if (shouldUseKafka()) {
- promises.push(produceIncomingEvent(queueData, partitionKey));
- } else {
- promises.push(
- getEventsGroupQueueShard(partitionKey).add({
- orderMs: timestamp.value,
- data: queueData,
- groupId,
- })
- );
- }
+ promises.push(produceIncomingEvent(queueData, partitionKey));
await Promise.all(promises);
}
diff --git a/apps/api/src/index.ts b/apps/api/src/index.ts
index 903e1401a..a071d9b56 100644
--- a/apps/api/src/index.ts
+++ b/apps/api/src/index.ts
@@ -4,14 +4,24 @@ process.env.TZ = 'UTC';
import './utils/observability';
import { rawStderrWrite } from '@openpanel/logger';
+import { assertKafkaConfigured } from '@openpanel/queue';
+import { getRedisPub } from '@openpanel/redis';
import sourceMapSupport from 'source-map-support';
import { buildApp } from './app';
import { shutdown } from './utils/graceful-shutdown';
import { logger } from './utils/logger';
-import { getRedisPub } from '@openpanel/redis';
sourceMapSupport.install();
+// Kafka/Redpanda is the sole events transport (ADR-004) — a missing broker list
+// must kill boot, not surface as a 5xx on the first tracked event.
+try {
+ assertKafkaConfigured();
+} catch (error) {
+ logger.fatal({ err: error }, 'Refusing to start');
+ process.exit(1);
+}
+
const port = Number.parseInt(process.env.API_PORT || '3000', 10);
const host =
process.env.API_HOST ||
diff --git a/apps/api/src/utils/graceful-shutdown.ts b/apps/api/src/utils/graceful-shutdown.ts
index c16626521..f95592499 100644
--- a/apps/api/src/utils/graceful-shutdown.ts
+++ b/apps/api/src/utils/graceful-shutdown.ts
@@ -2,7 +2,6 @@ import { ch, db } from '@openpanel/db';
import {
cronQueue,
disconnectKafka,
- eventsGroupQueues,
notificationQueue,
sessionsQueue,
} from '@openpanel/queue';
@@ -94,7 +93,6 @@ export async function shutdown(
// Step 6: Close Bull queues (graceful shutdown of queue state)
try {
await Promise.all([
- ...eventsGroupQueues.map((queue) => queue.close()),
sessionsQueue.close(),
cronQueue.close(),
notificationQueue.close(),
diff --git a/apps/worker/package.json b/apps/worker/package.json
index 7cd99969e..e24a8a299 100644
--- a/apps/worker/package.json
+++ b/apps/worker/package.json
@@ -29,7 +29,6 @@
"bullmq": "^5.63.0",
"date-fns": "^3.3.1",
"express": "^4.18.2",
- "groupmq": "catalog:",
"pino": "catalog:",
"pino-pretty": "catalog:",
"prom-client": "^15.1.3",
diff --git a/apps/worker/src/boot-workers.ts b/apps/worker/src/boot-workers.ts
index e2a6e1968..161af456c 100644
--- a/apps/worker/src/boot-workers.ts
+++ b/apps/worker/src/boot-workers.ts
@@ -4,24 +4,17 @@ import { rawStderrWrite } from '@openpanel/logger';
import {
cohortComputeQueue,
cronQueue,
- EVENTS_GROUP_QUEUES_SHARDS,
- type EventsQueuePayloadIncomingEvent,
- eventsGroupQueues,
gscQueue,
importQueue,
insightsQueue,
- isKafkaConfigured,
notificationQueue,
- queueLogger,
sessionsQueue,
} from '@openpanel/queue';
import { getRedisQueue } from '@openpanel/redis';
import type { Queue, WorkerOptions } from 'bullmq';
import { Worker } from 'bullmq';
-import { Worker as GroupWorker } from 'groupmq';
import { cohortComputeJob } from './jobs/cohort.compute';
import { cronJob } from './jobs/cron';
-import { incomingEvent } from './jobs/events.incoming-event';
import {
type KafkaConsumerHandle,
startKafkaEventsConsumer,
@@ -34,36 +27,24 @@ import { sessionsJob } from './jobs/sessions';
import { eventsGroupJobDuration } from './metrics';
import { setShuttingDown } from './utils/graceful-shutdown';
import { logger } from './utils/logger';
-import {
- enableEventsHeartbeat,
- markEventsActivity,
-} from './utils/worker-heartbeat';
+import { enableEventsHeartbeat } from './utils/worker-heartbeat';
const workerOptions: WorkerOptions = {
connection: getRedisQueue(),
};
-type QueueName = string; // Can be: events, events_N (where N is 0 to shards-1), sessions, cron, notification
+type QueueName = string; // Can be: events_kafka, sessions, cron, notification, import, insights, gsc, cohortCompute
/**
* Parses the ENABLED_QUEUES environment variable and returns an array of queue names to start.
* If no env var is provided, returns all queues.
- *
- * Supported queue names:
- * - events - All event shards (events_0, events_1, ..., events_N)
- * - events_N - Individual event shard (where N is 0 to EVENTS_GROUP_QUEUES_SHARDS-1)
- * - sessions, cron, notification
*/
function getEnabledQueues(): QueueName[] {
const enabledQueuesEnv = process.env.ENABLED_QUEUES?.trim();
if (!enabledQueuesEnv) {
- logger.info(
- { totalEventShards: EVENTS_GROUP_QUEUES_SHARDS },
- 'No ENABLED_QUEUES specified, starting all queues'
- );
+ logger.info('No ENABLED_QUEUES specified, starting all queues');
return [
- 'events',
'events_kafka',
'sessions',
'cron',
@@ -80,16 +61,13 @@ function getEnabledQueues(): QueueName[] {
.map((q) => q.trim())
.filter(Boolean);
- logger.info(
- { queues, totalEventShards: EVENTS_GROUP_QUEUES_SHARDS },
- 'Starting queues from ENABLED_QUEUES'
- );
+ logger.info({ queues }, 'Starting queues from ENABLED_QUEUES');
return queues;
}
/**
* Gets the concurrency setting for a queue from environment variables.
- * Env var format: {QUEUE_NAME}_CONCURRENCY (e.g., EVENTS_0_CONCURRENCY=32)
+ * Env var format: {QUEUE_NAME}_CONCURRENCY (e.g., SESSIONS_CONCURRENCY=32)
*/
function getConcurrencyFor(queueName: string, defaultValue = 1): number {
const envKey = `${queueName.toUpperCase().replace(/[^A-Z0-9]/g, '_')}_CONCURRENCY`;
@@ -108,84 +86,11 @@ function getConcurrencyFor(queueName: string, defaultValue = 1): number {
export function bootWorkers() {
const enabledQueues = getEnabledQueues();
- const workers: (Worker | GroupWorker<any>)[] = [];
+ const workers: Worker[] = [];
const extraStops: Array<() => Promise<unknown>> = [];
- // Start event workers based on enabled queues.
- // When Kafka is configured the producer routes every event to Kafka, so the
- // GroupMQ event shards would only poll an empty queue — skip them entirely
- // and let the Kafka consumer handle ingestion.
- const eventQueuesToStart: number[] = [];
-
- if (isKafkaConfigured()) {
- logger.info('Kafka is configured, skipping GroupMQ event workers');
- } else if (enabledQueues.includes('events')) {
- // Start all event shards
- for (let i = 0; i < EVENTS_GROUP_QUEUES_SHARDS; i++) {
- eventQueuesToStart.push(i);
- }
- } else {
- // Start specific event shards (events_0, events_1, etc.)
- for (let i = 0; i < EVENTS_GROUP_QUEUES_SHARDS; i++) {
- if (enabledQueues.includes(`events_${i}`)) {
- eventQueuesToStart.push(i);
- }
- }
- }
-
- if (eventQueuesToStart.length > 0) {
- enableEventsHeartbeat();
- }
-
- for (const index of eventQueuesToStart) {
- const queue = eventsGroupQueues[index];
- if (!queue) {
- continue;
- }
-
- const queueName = `events_${index}`;
- const concurrency = getConcurrencyFor(
- queueName,
- Number.parseInt(process.env.EVENT_JOB_CONCURRENCY || '10', 10)
- );
-
- const worker = new GroupWorker<EventsQueuePayloadIncomingEvent['payload']>({
- queue,
- concurrency,
- logger: process.env.NODE_ENV === 'production' ? queueLogger : undefined,
- blockingTimeoutSec: Number.parseFloat(
- process.env.EVENT_BLOCKING_TIMEOUT_SEC || '1'
- ),
- handler: async (job) => {
- return await incomingEvent(job.data);
- },
- });
-
- // Consumer-loop heartbeat for the readiness probe. `completed` fires after
- // each processed job; `drained` fires on each poll cycle that finds the
- // queue empty. Together they refresh the timestamp every poll cycle while
- // the consumer is alive — busy or idle.
- worker.on('completed', markEventsActivity);
- worker.on('drained', markEventsActivity);
-
- // Fail loud on startup — silent stuck shard otherwise.
- // Runtime errors are handled by the shared workers.forEach listener below.
- worker.run().catch((err) => {
- logger.error(
- { shard: index, queueName, err },
- 'Worker startup failed — exiting',
- );
- // setTimeout+unref to let the logger flush before exit (matches the
- // pattern used by uncaughtException/unhandledRejection handlers below).
- setTimeout(() => process.exit(1), 1000).unref();
- });
- workers.push(worker);
- logger.info({ concurrency }, `Started worker for ${queueName}`);
- }
-
- // Start Kafka events consumer. When Kafka is configured this fully replaces
- // the GroupMQ event workers (which are skipped above).
- if (enabledQueues.includes('events_kafka') && isKafkaConfigured()) {
+ // Start Kafka events consumer — the sole events transport (ADR-004).
+ if (enabledQueues.includes('events_kafka')) {
enableEventsHeartbeat();
let handle: KafkaConsumerHandle | null = null;
const startPromise = startKafkaEventsConsumer()
@@ -293,19 +198,19 @@ export function bootWorkers() {
}
workers.forEach((worker) => {
- (worker as Worker).on('error', (error) => {
+ worker.on('error', (error) => {
logger.error({ err: error, worker: worker.name }, 'worker error');
});
- (worker as Worker).on('closed', () => {
+ worker.on('closed', () => {
logger.info({ worker: worker.name }, 'worker closed');
});
- (worker as Worker).on('ready', () => {
+ worker.on('ready', () => {
logger.info({ worker: worker.name }, 'worker ready');
});
- (worker as Worker).on('failed', (job) => {
+ worker.on('failed', (job) => {
if (job) {
if (job.processedOn && job.finishedOn) {
const elapsed = job.finishedOn - job.processedOn;
@@ -327,7 +232,7 @@ export function bootWorkers() {
}
});
- (worker as Worker).on('ioredis:close', () => {
+ worker.on('ioredis:close', () => {
logger.error(
{ worker: worker.name },
'worker closed due to ioredis:close'
@@ -348,9 +253,7 @@ export function bootWorkers() {
// before Docker's stop_grace_period elapses. Without this the
// container sits in "Stopping" until SIGKILL (exit 137) and looks
// like a real crash to the swarm.
- const forceExitMs = Number(
- process.env.SHUTDOWN_FORCE_EXIT_MS || '20000'
- );
+ const forceExitMs = Number(process.env.SHUTDOWN_FORCE_EXIT_MS || '20000');
const exitCode = Number.isNaN(+evtOrExitCodeOrError)
? 1
: +evtOrExitCodeOrError;
@@ -418,7 +321,7 @@ export function bootWorkers() {
rawStderrWrite(
`Unhandled rejection — exiting: ${
reason instanceof Error ? reason.stack : String(reason)
- }\n`,
+ }\n`
);
setShuttingDown(true);
setTimeout(() => process.exit(1), 1000).unref();
diff --git a/apps/worker/src/index.ts b/apps/worker/src/index.ts
index 0a38bffa2..e4c879f02 100644
--- a/apps/worker/src/index.ts
+++ b/apps/worker/src/index.ts
@@ -6,9 +6,9 @@ import { ExpressAdapter } from '@bull-board/express';
import { tryCatch } from '@openpanel/common';
import { chQuery, createInitialSalts, db } from '@openpanel/db';
import {
+ assertKafkaConfigured,
cohortComputeQueue,
cronQueue,
- eventsGroupQueues,
gscQueue,
importQueue,
insightsQueue,
@@ -17,7 +17,6 @@ import {
} from '@openpanel/queue';
import { getRedisCache } from '@openpanel/redis';
import express from 'express';
-import { BullBoardGroupMQAdapter } from 'groupmq';
import client from 'prom-client';
import sourceMapSupport from 'source-map-support';
import { bootCron } from './boot-cron';
@@ -32,6 +31,15 @@ const EVENTS_HEARTBEAT_STALE_MS = 60_000;
sourceMapSupport.install();
+// Kafka/Redpanda is the sole events transport (ADR-004) — a missing broker list
+// must kill boot, not surface as a failing consumer nobody notices.
+try {
+ assertKafkaConfigured();
+} catch (error) {
+ logger.fatal({ err: error }, 'Refusing to start');
+ process.exit(1);
+}
+
async function start() {
const collectDefaultMetrics = client.collectDefaultMetrics;
collectDefaultMetrics({ register });
@@ -53,9 +61,6 @@ async function start() {
serverAdapter.setBasePath('/');
createBullBoard({
queues: [
- ...eventsGroupQueues.map(
- (queue) => new BullBoardGroupMQAdapter(queue) as any
- ),
new BullMQAdapter(sessionsQueue),
new BullMQAdapter(cronQueue),
new BullMQAdapter(notificationQueue),
diff --git a/apps/worker/src/metrics.ts b/apps/worker/src/metrics.ts
index 2f595e8a0..c48632127 100644
--- a/apps/worker/src/metrics.ts
+++ b/apps/worker/src/metrics.ts
@@ -7,7 +7,7 @@ import {
replayBuffer,
sessionBuffer,
} from '@openpanel/db';
-import { cronQueue, eventsGroupQueues, sessionsQueue } from '@openpanel/queue';
+import { cronQueue, sessionsQueue } from '@openpanel/queue';
import { getRedisCache } from '@openpanel/redis';
import client from 'prom-client';
@@ -15,7 +15,7 @@ const Registry = client.Registry;
export const register = new Registry();
-const queues = [sessionsQueue, cronQueue, ...eventsGroupQueues];
+const queues = [sessionsQueue, cronQueue];
export const eventsGroupJobDuration = new client.Histogram({
name: 'job_duration_ms',
@@ -54,12 +54,8 @@ queues.forEach((queue) => {
name: `${queue.name.replace(/[{}]/g, '')}_delayed_count`,
help: 'Delayed count',
async collect() {
- if ('getDelayedCount' in queue) {
- const metric = await queue.getDelayedCount();
- this.set(metric);
- } else {
- this.set(0);
- }
+ const metric = await queue.getDelayedCount();
+ this.set(metric);
},
})
);
diff --git a/packages/queue/package.json b/packages/queue/package.json
index 989f30f48..7023f98df 100644
--- a/packages/queue/package.json
+++ b/packages/queue/package.json
@@ -11,7 +11,6 @@
"@openpanel/logger": "workspace:*",
"@openpanel/redis": "workspace:*",
"bullmq": "^5.63.0",
- "groupmq": "catalog:",
"kafkajs": "^2.2.4"
},
"devDependencies": {
diff --git a/packages/queue/src/kafka.ts b/packages/queue/src/kafka.ts
index f0a0c5870..f6f8263d0 100644
--- a/packages/queue/src/kafka.ts
+++ b/packages/queue/src/kafka.ts
@@ -78,17 +78,20 @@ const KAFKA_MIN_BYTES = KAFKA_MIN_MESSAGES * KAFKA_BYTES_PER_MESSAGE;
const KAFKA_MAX_BYTES_PER_PARTITION =
KAFKA_MAX_MESSAGES_PER_PARTITION * KAFKA_BYTES_PER_MESSAGE;
-export const isKafkaConfigured = (): boolean => KAFKA_BROKERS.length > 0;
-
-export const shouldUseKafka = (): boolean => isKafkaConfigured();
-
-let kafka: Kafka | null = null;
-const getKafka = (): Kafka => {
- if (!isKafkaConfigured()) {
+// Kafka/Redpanda is the sole events transport (ADR-004): there is no fallback,
+// so an unset KAFKA_BROKERS must fail loudly at boot rather than quietly at the
+// first event.
+export const assertKafkaConfigured = (): void => {
+ if (KAFKA_BROKERS.length === 0) {
throw new Error(
- 'KAFKA_BROKERS env var is not set; cannot create Kafka client'
+ 'KAFKA_BROKERS is not set. Kafka/Redpanda is the only events transport — set KAFKA_BROKERS to a comma-separated broker list.'
);
}
+};
+
+let kafka: Kafka | null = null;
+const getKafka = (): Kafka => {
+ assertKafkaConfigured();
if (!kafka) {
kafka = new Kafka({
clientId: process.env.KAFKA_CLIENT_ID || 'openpanel',
diff --git a/packages/queue/src/queues.ts b/packages/queue/src/queues.ts
index e0ec709a6..4a4060342 100644
--- a/packages/queue/src/queues.ts
+++ b/packages/queue/src/queues.ts
@@ -1,4 +1,3 @@
-import { createHash } from 'node:crypto';
import type {
IClickhouseSession,
IServiceCreateEventPayload,
@@ -6,26 +5,13 @@ import type {
Prisma,
} from '@openpanel/db';
import { createLogger } from '@openpanel/logger';
-import { getRedisGroupQueue, getRedisQueue } from '@openpanel/redis';
+import { getRedisQueue } from '@openpanel/redis';
import { Queue } from 'bullmq';
-import { Queue as GroupQueue } from 'groupmq';
import type { ITrackPayload } from '../../validation';
-export const EVENTS_GROUP_QUEUES_SHARDS = Number.parseInt(
- process.env.EVENTS_GROUP_QUEUES_SHARDS || '1',
- 10
-);
-
export const getQueueName = (name: string) =>
process.env.QUEUE_CLUSTER ? `{${name}}` : name;
-function pickShard(projectId: string) {
- const h = createHash('sha1').update(projectId).digest(); // 20 bytes
- // take first 4 bytes as unsigned int
- const x = h.readUInt32BE(0);
- return x % EVENTS_GROUP_QUEUES_SHARDS; // 0..n-1
-}
-
export const queueLogger = createLogger({ name: 'queue' });
// BullMQ re-emits ioredis connection errors on every Queue instance; with no
@@ -213,49 +199,6 @@ export type CronQueuePayload =
export type CronQueueType = CronQueuePayload['type'];
-const orderingDelayMs = Number.parseInt(
- process.env.ORDERING_DELAY_MS || '100',
- 10
-);
-
-const autoBatchMaxWaitMs = Number.parseInt(
- process.env.AUTO_BATCH_MAX_WAIT_MS || '0',
- 10
-);
-const autoBatchSize = Number.parseInt(process.env.AUTO_BATCH_SIZE || '0', 10);
-
-export const eventsGroupQueues = Array.from({
- length: EVENTS_GROUP_QUEUES_SHARDS,
-}).map(
- (_, index, list) =>
- new GroupQueue<EventsQueuePayloadIncomingEvent['payload']>({
- logger: process.env.NODE_ENV === 'production' ? queueLogger : undefined,
- namespace: getQueueName(
- list.length === 1 ? 'group_events' : `group_events_${index}`
- ),
- redis: getRedisGroupQueue(),
- keepCompleted: 1,
- keepFailed: 10_000,
- orderingDelayMs,
- autoBatch:
- autoBatchMaxWaitMs && autoBatchSize
- ? {
- maxWaitMs: autoBatchMaxWaitMs,
- size: autoBatchSize,
- }
- : undefined,
- })
-);
-
-export const getEventsGroupQueueShard = (groupId: string) => {
- const shard = pickShard(groupId);
- const queue = eventsGroupQueues[shard];
- if (!queue) {
- throw new Error(`Queue not found for group ${groupId}`);
- }
- return queue;
-};
-
export const sessionsQueue = guardQueue(
new Queue<SessionsQueuePayload>(getQueueName('sessions'), {
connection: getRedisQueue(),
@@ -361,7 +304,7 @@ export const cohortComputeQueue = guardQueue(
// it with a count bound to keep the completed/failed sets from growing
// unbounded during quiet periods.
removeOnComplete: { age: 3600, count: 100 },
- removeOnFail: { age: 86400, count: 100 },
+ removeOnFail: { age: 86_400, count: 100 },
},
}),
'cohortCompute'
diff --git a/packages/redis/redis.ts b/packages/redis/redis.ts
index ea66b1712..9d5474ef8 100644
--- a/packages/redis/redis.ts
+++ b/packages/redis/redis.ts
@@ -110,21 +110,6 @@ export function getRedisQueue() {
return redisQueue;
}
-let redisGroupQueue: ExtendedRedis;
-export function getRedisGroupQueue() {
- if (!redisGroupQueue) {
- // Dedicated Redis connection for GroupWorker to avoid blocking BullMQ
- redisGroupQueue = createRedisClient('redis-group-queue', REDIS_URL, {
- ...options,
- enableReadyCheck: false,
- maxRetriesPerRequest: null,
- enableOfflineQueue: true,
- });
- }
-
- return redisGroupQueue;
-}
-
export async function getLock(key: string, value: string, timeout: number) {
const lock = await getRedisCache().set(key, value, 'PX', timeout, 'NX');
return lock === 'OK';
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index 0070319e8..2d2f8bede 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -18,9 +18,6 @@ catalogs:
'@types/react-dom':
specifier: ^19.2.3
version: 19.2.3
- groupmq:
- specifier: 2.0.0-next.6
- version: 2.0.0-next.6
pino:
specifier: ^10.3.1
version: 10.3.1
@@ -194,9 +191,6 @@ importers:
fastify-zod-openapi:
specifier: ^5.6.1
- groupmq:
- specifier: 'catalog:'
- version: 2.0.0-next.6([email protected]([email protected]))
pino:
specifier: 'catalog:'
version: 10.3.1
@@ -1005,9 +999,6 @@ importers:
express:
specifier: ^4.18.2
version: 4.18.2([email protected])
- groupmq:
- specifier: 'catalog:'
- version: 2.0.0-next.6([email protected]([email protected]))
pino:
specifier: 'catalog:'
version: 10.3.1
@@ -1641,9 +1632,6 @@ importers:
bullmq:
specifier: ^5.63.0
version: 5.63.0([email protected])
- groupmq:
- specifier: 'catalog:'
- version: 2.0.0-next.6([email protected]([email protected]))
kafkajs:
specifier: ^2.2.4
version: 2.2.4
@@ -13385,12 +13373,6 @@ packages:
resolution: {integrity: sha512-5gghUc24tP9HRznNpV2+FIoq3xKkj5dTQqf4v0CpdPbFVwFkWoxOM+o+2OC9ZSvjEMTjfmG9QT+gcvggTwW1zw==}
engines: {node: '>= 10.x'}
- resolution: {integrity: sha512-6YKsoBDnNC++oRfN4Dqzv248uGaQF7ijO8oXiLmDimae7sYwr7UiwlEC7cTbklys14AxHTpk0rMuPI7bxoR/fg==}
- engines: {node: '>=18'}
- peerDependencies:
- ioredis: '>=5'
-
resolution: {integrity: sha512-pCcEwRi+TKpMlxAQObHDQ56KawURgyAf6jtIY046fJ5tIv3zDe/LEIubckAO8fj6JnAxLdmWkUfNyulQ2iKdEw==}
engines: {node: '>=14.0.0'}
@@ -34894,11 +34876,6 @@ snapshots:
- dependencies:
- cron-parser: 4.9.0
- ioredis: 5.8.2([email protected])
-
dependencies:
gaxios: 6.7.1([email protected])
diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml
index 7d3f634d5..e677c0d4b 100644
--- a/pnpm-workspace.yaml
+++ b/pnpm-workspace.yaml
@@ -80,7 +80,6 @@ catalog:
"@types/react-dom": ^19.2.3
"@types/node": ^24.7.1
typescript: ^5.9.3
- groupmq: 2.0.0-next.6
pino: ^10.3.1
pino-pretty: ^13.1.3
"@hyperdx/node-opentelemetry": ^0.10.3