OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.
ralph(P1-001): Delete the GroupMQ transport: /track always produces to Kafka
ralph9d77b0df28f5
diff --git a/apps/api/package.json b/apps/api/package.jsonindex 7ed09ec36..c9e2ffd3b 100644--- a/apps/api/package.json+++ b/apps/api/package.json@@ -49,7 +49,6 @@"fastify-metrics": "^12.1.0","fastify-raw-body": "^5.0.0","fastify-zod-openapi": "^5.6.1",- "groupmq": "catalog:","pino": "catalog:","pino-pretty": "catalog:","ramda": "^0.29.1",diff --git a/apps/api/src/controllers/event.controller.ts b/apps/api/src/controllers/event.controller.tsindex a83ed359a..2fa2902aa 100644--- a/apps/api/src/controllers/event.controller.ts+++ b/apps/api/src/controllers/event.controller.ts@@ -4,9 +4,7 @@ import { getSalts } from '@openpanel/db';import { getAsnInfo, getGeoLocation } from '@openpanel/geo';import {type EventsQueuePayloadIncomingEvent,- getEventsGroupQueueShard,produceIncomingEvent,- shouldUseKafka,} from '@openpanel/queue';import type { DeprecatedPostEventPayload } from '@openpanel/validation';import type { FastifyReply, FastifyRequest } from 'fastify';@@ -74,15 +72,7 @@ export async function postEvent(sessionId: sessionId ?? '',};- if (shouldUseKafka()) {- await produceIncomingEvent(queueData, groupId);- } else {- await getEventsGroupQueueShard(groupId).add({- orderMs: new Date(timestamp).getTime(),- data: queueData,- groupId,- });- }+ await produceIncomingEvent(queueData, groupId);reply.status(202).send('ok');}diff --git a/apps/api/src/controllers/track.controller.test.ts b/apps/api/src/controllers/track.controller.test.tsindex 5632c63ec..adbcfe37a 100644--- a/apps/api/src/controllers/track.controller.test.ts+++ b/apps/api/src/controllers/track.controller.test.ts@@ -5,19 +5,50 @@* device_id, so it must be trimmed, non-empty, and length-bounded).* - handleReplay — replay files a chunk under the session id the SDK echoes back;* it needs no device resolution and trusts the client-sent session id.+ * - handler — the single events transport (ADR-004): every tracked event goes to+ * Kafka via produceIncomingEvent, with no branch and no fallback queue.+ *+ * `@openpanel/db` and `@openpanel/queue` are partially mocked (importActual ++ * overrides) rather than replaced: they import each other, and the replay test+ * below spies on the real `replayBuffer`.*/-import { replayBuffer } from '@openpanel/db';+import {+ getSalts,+ replayBuffer,+ sessionBuffer,+ upsertProfile,+} from '@openpanel/db';+import { getAsnInfo, getGeoLocation } from '@openpanel/geo';+import { produceIncomingEvent } from '@openpanel/queue';import type {IReplayPayload,ITrackHandlerPayload,} from '@openpanel/validation';+import type { FastifyReply, FastifyRequest } from 'fastify';import { beforeEach, describe, expect, it, vi } from 'vitest';-import { getOverrideDeviceId, handleReplay } from './track.controller';+import { getOverrideDeviceId, handleReplay, handler } from './track.controller';-const track = (- properties?: Record<string, unknown>-): ITrackHandlerPayload =>+vi.mock('@openpanel/queue', async () => {+ const actual =+ await vi.importActual<typeof import('@openpanel/queue')>(+ '@openpanel/queue'+ );+ return { ...actual, produceIncomingEvent: vi.fn() };+});++vi.mock('@openpanel/geo', () => ({+ getGeoLocation: vi.fn(),+ getAsnInfo: vi.fn(),+}));++vi.mock('@openpanel/db', async () => {+ const actual =+ await vi.importActual<typeof import('@openpanel/db')>('@openpanel/db');+ return { ...actual, getSalts: vi.fn(), upsertProfile: vi.fn() };+});++const track = (properties?: Record<string, unknown>): ITrackHandlerPayload =>({type: 'track',payload: { name: 'page_view', properties },@@ -106,3 +137,113 @@ describe('handleReplay', () => {expect(add).not.toHaveBeenCalled();});});++const PROJECT_ID = 'proj-kafka';+const DEVICE_ID = 'device-kafka-1';++const makeRequest = (body: ITrackHandlerPayload) =>+ ({+ body,+ client: { projectId: PROJECT_ID },+ clientIp: '1.2.3.4',+ clientSecretAuth: false,+ timestamp: new Date('2026-06-08T12:00:00.000Z').getTime(),+ headers: { 'user-agent': 'Mozilla/5.0 (Macintosh) Chrome/148.0.0.0' },+ }) as unknown as FastifyRequest<{ Body: ITrackHandlerPayload }>;++const makeReply = () => {+ const reply = {+ statusCode: 0,+ body: undefined as unknown,+ status(code: number) {+ reply.statusCode = code;+ return reply;+ },+ send(payload: unknown) {+ reply.body = payload;+ return reply;+ },+ };+ return reply as unknown as FastifyReply & typeof reply;+};++describe('handler — events transport', () => {+ beforeEach(() => {+ vi.clearAllMocks();+ vi.mocked(getSalts).mockResolvedValue({+ current: 'salt-current',+ previous: 'salt-previous',+ });+ vi.mocked(getGeoLocation).mockResolvedValue({+ country: 'SE',+ city: 'Stockholm',+ region: 'AB',+ longitude: 18,+ latitude: 59,+ });+ vi.mocked(getAsnInfo).mockResolvedValue({+ asn: undefined,+ org: undefined,+ isHosting: false,+ });+ vi.mocked(produceIncomingEvent).mockResolvedValue(undefined);+ vi.spyOn(sessionBuffer, 'getExistingSession').mockResolvedValue(null);+ });++ it('produces every tracked event to Kafka — the only transport', async () => {+ await handler(+ makeRequest({+ type: 'track',+ payload: {+ name: 'page_view',+ properties: { __deviceId: DEVICE_ID },+ },+ } as ITrackHandlerPayload),+ makeReply()+ );++ expect(produceIncomingEvent).toHaveBeenCalledTimes(1);+ const [queueData, partitionKey] =+ vi.mocked(produceIncomingEvent).mock.calls[0]!;+ expect(queueData).toMatchObject({+ projectId: PROJECT_ID,+ deviceId: DEVICE_ID,+ event: { name: 'page_view' },+ });+ // Browser events partition on the device id, which is what preserves+ // per-device mutual exclusion in the consumer.+ expect(partitionKey).toBe(DEVICE_ID);+ });++ it('surfaces a produce failure instead of falling back to another queue', async () => {+ vi.mocked(produceIncomingEvent).mockRejectedValue(+ new Error('kafka unavailable')+ );++ await expect(+ handler(+ makeRequest({+ type: 'track',+ payload: { name: 'page_view', properties: { __deviceId: DEVICE_ID } },+ } as ITrackHandlerPayload),+ makeReply()+ )+ ).rejects.toThrow('kafka unavailable');++ expect(produceIncomingEvent).toHaveBeenCalledTimes(1);+ });++ it('does not produce for non-event payload types', async () => {+ vi.mocked(upsertProfile).mockResolvedValue(undefined as never);++ await handler(+ makeRequest({+ type: 'identify',+ payload: { profileId: 'p1', properties: {} },+ } as ITrackHandlerPayload),+ makeReply()+ );++ expect(produceIncomingEvent).not.toHaveBeenCalled();+ });+});diff --git a/apps/api/src/controllers/track.controller.ts b/apps/api/src/controllers/track.controller.tsindex 3452de4f8..a79bc4ae1 100644--- a/apps/api/src/controllers/track.controller.ts+++ b/apps/api/src/controllers/track.controller.ts@@ -18,9 +18,7 @@ import {} from '@openpanel/geo';import {type EventsQueuePayloadIncomingEvent,- getEventsGroupQueueShard,produceIncomingEvent,- shouldUseKafka,} from '@openpanel/queue';import type {IAssignGroupPayload,@@ -268,17 +266,7 @@ async function handleTrack(const partitionKey = groupId || generateId();- if (shouldUseKafka()) {- promises.push(produceIncomingEvent(queueData, partitionKey));- } else {- promises.push(- getEventsGroupQueueShard(partitionKey).add({- orderMs: timestamp.value,- data: queueData,- groupId,- })- );- }+ promises.push(produceIncomingEvent(queueData, partitionKey));await Promise.all(promises);}diff --git a/apps/api/src/index.ts b/apps/api/src/index.tsindex 903e1401a..a071d9b56 100644--- a/apps/api/src/index.ts+++ b/apps/api/src/index.ts@@ -4,14 +4,24 @@ process.env.TZ = 'UTC';import './utils/observability';import { rawStderrWrite } from '@openpanel/logger';+import { assertKafkaConfigured } from '@openpanel/queue';+import { getRedisPub } from '@openpanel/redis';import sourceMapSupport from 'source-map-support';import { buildApp } from './app';import { shutdown } from './utils/graceful-shutdown';import { logger } from './utils/logger';-import { getRedisPub } from '@openpanel/redis';sourceMapSupport.install();+// Kafka/Redpanda is the sole events transport (ADR-004) — a missing broker list+// must kill boot, not surface as a 5xx on the first tracked event.+try {+ assertKafkaConfigured();+} catch (error) {+ logger.fatal({ err: error }, 'Refusing to start');+ process.exit(1);+}+const port = Number.parseInt(process.env.API_PORT || '3000', 10);const host =process.env.API_HOST ||diff --git a/apps/api/src/utils/graceful-shutdown.ts b/apps/api/src/utils/graceful-shutdown.tsindex c16626521..f95592499 100644--- a/apps/api/src/utils/graceful-shutdown.ts+++ b/apps/api/src/utils/graceful-shutdown.ts@@ -2,7 +2,6 @@ import { ch, db } from '@openpanel/db';import {cronQueue,disconnectKafka,- eventsGroupQueues,notificationQueue,sessionsQueue,} from '@openpanel/queue';@@ -94,7 +93,6 @@ export async function shutdown(// Step 6: Close Bull queues (graceful shutdown of queue state)try {await Promise.all([- ...eventsGroupQueues.map((queue) => queue.close()),sessionsQueue.close(),cronQueue.close(),notificationQueue.close(),diff --git a/apps/worker/package.json b/apps/worker/package.jsonindex 7cd99969e..e24a8a299 100644--- a/apps/worker/package.json+++ b/apps/worker/package.json@@ -29,7 +29,6 @@"bullmq": "^5.63.0","date-fns": "^3.3.1","express": "^4.18.2",- "groupmq": "catalog:","pino": "catalog:","pino-pretty": "catalog:","prom-client": "^15.1.3",diff --git a/apps/worker/src/boot-workers.ts b/apps/worker/src/boot-workers.tsindex e2a6e1968..161af456c 100644--- a/apps/worker/src/boot-workers.ts+++ b/apps/worker/src/boot-workers.ts@@ -4,24 +4,17 @@ import { rawStderrWrite } from '@openpanel/logger';import {cohortComputeQueue,cronQueue,- EVENTS_GROUP_QUEUES_SHARDS,- type EventsQueuePayloadIncomingEvent,- eventsGroupQueues,gscQueue,importQueue,insightsQueue,- isKafkaConfigured,notificationQueue,- queueLogger,sessionsQueue,} from '@openpanel/queue';import { getRedisQueue } from '@openpanel/redis';import type { Queue, WorkerOptions } from 'bullmq';import { Worker } from 'bullmq';-import { Worker as GroupWorker } from 'groupmq';import { cohortComputeJob } from './jobs/cohort.compute';import { cronJob } from './jobs/cron';-import { incomingEvent } from './jobs/events.incoming-event';import {type KafkaConsumerHandle,startKafkaEventsConsumer,@@ -34,36 +27,24 @@ import { sessionsJob } from './jobs/sessions';import { eventsGroupJobDuration } from './metrics';import { setShuttingDown } from './utils/graceful-shutdown';import { logger } from './utils/logger';-import {- enableEventsHeartbeat,- markEventsActivity,-} from './utils/worker-heartbeat';+import { enableEventsHeartbeat } from './utils/worker-heartbeat';const workerOptions: WorkerOptions = {connection: getRedisQueue(),};-type QueueName = string; // Can be: events, events_N (where N is 0 to shards-1), sessions, cron, notification+type QueueName = string; // Can be: events_kafka, sessions, cron, notification, import, insights, gsc, cohortCompute/*** Parses the ENABLED_QUEUES environment variable and returns an array of queue names to start.* If no env var is provided, returns all queues.- *- * Supported queue names:- * - events - All event shards (events_0, events_1, ..., events_N)- * - events_N - Individual event shard (where N is 0 to EVENTS_GROUP_QUEUES_SHARDS-1)- * - sessions, cron, notification*/function getEnabledQueues(): QueueName[] {const enabledQueuesEnv = process.env.ENABLED_QUEUES?.trim();if (!enabledQueuesEnv) {- logger.info(- { totalEventShards: EVENTS_GROUP_QUEUES_SHARDS },- 'No ENABLED_QUEUES specified, starting all queues'- );+ logger.info('No ENABLED_QUEUES specified, starting all queues');return [- 'events','events_kafka','sessions','cron',@@ -80,16 +61,13 @@ function getEnabledQueues(): QueueName[] {.map((q) => q.trim()).filter(Boolean);- logger.info(- { queues, totalEventShards: EVENTS_GROUP_QUEUES_SHARDS },- 'Starting queues from ENABLED_QUEUES'- );+ logger.info({ queues }, 'Starting queues from ENABLED_QUEUES');return queues;}/*** Gets the concurrency setting for a queue from environment variables.- * Env var format: {QUEUE_NAME}_CONCURRENCY (e.g., EVENTS_0_CONCURRENCY=32)+ * Env var format: {QUEUE_NAME}_CONCURRENCY (e.g., SESSIONS_CONCURRENCY=32)*/function getConcurrencyFor(queueName: string, defaultValue = 1): number {const envKey = `${queueName.toUpperCase().replace(/[^A-Z0-9]/g, '_')}_CONCURRENCY`;@@ -108,84 +86,11 @@ function getConcurrencyFor(queueName: string, defaultValue = 1): number {export function bootWorkers() {const enabledQueues = getEnabledQueues();- const workers: (Worker | GroupWorker<any>)[] = [];+ const workers: Worker[] = [];const extraStops: Array<() => Promise<unknown>> = [];- // Start event workers based on enabled queues.- // When Kafka is configured the producer routes every event to Kafka, so the- // GroupMQ event shards would only poll an empty queue — skip them entirely- // and let the Kafka consumer handle ingestion.- const eventQueuesToStart: number[] = [];-- if (isKafkaConfigured()) {- logger.info('Kafka is configured, skipping GroupMQ event workers');- } else if (enabledQueues.includes('events')) {- // Start all event shards- for (let i = 0; i < EVENTS_GROUP_QUEUES_SHARDS; i++) {- eventQueuesToStart.push(i);- }- } else {- // Start specific event shards (events_0, events_1, etc.)- for (let i = 0; i < EVENTS_GROUP_QUEUES_SHARDS; i++) {- if (enabledQueues.includes(`events_${i}`)) {- eventQueuesToStart.push(i);- }- }- }-- if (eventQueuesToStart.length > 0) {- enableEventsHeartbeat();- }-- for (const index of eventQueuesToStart) {- const queue = eventsGroupQueues[index];- if (!queue) {- continue;- }-- const queueName = `events_${index}`;- const concurrency = getConcurrencyFor(- queueName,- Number.parseInt(process.env.EVENT_JOB_CONCURRENCY || '10', 10)- );-- const worker = new GroupWorker<EventsQueuePayloadIncomingEvent['payload']>({- queue,- concurrency,- logger: process.env.NODE_ENV === 'production' ? queueLogger : undefined,- blockingTimeoutSec: Number.parseFloat(- process.env.EVENT_BLOCKING_TIMEOUT_SEC || '1'- ),- handler: async (job) => {- return await incomingEvent(job.data);- },- });-- // Consumer-loop heartbeat for the readiness probe. `completed` fires after- // each processed job; `drained` fires on each poll cycle that finds the- // queue empty. Together they refresh the timestamp every poll cycle while- // the consumer is alive — busy or idle.- worker.on('completed', markEventsActivity);- worker.on('drained', markEventsActivity);-- // Fail loud on startup — silent stuck shard otherwise.- // Runtime errors are handled by the shared workers.forEach listener below.- worker.run().catch((err) => {- logger.error(- { shard: index, queueName, err },- 'Worker startup failed — exiting',- );- // setTimeout+unref to let the logger flush before exit (matches the- // pattern used by uncaughtException/unhandledRejection handlers below).- setTimeout(() => process.exit(1), 1000).unref();- });- workers.push(worker);- logger.info({ concurrency }, `Started worker for ${queueName}`);- }-- // Start Kafka events consumer. When Kafka is configured this fully replaces- // the GroupMQ event workers (which are skipped above).- if (enabledQueues.includes('events_kafka') && isKafkaConfigured()) {+ // Start Kafka events consumer — the sole events transport (ADR-004).+ if (enabledQueues.includes('events_kafka')) {enableEventsHeartbeat();let handle: KafkaConsumerHandle | null = null;const startPromise = startKafkaEventsConsumer()@@ -293,19 +198,19 @@ export function bootWorkers() {}workers.forEach((worker) => {- (worker as Worker).on('error', (error) => {+ worker.on('error', (error) => {logger.error({ err: error, worker: worker.name }, 'worker error');});- (worker as Worker).on('closed', () => {+ worker.on('closed', () => {logger.info({ worker: worker.name }, 'worker closed');});- (worker as Worker).on('ready', () => {+ worker.on('ready', () => {logger.info({ worker: worker.name }, 'worker ready');});- (worker as Worker).on('failed', (job) => {+ worker.on('failed', (job) => {if (job) {if (job.processedOn && job.finishedOn) {const elapsed = job.finishedOn - job.processedOn;@@ -327,7 +232,7 @@ export function bootWorkers() {}});- (worker as Worker).on('ioredis:close', () => {+ worker.on('ioredis:close', () => {logger.error({ worker: worker.name },'worker closed due to ioredis:close'@@ -348,9 +253,7 @@ export function bootWorkers() {// before Docker's stop_grace_period elapses. Without this the// container sits in "Stopping" until SIGKILL (exit 137) and looks// like a real crash to the swarm.- const forceExitMs = Number(- process.env.SHUTDOWN_FORCE_EXIT_MS || '20000'- );+ const forceExitMs = Number(process.env.SHUTDOWN_FORCE_EXIT_MS || '20000');const exitCode = Number.isNaN(+evtOrExitCodeOrError)? 1: +evtOrExitCodeOrError;@@ -418,7 +321,7 @@ export function bootWorkers() {rawStderrWrite(`Unhandled rejection — exiting: ${reason instanceof Error ? reason.stack : String(reason)- }\n`,+ }\n`);setShuttingDown(true);setTimeout(() => process.exit(1), 1000).unref();diff --git a/apps/worker/src/index.ts b/apps/worker/src/index.tsindex 0a38bffa2..e4c879f02 100644--- a/apps/worker/src/index.ts+++ b/apps/worker/src/index.ts@@ -6,9 +6,9 @@ import { ExpressAdapter } from '@bull-board/express';import { tryCatch } from '@openpanel/common';import { chQuery, createInitialSalts, db } from '@openpanel/db';import {+ assertKafkaConfigured,cohortComputeQueue,cronQueue,- eventsGroupQueues,gscQueue,importQueue,insightsQueue,@@ -17,7 +17,6 @@ import {} from '@openpanel/queue';import { getRedisCache } from '@openpanel/redis';import express from 'express';-import { BullBoardGroupMQAdapter } from 'groupmq';import client from 'prom-client';import sourceMapSupport from 'source-map-support';import { bootCron } from './boot-cron';@@ -32,6 +31,15 @@ const EVENTS_HEARTBEAT_STALE_MS = 60_000;sourceMapSupport.install();+// Kafka/Redpanda is the sole events transport (ADR-004) — a missing broker list+// must kill boot, not surface as a failing consumer nobody notices.+try {+ assertKafkaConfigured();+} catch (error) {+ logger.fatal({ err: error }, 'Refusing to start');+ process.exit(1);+}+async function start() {const collectDefaultMetrics = client.collectDefaultMetrics;collectDefaultMetrics({ register });@@ -53,9 +61,6 @@ async function start() {serverAdapter.setBasePath('/');createBullBoard({queues: [- ...eventsGroupQueues.map(- (queue) => new BullBoardGroupMQAdapter(queue) as any- ),new BullMQAdapter(sessionsQueue),new BullMQAdapter(cronQueue),new BullMQAdapter(notificationQueue),diff --git a/apps/worker/src/metrics.ts b/apps/worker/src/metrics.tsindex 2f595e8a0..c48632127 100644--- a/apps/worker/src/metrics.ts+++ b/apps/worker/src/metrics.ts@@ -7,7 +7,7 @@ import {replayBuffer,sessionBuffer,} from '@openpanel/db';-import { cronQueue, eventsGroupQueues, sessionsQueue } from '@openpanel/queue';+import { cronQueue, sessionsQueue } from '@openpanel/queue';import { getRedisCache } from '@openpanel/redis';import client from 'prom-client';@@ -15,7 +15,7 @@ const Registry = client.Registry;export const register = new Registry();-const queues = [sessionsQueue, cronQueue, ...eventsGroupQueues];+const queues = [sessionsQueue, cronQueue];export const eventsGroupJobDuration = new client.Histogram({name: 'job_duration_ms',@@ -54,12 +54,8 @@ queues.forEach((queue) => {name: `${queue.name.replace(/[{}]/g, '')}_delayed_count`,help: 'Delayed count',async collect() {- if ('getDelayedCount' in queue) {- const metric = await queue.getDelayedCount();- this.set(metric);- } else {- this.set(0);- }+ const metric = await queue.getDelayedCount();+ this.set(metric);},}));diff --git a/packages/queue/package.json b/packages/queue/package.jsonindex 989f30f48..7023f98df 100644--- a/packages/queue/package.json+++ b/packages/queue/package.json@@ -11,7 +11,6 @@"@openpanel/logger": "workspace:*","@openpanel/redis": "workspace:*","bullmq": "^5.63.0",- "groupmq": "catalog:","kafkajs": "^2.2.4"},"devDependencies": {diff --git a/packages/queue/src/kafka.ts b/packages/queue/src/kafka.tsindex f0a0c5870..f6f8263d0 100644--- a/packages/queue/src/kafka.ts+++ b/packages/queue/src/kafka.ts@@ -78,17 +78,20 @@ const KAFKA_MIN_BYTES = KAFKA_MIN_MESSAGES * KAFKA_BYTES_PER_MESSAGE;const KAFKA_MAX_BYTES_PER_PARTITION =KAFKA_MAX_MESSAGES_PER_PARTITION * KAFKA_BYTES_PER_MESSAGE;-export const isKafkaConfigured = (): boolean => KAFKA_BROKERS.length > 0;--export const shouldUseKafka = (): boolean => isKafkaConfigured();--let kafka: Kafka | null = null;-const getKafka = (): Kafka => {- if (!isKafkaConfigured()) {+// Kafka/Redpanda is the sole events transport (ADR-004): there is no fallback,+// so an unset KAFKA_BROKERS must fail loudly at boot rather than quietly at the+// first event.+export const assertKafkaConfigured = (): void => {+ if (KAFKA_BROKERS.length === 0) {throw new Error(- 'KAFKA_BROKERS env var is not set; cannot create Kafka client'+ 'KAFKA_BROKERS is not set. Kafka/Redpanda is the only events transport — set KAFKA_BROKERS to a comma-separated broker list.');}+};++let kafka: Kafka | null = null;+const getKafka = (): Kafka => {+ assertKafkaConfigured();if (!kafka) {kafka = new Kafka({clientId: process.env.KAFKA_CLIENT_ID || 'openpanel',diff --git a/packages/queue/src/queues.ts b/packages/queue/src/queues.tsindex e0ec709a6..4a4060342 100644--- a/packages/queue/src/queues.ts+++ b/packages/queue/src/queues.ts@@ -1,4 +1,3 @@-import { createHash } from 'node:crypto';import type {IClickhouseSession,IServiceCreateEventPayload,@@ -6,26 +5,13 @@ import type {Prisma,} from '@openpanel/db';import { createLogger } from '@openpanel/logger';-import { getRedisGroupQueue, getRedisQueue } from '@openpanel/redis';+import { getRedisQueue } from '@openpanel/redis';import { Queue } from 'bullmq';-import { Queue as GroupQueue } from 'groupmq';import type { ITrackPayload } from '../../validation';-export const EVENTS_GROUP_QUEUES_SHARDS = Number.parseInt(- process.env.EVENTS_GROUP_QUEUES_SHARDS || '1',- 10-);-export const getQueueName = (name: string) =>process.env.QUEUE_CLUSTER ? `{${name}}` : name;-function pickShard(projectId: string) {- const h = createHash('sha1').update(projectId).digest(); // 20 bytes- // take first 4 bytes as unsigned int- const x = h.readUInt32BE(0);- return x % EVENTS_GROUP_QUEUES_SHARDS; // 0..n-1-}-export const queueLogger = createLogger({ name: 'queue' });// BullMQ re-emits ioredis connection errors on every Queue instance; with no@@ -213,49 +199,6 @@ export type CronQueuePayload =export type CronQueueType = CronQueuePayload['type'];-const orderingDelayMs = Number.parseInt(- process.env.ORDERING_DELAY_MS || '100',- 10-);--const autoBatchMaxWaitMs = Number.parseInt(- process.env.AUTO_BATCH_MAX_WAIT_MS || '0',- 10-);-const autoBatchSize = Number.parseInt(process.env.AUTO_BATCH_SIZE || '0', 10);--export const eventsGroupQueues = Array.from({- length: EVENTS_GROUP_QUEUES_SHARDS,-}).map(- (_, index, list) =>- new GroupQueue<EventsQueuePayloadIncomingEvent['payload']>({- logger: process.env.NODE_ENV === 'production' ? queueLogger : undefined,- namespace: getQueueName(- list.length === 1 ? 'group_events' : `group_events_${index}`- ),- redis: getRedisGroupQueue(),- keepCompleted: 1,- keepFailed: 10_000,- orderingDelayMs,- autoBatch:- autoBatchMaxWaitMs && autoBatchSize- ? {- maxWaitMs: autoBatchMaxWaitMs,- size: autoBatchSize,- }- : undefined,- })-);--export const getEventsGroupQueueShard = (groupId: string) => {- const shard = pickShard(groupId);- const queue = eventsGroupQueues[shard];- if (!queue) {- throw new Error(`Queue not found for group ${groupId}`);- }- return queue;-};-export const sessionsQueue = guardQueue(new Queue<SessionsQueuePayload>(getQueueName('sessions'), {connection: getRedisQueue(),@@ -361,7 +304,7 @@ export const cohortComputeQueue = guardQueue(// it with a count bound to keep the completed/failed sets from growing// unbounded during quiet periods.removeOnComplete: { age: 3600, count: 100 },- removeOnFail: { age: 86400, count: 100 },+ removeOnFail: { age: 86_400, count: 100 },},}),'cohortCompute'diff --git a/packages/redis/redis.ts b/packages/redis/redis.tsindex ea66b1712..9d5474ef8 100644--- a/packages/redis/redis.ts+++ b/packages/redis/redis.ts@@ -110,21 +110,6 @@ export function getRedisQueue() {return redisQueue;}-let redisGroupQueue: ExtendedRedis;-export function getRedisGroupQueue() {- if (!redisGroupQueue) {- // Dedicated Redis connection for GroupWorker to avoid blocking BullMQ- redisGroupQueue = createRedisClient('redis-group-queue', REDIS_URL, {- ...options,- enableReadyCheck: false,- maxRetriesPerRequest: null,- enableOfflineQueue: true,- });- }-- return redisGroupQueue;-}-export async function getLock(key: string, value: string, timeout: number) {const lock = await getRedisCache().set(key, value, 'PX', timeout, 'NX');return lock === 'OK';diff --git a/pnpm-lock.yaml b/pnpm-lock.yamlindex 0070319e8..2d2f8bede 100644--- a/pnpm-lock.yaml+++ b/pnpm-lock.yaml@@ -18,9 +18,6 @@ catalogs:'@types/react-dom':specifier: ^19.2.3version: 19.2.3- groupmq:- specifier: 2.0.0-next.6- version: 2.0.0-next.6pino:specifier: ^10.3.1version: 10.3.1@@ -194,9 +191,6 @@ importers:fastify-zod-openapi:specifier: ^5.6.1version: 5.6.1(@fastify/[email protected])(@fastify/[email protected]([email protected]))([email protected])([email protected])- groupmq:- specifier: 'catalog:'- version: 2.0.0-next.6([email protected]([email protected]))pino:specifier: 'catalog:'version: 10.3.1@@ -1005,9 +999,6 @@ importers:express:specifier: ^4.18.2version: 4.18.2([email protected])- groupmq:- specifier: 'catalog:'- version: 2.0.0-next.6([email protected]([email protected]))pino:specifier: 'catalog:'version: 10.3.1@@ -1641,9 +1632,6 @@ importers:bullmq:specifier: ^5.63.0version: 5.63.0([email protected])- groupmq:- specifier: 'catalog:'- version: 2.0.0-next.6([email protected]([email protected]))kafkajs:specifier: ^2.2.4version: 2.2.4@@ -13385,12 +13373,6 @@ packages:resolution: {integrity: sha512-5gghUc24tP9HRznNpV2+FIoq3xKkj5dTQqf4v0CpdPbFVwFkWoxOM+o+2OC9ZSvjEMTjfmG9QT+gcvggTwW1zw==}engines: {node: '>= 10.x'}- resolution: {integrity: sha512-6YKsoBDnNC++oRfN4Dqzv248uGaQF7ijO8oXiLmDimae7sYwr7UiwlEC7cTbklys14AxHTpk0rMuPI7bxoR/fg==}- engines: {node: '>=18'}- peerDependencies:- ioredis: '>=5'-resolution: {integrity: sha512-pCcEwRi+TKpMlxAQObHDQ56KawURgyAf6jtIY046fJ5tIv3zDe/LEIubckAO8fj6JnAxLdmWkUfNyulQ2iKdEw==}engines: {node: '>=14.0.0'}@@ -34894,11 +34876,6 @@ snapshots:- dependencies:- cron-parser: 4.9.0- ioredis: 5.8.2([email protected])-dependencies:gaxios: 6.7.1([email protected])diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yamlindex 7d3f634d5..e677c0d4b 100644--- a/pnpm-workspace.yaml+++ b/pnpm-workspace.yaml@@ -80,7 +80,6 @@ catalog:"@types/react-dom": ^19.2.3"@types/node": ^24.7.1typescript: ^5.9.3- groupmq: 2.0.0-next.6pino: ^10.3.1pino-pretty: ^13.1.3"@hyperdx/node-opentelemetry": ^0.10.3