OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.
add batching
Carl-Gerhard Lindesvärd81f049fa9bed
diff --git a/apps/worker/src/jobs/events.redpanda-consumer.ts b/apps/worker/src/jobs/events.redpanda-consumer.tsindex 8262435ba..616836dab 100644--- a/apps/worker/src/jobs/events.redpanda-consumer.ts+++ b/apps/worker/src/jobs/events.redpanda-consumer.ts@@ -1,6 +1,7 @@import {createRedpandaEventsConsumer,type EventsQueuePayloadIncomingEvent,+ type KafkaMessage,REDPANDA_EVENTS_TOPIC,REDPANDA_PARTITIONS_CONCURRENT,redpandaLogger,@@ -13,6 +14,11 @@ export interface RedpandaConsumerHandle {stop: () => Promise<void>;}+// Heartbeat every N messages within a per-key group. The default kafkajs+// sessionTimeout is 30s — calling heartbeat every 16 messages keeps us+// comfortably under that even for slow handlers.+const HEARTBEAT_EVERY = 16;+export async function startRedpandaEventsConsumer(): Promise<RedpandaConsumerHandle> {const consumer = createRedpandaEventsConsumer();await consumer.connect();@@ -25,40 +31,85 @@ export async function startRedpandaEventsConsumer(): Promise<RedpandaConsumerHanawait consumer.run({partitionsConsumedConcurrently: REDPANDA_PARTITIONS_CONCURRENT,- eachMessage: async ({ message, partition }) => {- if (!message.value) {- return;- }- let payload: EventsQueuePayloadIncomingEvent['payload'];- try {- payload = JSON.parse(- message.value.toString()- ) as EventsQueuePayloadIncomingEvent['payload'];- } catch (err) {- logger.error(- { err, partition, offset: message.offset },- 'redpanda message parse failed'- );+ eachBatchAutoResolve: false,+ eachBatch: async ({+ batch,+ resolveOffset,+ heartbeat,+ isRunning,+ isStale,+ }) => {+ if (batch.messages.length === 0) {return;}- try {- await incomingEvent(payload);- } catch (err) {- // Match the GroupWorker behaviour: log and ack. At-most-once on- // handler exceptions; failures here would otherwise block the- // partition. Future hardening can switch to retry/DLQ.- logger.error(- {- err,- partition,- offset: message.offset,- projectId: payload.projectId,- },- 'redpanda incomingEvent handler failed'- );- } finally {- markEventsActivity();++ // Group by partition key (= deviceId or `${projectId}:${profileId}`).+ // Same-key messages stay serial so sessionBuffer/session-end-job state+ // can't race; different keys run in parallel via Promise.all.+ // Keyless messages get their own singleton group.+ const groups = new Map<string, KafkaMessage[]>();+ for (const m of batch.messages) {+ const key = m.key ? m.key.toString() : `__no_key__:${m.offset}`;+ const arr = groups.get(key);+ if (arr) {+ arr.push(m);+ } else {+ groups.set(key, [m]);+ }}++ await Promise.all(+ [...groups.values()].map(async (msgs) => {+ let processed = 0;+ for (const m of msgs) {+ if (!isRunning() || isStale()) {+ return;+ }++ if (m.value) {+ let payload: EventsQueuePayloadIncomingEvent['payload'] | null =+ null;+ try {+ payload = JSON.parse(+ m.value.toString()+ ) as EventsQueuePayloadIncomingEvent['payload'];+ } catch (err) {+ logger.error(+ { err, partition: batch.partition, offset: m.offset },+ 'redpanda message parse failed'+ );+ }+ if (payload) {+ try {+ await incomingEvent(payload);+ } catch (err) {+ // Match the previous eachMessage behaviour: log and ack.+ // At-most-once on handler exceptions; failures here would+ // otherwise block the partition.+ logger.error(+ {+ err,+ partition: batch.partition,+ offset: m.offset,+ projectId: payload.projectId,+ },+ 'redpanda incomingEvent handler failed'+ );+ }+ }+ }++ resolveOffset(m.offset);+ processed += 1;+ if (processed % HEARTBEAT_EVERY === 0) {+ await heartbeat();+ }+ }+ })+ );++ await heartbeat();+ markEventsActivity();},});diff --git a/packages/queue/src/redpanda.ts b/packages/queue/src/redpanda.tsindex e7d54f322..58cd0596d 100644--- a/packages/queue/src/redpanda.ts+++ b/packages/queue/src/redpanda.ts@@ -2,6 +2,8 @@ import { createLogger } from '@openpanel/logger';import { type Consumer, Kafka, logLevel, type Producer } from 'kafkajs';import type { EventsQueuePayloadIncomingEvent } from './queues';+export type { KafkaMessage } from 'kafkajs';+export const redpandaLogger = createLogger({ name: 'redpanda' });const parseBrokers = (raw: string | undefined): string[] => {@@ -23,6 +25,35 @@ export const REDPANDA_PARTITIONS_CONCURRENT = Number.parseInt(process.env.REDPANDA_PARTITIONS_CONCURRENT || '8',10);+// Approx size of one event payload (observed range ~0.9–1.3 KiB).+// We size fetch knobs in messages and convert to bytes via this constant.+const REDPANDA_BYTES_PER_MESSAGE = 1024;++export const REDPANDA_MIN_MESSAGES = Number.parseInt(+ process.env.REDPANDA_MIN_MESSAGES || '1',+ 10+);+export const REDPANDA_MAX_WAIT_MS = Number.parseInt(+ process.env.REDPANDA_MAX_WAIT_MS || '500',+ 10+);+export const REDPANDA_MAX_MESSAGES_PER_PARTITION = Number.parseInt(+ process.env.REDPANDA_MAX_MESSAGES_PER_PARTITION || '256',+ 10+);+export const REDPANDA_SESSION_TIMEOUT_MS = Number.parseInt(+ process.env.REDPANDA_SESSION_TIMEOUT_MS || '30000',+ 10+);+export const REDPANDA_HEARTBEAT_INTERVAL_MS = Number.parseInt(+ process.env.REDPANDA_HEARTBEAT_INTERVAL_MS || '3000',+ 10+);++const REDPANDA_MIN_BYTES =+ REDPANDA_MIN_MESSAGES * REDPANDA_BYTES_PER_MESSAGE;+const REDPANDA_MAX_BYTES_PER_PARTITION =+ REDPANDA_MAX_MESSAGES_PER_PARTITION * REDPANDA_BYTES_PER_MESSAGE;const projectIdsEnv = (process.env.REDPANDA_PROJECT_IDS || '').trim();const allowAllProjects = projectIdsEnv === '*';@@ -120,6 +151,11 @@ export const createRedpandaEventsConsumer = (options?: {const client = getKafka();const consumer = client.consumer({groupId: options?.groupId || REDPANDA_CONSUMER_GROUP,+ sessionTimeout: REDPANDA_SESSION_TIMEOUT_MS,+ heartbeatInterval: REDPANDA_HEARTBEAT_INTERVAL_MS,+ minBytes: REDPANDA_MIN_BYTES,+ maxWaitTimeInMs: REDPANDA_MAX_WAIT_MS,+ maxBytesPerPartition: REDPANDA_MAX_BYTES_PER_PARTITION,});consumers.add(consumer);return consumer;