Skip to content

OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.

add batching

Carl-Gerhard Lindesvärd81f049fa9bed
diff --git a/apps/worker/src/jobs/events.redpanda-consumer.ts b/apps/worker/src/jobs/events.redpanda-consumer.ts
index 8262435ba..616836dab 100644
--- a/apps/worker/src/jobs/events.redpanda-consumer.ts
+++ b/apps/worker/src/jobs/events.redpanda-consumer.ts
@@ -1,6 +1,7 @@
import {
createRedpandaEventsConsumer,
type EventsQueuePayloadIncomingEvent,
+ type KafkaMessage,
REDPANDA_EVENTS_TOPIC,
REDPANDA_PARTITIONS_CONCURRENT,
redpandaLogger,
@@ -13,6 +14,11 @@ export interface RedpandaConsumerHandle {
stop: () => Promise<void>;
}
+// Heartbeat every N messages within a per-key group. The default kafkajs
+// sessionTimeout is 30s — calling heartbeat every 16 messages keeps us
+// comfortably under that even for slow handlers.
+const HEARTBEAT_EVERY = 16;
+
export async function startRedpandaEventsConsumer(): Promise<RedpandaConsumerHandle> {
const consumer = createRedpandaEventsConsumer();
await consumer.connect();
@@ -25,40 +31,85 @@ export async function startRedpandaEventsConsumer(): Promise<RedpandaConsumerHan
await consumer.run({
partitionsConsumedConcurrently: REDPANDA_PARTITIONS_CONCURRENT,
- eachMessage: async ({ message, partition }) => {
- if (!message.value) {
- return;
- }
- let payload: EventsQueuePayloadIncomingEvent['payload'];
- try {
- payload = JSON.parse(
- message.value.toString()
- ) as EventsQueuePayloadIncomingEvent['payload'];
- } catch (err) {
- logger.error(
- { err, partition, offset: message.offset },
- 'redpanda message parse failed'
- );
+ eachBatchAutoResolve: false,
+ eachBatch: async ({
+ batch,
+ resolveOffset,
+ heartbeat,
+ isRunning,
+ isStale,
+ }) => {
+ if (batch.messages.length === 0) {
return;
}
- try {
- await incomingEvent(payload);
- } catch (err) {
- // Match the GroupWorker behaviour: log and ack. At-most-once on
- // handler exceptions; failures here would otherwise block the
- // partition. Future hardening can switch to retry/DLQ.
- logger.error(
- {
- err,
- partition,
- offset: message.offset,
- projectId: payload.projectId,
- },
- 'redpanda incomingEvent handler failed'
- );
- } finally {
- markEventsActivity();
+
+ // Group by partition key (= deviceId or `${projectId}:${profileId}`).
+ // Same-key messages stay serial so sessionBuffer/session-end-job state
+ // can't race; different keys run in parallel via Promise.all.
+ // Keyless messages get their own singleton group.
+ const groups = new Map<string, KafkaMessage[]>();
+ for (const m of batch.messages) {
+ const key = m.key ? m.key.toString() : `__no_key__:${m.offset}`;
+ const arr = groups.get(key);
+ if (arr) {
+ arr.push(m);
+ } else {
+ groups.set(key, [m]);
+ }
}
+
+ await Promise.all(
+ [...groups.values()].map(async (msgs) => {
+ let processed = 0;
+ for (const m of msgs) {
+ if (!isRunning() || isStale()) {
+ return;
+ }
+
+ if (m.value) {
+ let payload: EventsQueuePayloadIncomingEvent['payload'] | null =
+ null;
+ try {
+ payload = JSON.parse(
+ m.value.toString()
+ ) as EventsQueuePayloadIncomingEvent['payload'];
+ } catch (err) {
+ logger.error(
+ { err, partition: batch.partition, offset: m.offset },
+ 'redpanda message parse failed'
+ );
+ }
+ if (payload) {
+ try {
+ await incomingEvent(payload);
+ } catch (err) {
+ // Match the previous eachMessage behaviour: log and ack.
+ // At-most-once on handler exceptions; failures here would
+ // otherwise block the partition.
+ logger.error(
+ {
+ err,
+ partition: batch.partition,
+ offset: m.offset,
+ projectId: payload.projectId,
+ },
+ 'redpanda incomingEvent handler failed'
+ );
+ }
+ }
+ }
+
+ resolveOffset(m.offset);
+ processed += 1;
+ if (processed % HEARTBEAT_EVERY === 0) {
+ await heartbeat();
+ }
+ }
+ })
+ );
+
+ await heartbeat();
+ markEventsActivity();
},
});
diff --git a/packages/queue/src/redpanda.ts b/packages/queue/src/redpanda.ts
index e7d54f322..58cd0596d 100644
--- a/packages/queue/src/redpanda.ts
+++ b/packages/queue/src/redpanda.ts
@@ -2,6 +2,8 @@ import { createLogger } from '@openpanel/logger';
import { type Consumer, Kafka, logLevel, type Producer } from 'kafkajs';
import type { EventsQueuePayloadIncomingEvent } from './queues';
+export type { KafkaMessage } from 'kafkajs';
+
export const redpandaLogger = createLogger({ name: 'redpanda' });
const parseBrokers = (raw: string | undefined): string[] => {
@@ -23,6 +25,35 @@ export const REDPANDA_PARTITIONS_CONCURRENT = Number.parseInt(
process.env.REDPANDA_PARTITIONS_CONCURRENT || '8',
10
);
+// Approx size of one event payload (observed range ~0.9–1.3 KiB).
+// We size fetch knobs in messages and convert to bytes via this constant.
+const REDPANDA_BYTES_PER_MESSAGE = 1024;
+
+export const REDPANDA_MIN_MESSAGES = Number.parseInt(
+ process.env.REDPANDA_MIN_MESSAGES || '1',
+ 10
+);
+export const REDPANDA_MAX_WAIT_MS = Number.parseInt(
+ process.env.REDPANDA_MAX_WAIT_MS || '500',
+ 10
+);
+export const REDPANDA_MAX_MESSAGES_PER_PARTITION = Number.parseInt(
+ process.env.REDPANDA_MAX_MESSAGES_PER_PARTITION || '256',
+ 10
+);
+export const REDPANDA_SESSION_TIMEOUT_MS = Number.parseInt(
+ process.env.REDPANDA_SESSION_TIMEOUT_MS || '30000',
+ 10
+);
+export const REDPANDA_HEARTBEAT_INTERVAL_MS = Number.parseInt(
+ process.env.REDPANDA_HEARTBEAT_INTERVAL_MS || '3000',
+ 10
+);
+
+const REDPANDA_MIN_BYTES =
+ REDPANDA_MIN_MESSAGES * REDPANDA_BYTES_PER_MESSAGE;
+const REDPANDA_MAX_BYTES_PER_PARTITION =
+ REDPANDA_MAX_MESSAGES_PER_PARTITION * REDPANDA_BYTES_PER_MESSAGE;
const projectIdsEnv = (process.env.REDPANDA_PROJECT_IDS || '').trim();
const allowAllProjects = projectIdsEnv === '*';
@@ -120,6 +151,11 @@ export const createRedpandaEventsConsumer = (options?: {
const client = getKafka();
const consumer = client.consumer({
groupId: options?.groupId || REDPANDA_CONSUMER_GROUP,
+ sessionTimeout: REDPANDA_SESSION_TIMEOUT_MS,
+ heartbeatInterval: REDPANDA_HEARTBEAT_INTERVAL_MS,
+ minBytes: REDPANDA_MIN_BYTES,
+ maxWaitTimeInMs: REDPANDA_MAX_WAIT_MS,
+ maxBytesPerPartition: REDPANDA_MAX_BYTES_PER_PARTITION,
});
consumers.add(consumer);
return consumer;