Skip to content

OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.

fix: improve kafka consumer

Carl-Gerhard Lindesvärd97d335d5fcba
diff --git a/apps/api/src/controllers/event.controller.ts b/apps/api/src/controllers/event.controller.ts
index 59a6d0f4a..cceb4b4e3 100644
--- a/apps/api/src/controllers/event.controller.ts
+++ b/apps/api/src/controllers/event.controller.ts
@@ -2,7 +2,12 @@ import { generateId } from '@openpanel/common';
import { parseUserAgent } from '@openpanel/common/server';
import { getSalts } from '@openpanel/db';
import { getGeoLocation } from '@openpanel/geo';
-import { getEventsGroupQueueShard } from '@openpanel/queue';
+import {
+ type EventsQueuePayloadIncomingEvent,
+ getEventsGroupQueueShard,
+ produceIncomingEvent,
+ shouldUseKafka,
+} from '@openpanel/queue';
import type { DeprecatedPostEventPayload } from '@openpanel/validation';
import type { FastifyReply, FastifyRequest } from 'fastify';
import { getStringHeaders, getTimestamp } from './track.controller';
@@ -40,23 +45,29 @@ export async function postEvent(
const groupId = uaInfo.isServer
? `${projectId}:${request.body?.profileId ?? generateId()}`
: deviceId;
- await getEventsGroupQueueShard(groupId).add({
- orderMs: new Date(timestamp).getTime(),
- data: {
- projectId,
- headers,
- event: {
- ...request.body,
- timestamp,
- isTimestampFromThePast,
- },
- uaInfo,
- geo,
- deviceId,
- sessionId: sessionId ?? '',
+ const queueData: EventsQueuePayloadIncomingEvent['payload'] = {
+ projectId,
+ headers,
+ event: {
+ ...request.body,
+ timestamp,
+ isTimestampFromThePast,
},
- groupId,
- });
+ uaInfo,
+ geo,
+ deviceId,
+ sessionId: sessionId ?? '',
+ };
+
+ if (shouldUseKafka()) {
+ await produceIncomingEvent(queueData, groupId);
+ } else {
+ await getEventsGroupQueueShard(groupId).add({
+ orderMs: new Date(timestamp).getTime(),
+ data: queueData,
+ groupId,
+ });
+ }
reply.status(202).send('ok');
}
diff --git a/apps/api/src/controllers/track.controller.ts b/apps/api/src/controllers/track.controller.ts
index bf6b2d4e3..3a32f0fed 100644
--- a/apps/api/src/controllers/track.controller.ts
+++ b/apps/api/src/controllers/track.controller.ts
@@ -218,7 +218,7 @@ async function handleTrack(
const partitionKey = groupId || generateId();
- if (shouldUseKafka(projectId)) {
+ if (shouldUseKafka()) {
promises.push(produceIncomingEvent(queueData, partitionKey));
} else {
promises.push(
diff --git a/apps/public/content/docs/self-hosting/environment-variables.mdx b/apps/public/content/docs/self-hosting/environment-variables.mdx
index e14ab6c18..f53420eaf 100644
--- a/apps/public/content/docs/self-hosting/environment-variables.mdx
+++ b/apps/public/content/docs/self-hosting/environment-variables.mdx
@@ -715,9 +715,9 @@ AUTO_BATCH_SIZE=100
## Event Streaming (Kafka)
-OpenPanel can optionally ingest events through a Kafka-compatible broker (Kafka itself, Redpanda, etc.) instead of (or alongside) the default Redis-backed events queue. This is opt-in per project and useful at high throughput, where many consumer instances can process partitions in parallel.
+OpenPanel can optionally ingest events through a Kafka-compatible broker (Kafka itself, Redpanda, etc.) instead of the default Redis-backed events queue. This is useful at high throughput, where many consumer instances can process partitions in parallel.
-Routing is decided per event at produce time: if the project's ID matches `KAFKA_PROJECT_IDS` and `KAFKA_BROKERS` is set, the event is produced to the Kafka topic; otherwise it stays on the default Redis-backed `events` queue. Set `ENABLED_QUEUES=events_kafka` on the worker(s) that should consume from Kafka.
+Routing is all-or-nothing: if `KAFKA_BROKERS` is set, every event is produced to the Kafka topic; otherwise all events stay on the default Redis-backed `events` queue. Set `ENABLED_QUEUES=events_kafka` on the worker(s) that should consume from Kafka.
<Callout>
The client speaks the Kafka wire protocol via [kafkajs](https://kafka.js.org/), so any Kafka-compatible broker works. The bundled `docker-compose.yml` uses [Redpanda](https://redpanda.com/) as a lightweight, single-binary option, but you can point `KAFKA_BROKERS` at any Kafka cluster.
@@ -729,7 +729,7 @@ The client speaks the Kafka wire protocol via [kafkajs](https://kafka.js.org/),
**Required**: Yes (to enable the Kafka path)
**Default**: None
-Comma-separated list of Kafka broker addresses. When unset, the Kafka producer/consumer is disabled and all events fall back to the default Redis-backed queue.
+Comma-separated list of Kafka broker addresses. When set, all events are produced to Kafka; when unset, the Kafka producer/consumer is disabled and all events fall back to the default Redis-backed queue.
**Example**:
```bash
@@ -738,23 +738,6 @@ KAFKA_BROKERS=redpanda:9092
KAFKA_BROKERS=kafka-1:9092,kafka-2:9092,kafka-3:9092
```
-### KAFKA_PROJECT_IDS
-
-**Type**: `string` (comma-separated or `*`)
-**Required**: No
-**Default**: None (no projects route to Kafka)
-
-Allow-list of project IDs whose events should be produced to Kafka. Use `*` to route every project's events through Kafka. If unset, no project uses Kafka even when `KAFKA_BROKERS` is configured.
-
-**Example**:
-```bash
-# Route every project through Kafka
-KAFKA_PROJECT_IDS=*
-
-# Route only specific projects
-KAFKA_PROJECT_IDS=proj_abc123,proj_def456
-```
-
### KAFKA_EVENTS_TOPIC
**Type**: `string`
diff --git a/apps/worker/src/boot-workers.ts b/apps/worker/src/boot-workers.ts
index 5324fce9e..ec1631ac4 100644
--- a/apps/worker/src/boot-workers.ts
+++ b/apps/worker/src/boot-workers.ts
@@ -9,6 +9,7 @@ import {
gscQueue,
importQueue,
insightsQueue,
+ isKafkaConfigured,
miscQueue,
notificationQueue,
queueLogger,
@@ -112,10 +113,15 @@ export function bootWorkers() {
const workers: (Worker | GroupWorker<any>)[] = [];
const extraStops: Array<() => Promise<unknown>> = [];
- // Start event workers based on enabled queues
+ // Start event workers based on enabled queues.
+ // When Kafka is configured the producer routes every event to Kafka, so the
+ // GroupMQ event shards would only poll an empty queue — skip them entirely
+ // and let the Kafka consumer handle ingestion.
const eventQueuesToStart: number[] = [];
- if (enabledQueues.includes('events')) {
+ if (isKafkaConfigured()) {
+ logger.info('Kafka is configured, skipping GroupMQ event workers');
+ } else if (enabledQueues.includes('events')) {
// Start all event shards
for (let i = 0; i < EVENTS_GROUP_QUEUES_SHARDS; i++) {
eventQueuesToStart.push(i);
@@ -169,8 +175,9 @@ export function bootWorkers() {
logger.info({ concurrency }, `Started worker for ${queueName}`);
}
- // Start Kafka events consumer (parallel to groupmq events for allow-listed project IDs)
- if (enabledQueues.includes('events_kafka') && process.env.KAFKA_BROKERS) {
+ // Start Kafka events consumer. When Kafka is configured this fully replaces
+ // the GroupMQ event workers (which are skipped above).
+ if (enabledQueues.includes('events_kafka') && isKafkaConfigured()) {
enableEventsHeartbeat();
let handle: KafkaConsumerHandle | null = null;
const startPromise = startKafkaEventsConsumer()
diff --git a/apps/worker/src/jobs/events.incoming-event.ts b/apps/worker/src/jobs/events.incoming-event.ts
index 5035e3186..cf89391fe 100644
--- a/apps/worker/src/jobs/events.incoming-event.ts
+++ b/apps/worker/src/jobs/events.incoming-event.ts
@@ -83,7 +83,11 @@ const parseRevenue = (revenue: unknown): number | undefined => {
};
export async function incomingEvent(
- jobPayload: EventsQueuePayloadIncomingEvent['payload']
+ jobPayload: EventsQueuePayloadIncomingEvent['payload'],
+ // Kafka delivery coordinates, when the event came through the Kafka consumer.
+ // Logged so a duplicate row in ClickHouse can be traced back to the exact
+ // partition/offset that produced it.
+ meta?: { partition: number; offset: string }
) {
const {
geo,
@@ -98,6 +102,9 @@ export async function incomingEvent(
const reqId = headers['request-id'] ?? 'unknown';
const logger = baseLogger.child({
reqId,
+ ...(meta
+ ? { kafkaPartition: meta.partition, kafkaOffset: meta.offset }
+ : {}),
});
const getProperty = (name: string): string | undefined => {
// replace thing is just for older sdks when we didn't have `__`
diff --git a/apps/worker/src/jobs/events.kafka-consumer.ts b/apps/worker/src/jobs/events.kafka-consumer.ts
index deca022ad..71e2f02d7 100644
--- a/apps/worker/src/jobs/events.kafka-consumer.ts
+++ b/apps/worker/src/jobs/events.kafka-consumer.ts
@@ -6,6 +6,7 @@ import {
kafkaLogger,
type KafkaMessage,
} from '@openpanel/queue';
+import { kafkaReprocessedTotal } from '../metrics';
import { logger } from '../utils/logger';
import { markEventsActivity } from '../utils/worker-heartbeat';
import { incomingEvent } from './events.incoming-event';
@@ -19,6 +20,14 @@ export interface KafkaConsumerHandle {
// comfortably under that even for slow handlers.
const HEARTBEAT_EVERY = 16;
+// Highest Kafka offset we have *resolved* (committed) per `topic-partition`,
+// tracked across batches. Used purely for the reprocess detector below: if we
+// ever see an offset at or below this watermark again, the message is being
+// redelivered (at-least-once duplicate) outside of a rebalance — which is the
+// signature of an offset-handling bug. Cleared on GROUP_JOIN so legitimate
+// post-rebalance redelivery from the last committed offset doesn't trip it.
+const resolvedHWM = new Map<string, number>();
+
export async function startKafkaEventsConsumer(): Promise<KafkaConsumerHandle> {
const consumer = createKafkaEventsConsumer();
await consumer.connect();
@@ -29,6 +38,51 @@ export async function startKafkaEventsConsumer(): Promise<KafkaConsumerHandle> {
consumer.on(consumer.events.HEARTBEAT, markEventsActivity);
+ // ---- Lifecycle / rebalance ("re-election") visibility ----
+ // We had no logging for partition reassignment before; without it a rebalance
+ // storm (a common source of at-least-once duplicates) is invisible.
+ consumer.on(consumer.events.GROUP_JOIN, ({ payload }) => {
+ // A new assignment means partitions may have moved between members; reset
+ // the reprocess watermarks so legitimate resume-from-committed-offset after
+ // a rebalance is not flagged as a duplicate.
+ resolvedHWM.clear();
+ logger.info(
+ {
+ memberId: payload.memberId,
+ groupId: payload.groupId,
+ isLeader: payload.isLeader,
+ memberAssignment: payload.memberAssignment,
+ duration: payload.duration,
+ },
+ 'kafka consumer joined group (rebalance complete)'
+ );
+ });
+ consumer.on(consumer.events.REBALANCING, ({ payload }) => {
+ logger.warn(
+ { memberId: payload.memberId, groupId: payload.groupId },
+ 'kafka consumer rebalancing'
+ );
+ });
+ consumer.on(consumer.events.CRASH, ({ payload }) => {
+ logger.error(
+ {
+ err: payload.error,
+ groupId: payload.groupId,
+ restart: payload.restart,
+ },
+ 'kafka consumer crashed'
+ );
+ });
+ consumer.on(consumer.events.DISCONNECT, () => {
+ logger.warn('kafka consumer disconnected');
+ });
+ consumer.on(consumer.events.REQUEST_TIMEOUT, ({ payload }) => {
+ logger.warn(
+ { broker: payload.broker, clientId: payload.clientId },
+ 'kafka consumer request timeout'
+ );
+ });
+
await consumer.run({
partitionsConsumedConcurrently: KAFKA_PARTITIONS_CONCURRENT,
eachBatchAutoResolve: false,
@@ -43,6 +97,12 @@ export async function startKafkaEventsConsumer(): Promise<KafkaConsumerHandle> {
return;
}
+ const pk = `${batch.topic}-${batch.partition}`;
+ // Watermark from *previous* batches. Anything at or below this that we see
+ // now is a redelivery. Captured before this batch so intra-batch
+ // out-of-order processing (normal, see below) is never counted.
+ const priorHWM = resolvedHWM.get(pk) ?? -1;
+
// Group by partition key (= deviceId or `${projectId}:${profileId}`).
// Same-key messages stay serial so sessionBuffer/session-end-job state
// can't race; different keys run in parallel via Promise.all.
@@ -58,14 +118,41 @@ export async function startKafkaEventsConsumer(): Promise<KafkaConsumerHandle> {
}
}
+ // Offsets that finished processing this batch. We resolve them AFTER all
+ // groups complete, in strict ascending order (see the loop below).
+ //
+ // Why: KafkaJS `resolveOffset` is last-write-wins and the next fetch
+ // starts from the last resolved offset (offsetManager.nextOffset). If we
+ // resolved inside the concurrent per-key loop, a lower offset resolving
+ // after a higher one would move the fetch position BACKWARDS and
+ // re-deliver everything in between — the root cause of the duplicate
+ // events. Resolving the contiguous ascending prefix at the end avoids
+ // both duplicates (never regress) and loss (stop at the first gap).
+ const processed = new Set<string>();
+ let processedCount = 0;
+
await Promise.all(
[...groups.values()].map(async (msgs) => {
- let processed = 0;
for (const m of msgs) {
if (!isRunning() || isStale()) {
return;
}
+ // Reprocess detector: only fires for offsets already resolved in a
+ // PRIOR batch (redelivery). Intra-batch out-of-order processing
+ // across key-groups is expected and is not flagged.
+ if (Number(m.offset) <= priorHWM) {
+ kafkaReprocessedTotal.inc({ partition: String(batch.partition) });
+ logger.warn(
+ {
+ partition: batch.partition,
+ offset: m.offset,
+ resolvedHighWaterMark: priorHWM,
+ },
+ 'kafka offset REPROCESSED — at-least-once duplicate (outside rebalance)'
+ );
+ }
+
if (m.value) {
let payload: EventsQueuePayloadIncomingEvent['payload'] | null =
null;
@@ -81,7 +168,10 @@ export async function startKafkaEventsConsumer(): Promise<KafkaConsumerHandle> {
}
if (payload) {
try {
- await incomingEvent(payload);
+ await incomingEvent(payload, {
+ partition: batch.partition,
+ offset: m.offset,
+ });
} catch (err) {
// Match the previous eachMessage behaviour: log and ack.
// At-most-once on handler exceptions; failures here would
@@ -99,15 +189,28 @@ export async function startKafkaEventsConsumer(): Promise<KafkaConsumerHandle> {
}
}
- resolveOffset(m.offset);
- processed += 1;
- if (processed % HEARTBEAT_EVERY === 0) {
+ processed.add(m.offset);
+ processedCount += 1;
+ if (processedCount % HEARTBEAT_EVERY === 0) {
await heartbeat();
}
}
})
);
+ // Resolve in strict ascending offset order, stopping at the first offset
+ // that did not finish (e.g. an isStale/isRunning early-return mid-batch).
+ // batch.messages is already ordered by offset.
+ let newHWM = priorHWM;
+ for (const m of batch.messages) {
+ if (!processed.has(m.offset)) {
+ break;
+ }
+ resolveOffset(m.offset);
+ newHWM = Math.max(newHWM, Number(m.offset));
+ }
+ resolvedHWM.set(pk, newHWM);
+
await heartbeat();
markEventsActivity();
},
diff --git a/apps/worker/src/metrics.ts b/apps/worker/src/metrics.ts
index 00839945d..a197d9865 100644
--- a/apps/worker/src/metrics.ts
+++ b/apps/worker/src/metrics.ts
@@ -25,6 +25,17 @@ export const eventsGroupJobDuration = new client.Histogram({
register.registerMetric(eventsGroupJobDuration);
+// Kafka event messages reprocessed (same offset redelivered outside a
+// rebalance). Should stay ~0. A sustained non-zero rate means the consumer is
+// re-delivering messages it already handled — an offset-handling/duplicate bug.
+export const kafkaReprocessedTotal = new client.Counter({
+ name: 'kafka_events_reprocessed_total',
+ help: 'Kafka event messages reprocessed (offset redelivered outside a rebalance)',
+ labelNames: ['partition'],
+});
+
+register.registerMetric(kafkaReprocessedTotal);
+
queues.forEach((queue) => {
register.registerMetric(
new client.Gauge({
diff --git a/packages/db/package.json b/packages/db/package.json
index 634a264b1..abce690f7 100644
--- a/packages/db/package.json
+++ b/packages/db/package.json
@@ -9,6 +9,7 @@
"migrate:deploy:code": "pnpm with-env jiti ./code-migrations/migrate.ts",
"migrate:deploy:db": "pnpm with-env prisma migrate deploy",
"migrate:deploy": "pnpm migrate:deploy:db && pnpm migrate:deploy:code",
+ "duplicate-events": "pnpm with-env jiti ./scripts/find-duplicate-events.ts",
"typecheck": "tsc --noEmit",
"with-env": "dotenv -e ../../.env -c --"
},
diff --git a/packages/db/scripts/find-duplicate-events.ts b/packages/db/scripts/find-duplicate-events.ts
new file mode 100644
index 000000000..ef9370cc3
--- /dev/null
+++ b/packages/db/scripts/find-duplicate-events.ts
@@ -0,0 +1,344 @@
+/**
+ * find-duplicate-events.ts
+ *
+ * Reports (and optionally deletes) duplicate rows in the ClickHouse `events`
+ * table. A "duplicate" is two or more rows that are logically the same event
+ * but have different `id`s — the signature of the event being processed more
+ * than once (see the Kafka consumer offset-handling fix).
+ *
+ * Duplicates are matched by a content hash over the meaningful columns
+ * (everything that identifies the event, excluding `id`), so genuinely
+ * distinct events are never grouped together — only byte-identical rows.
+ *
+ * Usage (always dry-run unless --danger-yes-delete is passed):
+ *
+ * pnpm --filter @openpanel/db duplicate-events # last 24h, per day
+ * pnpm --filter @openpanel/db duplicate-events --since=7d --bucket=day
+ * pnpm --filter @openpanel/db duplicate-events --since=2026-06-01 --bucket=hour
+ * pnpm --filter @openpanel/db duplicate-events --project=my-project
+ * pnpm --filter @openpanel/db duplicate-events --since=48h --danger-yes-delete
+ *
+ * Flags:
+ * --since=<24h|7d|ISO> Lookback window (default: 24h)
+ * --bucket=<day|hour> Reporting granularity (default: day)
+ * --project=<projectId> Restrict to a single project (default: all)
+ * --limit=<n> Max example duplicate groups to print (default: 20)
+ * --batch=<n> Ids per DELETE statement when deleting (default: 10000)
+ * --danger-yes-delete Actually delete the extra rows. Keeps the lowest `id`
+ * per duplicate group, deletes the rest. Without this
+ * flag the script only reports.
+ */
+import {
+ TABLE_NAMES,
+ ch,
+ getReplicatedTableName,
+} from '../src/clickhouse/client';
+
+// ---------------------------------------------------------------------------
+// Arg parsing
+// ---------------------------------------------------------------------------
+
+type Bucket = 'day' | 'hour';
+
+interface Args {
+ since: Date;
+ bucket: Bucket;
+ project: string | null;
+ limit: number;
+ batch: number;
+ delete: boolean;
+}
+
+function getArg(name: string): string | undefined {
+ const prefix = `--${name}=`;
+ const match = process.argv.find((a) => a.startsWith(prefix));
+ return match ? match.slice(prefix.length) : undefined;
+}
+
+function hasFlag(name: string): boolean {
+ return process.argv.includes(`--${name}`);
+}
+
+function parseSince(raw: string): Date {
+ const relative = raw.match(/^(\d+)([hd])$/);
+ if (relative) {
+ const amount = Number.parseInt(relative[1]!, 10);
+ const unitMs = relative[2] === 'h' ? 3600_000 : 86_400_000;
+ return new Date(Date.now() - amount * unitMs);
+ }
+ const parsed = new Date(raw);
+ if (Number.isNaN(parsed.getTime())) {
+ throw new Error(
+ `Invalid --since="${raw}". Use e.g. 24h, 7d, or an ISO date like 2026-06-01.`
+ );
+ }
+ return parsed;
+}
+
+function parseArgs(): Args {
+ const bucket = (getArg('bucket') ?? 'day') as Bucket;
+ if (bucket !== 'day' && bucket !== 'hour') {
+ throw new Error(`Invalid --bucket="${bucket}". Use day or hour.`);
+ }
+ return {
+ since: parseSince(getArg('since') ?? '24h'),
+ bucket,
+ project: getArg('project') ?? null,
+ limit: Number.parseInt(getArg('limit') ?? '20', 10),
+ // Ids are inlined into `id IN (...)`. ~39 bytes/uuid, so 5000 ≈ 195KB stays
+ // safely under ClickHouse's default max_query_size (256KB).
+ batch: Number.parseInt(getArg('batch') ?? '5000', 10),
+ delete: hasFlag('danger-yes-delete'),
+ };
+}
+
+// ---------------------------------------------------------------------------
+// SQL helpers
+// ---------------------------------------------------------------------------
+
+// Format a JS Date as a ClickHouse DateTime64(3) literal (UTC).
+function chDateTime(date: Date): string {
+ return date.toISOString().replace('T', ' ').replace('Z', '').slice(0, 23);
+}
+
+// Content hash identifying a logical event (everything but `id`). Properties
+// are normalized to a canonical, order-independent "key=value" list so map
+// ordering can never split an otherwise-identical pair. char(0)/char(1) are
+// used as separators because they cannot appear in the textual values.
+const DEDUP_KEY = `cityHash64(
+ project_id, name, profile_id, device_id, session_id,
+ toString(created_at),
+ path, origin, referrer, referrer_name, referrer_type,
+ country, city, region, os, os_version, browser, browser_version,
+ device, brand, model, sdk_name, sdk_version,
+ toString(revenue), toString(duration),
+ arrayStringConcat(
+ arraySort(
+ arrayMap(
+ (k, v) -> concat(k, char(0), v),
+ mapKeys(properties), mapValues(properties)
+ )
+ ),
+ char(1)
+ )
+)`;
+
+function projectClause(project: string | null): string {
+ return project ? ` AND project_id = '${project.replace(/'/g, "''")}'` : '';
+}
+
+async function query<T>(sql: string): Promise<T[]> {
+ return ch
+ .query({ query: sql, format: 'JSONEachRow' })
+ .then((res) => res.json<T>());
+}
+
+// ---------------------------------------------------------------------------
+// Report
+// ---------------------------------------------------------------------------
+
+async function report(args: Args): Promise<void> {
+ const bucketFn = args.bucket === 'day' ? 'toStartOfDay' : 'toStartOfHour';
+ const sinceLiteral = chDateTime(args.since);
+ const where = `created_at >= '${sinceLiteral}'${projectClause(args.project)}`;
+
+ console.log(
+ `\nScanning ${TABLE_NAMES.events} since ${sinceLiteral} UTC` +
+ (args.project ? ` (project=${args.project})` : '') +
+ `, bucketed by ${args.bucket}\n`
+ );
+
+ const rows = await query<{
+ bucket: string;
+ total_rows: string;
+ unique_rows: string;
+ duplicate_rows: string;
+ }>(`
+ SELECT
+ ${bucketFn}(created_at) AS bucket,
+ count() AS total_rows,
+ uniqExact(${DEDUP_KEY}) AS unique_rows,
+ count() - uniqExact(${DEDUP_KEY}) AS duplicate_rows
+ FROM ${TABLE_NAMES.events}
+ WHERE ${where}
+ GROUP BY bucket
+ ORDER BY bucket
+ `);
+
+ if (rows.length === 0) {
+ console.log('No events found in this window.');
+ return;
+ }
+
+ let totalRows = 0;
+ let totalDupes = 0;
+ console.table(
+ rows.map((r) => {
+ const total = Number(r.total_rows);
+ const dupes = Number(r.duplicate_rows);
+ totalRows += total;
+ totalDupes += dupes;
+ return {
+ bucket: r.bucket,
+ total: total.toLocaleString(),
+ duplicates: dupes.toLocaleString(),
+ pct: total > 0 ? `${((dupes / total) * 100).toFixed(2)}%` : '0%',
+ };
+ })
+ );
+ console.log(
+ `\nTotal: ${totalRows.toLocaleString()} rows, ` +
+ `${totalDupes.toLocaleString()} duplicate rows ` +
+ `(${totalRows > 0 ? ((totalDupes / totalRows) * 100).toFixed(2) : '0'}%)\n`
+ );
+
+ // Top example groups, so it's clear *what* is duplicating.
+ // Note: SELECT aliases must NOT shadow real column names — ClickHouse
+ // resolves identifiers in WHERE/GROUP BY against SELECT aliases first, so an
+ // alias named `project_id`/`created_at`/etc. (all referenced inside DEDUP_KEY
+ // in GROUP BY, and in WHERE) turns into an aggregate-in-GROUP-BY/WHERE error.
+ // Hence the de-shadowed alias names below.
+ const examples = await query<{
+ proj: string;
+ event_name: string;
+ profile: string;
+ first_seen: string;
+ occurrences: string;
+ ids: string[];
+ }>(`
+ SELECT
+ any(project_id) AS proj,
+ any(name) AS event_name,
+ any(profile_id) AS profile,
+ toString(min(created_at)) AS first_seen,
+ count() AS occurrences,
+ arraySlice(groupArray(id), 1, 5) AS ids
+ FROM ${TABLE_NAMES.events}
+ WHERE ${where}
+ GROUP BY ${DEDUP_KEY}
+ HAVING occurrences > 1
+ ORDER BY occurrences DESC
+ LIMIT ${args.limit}
+ `);
+
+ if (examples.length > 0) {
+ console.log(`Top ${examples.length} duplicate groups:`);
+ console.table(
+ examples.map((e) => ({
+ project: e.proj,
+ name: e.event_name,
+ profile: e.profile,
+ created_at: e.first_seen,
+ copies: e.occurrences,
+ sample_ids: e.ids.join(', '),
+ }))
+ );
+ }
+}
+
+// ---------------------------------------------------------------------------
+// Delete
+// ---------------------------------------------------------------------------
+
+function* bucketRanges(
+ since: Date,
+ bucket: Bucket
+): Generator<{ start: Date; end: Date }> {
+ const stepMs = bucket === 'day' ? 86_400_000 : 3600_000;
+ // Align the first bucket to the start of the day/hour.
+ const start = new Date(since);
+ if (bucket === 'day') {
+ start.setUTCHours(0, 0, 0, 0);
+ } else {
+ start.setUTCMinutes(0, 0, 0);
+ }
+ const now = Date.now();
+ for (let t = start.getTime(); t < now; t += stepMs) {
+ yield { start: new Date(t), end: new Date(t + stepMs) };
+ }
+}
+
+async function deleteDuplicates(args: Args): Promise<void> {
+ const target = getReplicatedTableName(TABLE_NAMES.events);
+ console.log(
+ `\n⚠️ DELETE MODE — keeping the lowest id per group, deleting the rest.\n` +
+ ` Target: ${target}\n` +
+ ` Note: ClickHouse mutations run asynchronously; rows disappear once the\n` +
+ ` mutation completes server-side.\n`
+ );
+
+ let grandTotal = 0;
+ for (const range of bucketRanges(args.since, args.bucket)) {
+ const startLiteral = chDateTime(range.start);
+ const endLiteral = chDateTime(range.end);
+ const where =
+ `created_at >= '${startLiteral}' AND created_at < '${endLiteral}'` +
+ projectClause(args.project);
+
+ // Extra rows = every row in a duplicate group except the lowest id.
+ const extras = await query<{ id: string }>(`
+ SELECT id FROM (
+ SELECT id, row_number() OVER (
+ PARTITION BY ${DEDUP_KEY} ORDER BY id ASC
+ ) AS rn
+ FROM ${TABLE_NAMES.events}
+ WHERE ${where}
+ ) WHERE rn > 1
+ `);
+
+ if (extras.length === 0) {
+ continue;
+ }
+
+ const ids = extras.map((e) => e.id);
+ console.log(
+ `${startLiteral} → ${ids.length.toLocaleString()} duplicate rows to delete`
+ );
+ grandTotal += ids.length;
+
+ for (let i = 0; i < ids.length; i += args.batch) {
+ const slice = ids.slice(i, i + args.batch);
+ const idList = slice.map((id) => `'${id}'`).join(',');
+ // Scope the mutation to this bucket's time range so it only touches the
+ // relevant monthly partition (and the primary index, when --project is
+ // set) instead of scanning all history — `id` alone is not in the sort
+ // key, so an unbounded `id IN (...)` mutates every partition.
+ await ch.command({
+ query:
+ `ALTER TABLE ${target} DELETE WHERE ` +
+ `created_at >= '${startLiteral}' AND created_at < '${endLiteral}'` +
+ `${projectClause(args.project)} AND id IN (${idList})`,
+ });
+ // Avoid piling up mutations.
+ await new Promise((resolve) => setTimeout(resolve, 200));
+ }
+ }
+
+ console.log(
+ `\nSubmitted deletes for ${grandTotal.toLocaleString()} duplicate rows.\n` +
+ `Watch system.mutations for completion:\n` +
+ ` SELECT * FROM system.mutations WHERE is_done = 0\n`
+ );
+}
+
+// ---------------------------------------------------------------------------
+
+async function main(): Promise<void> {
+ const args = parseArgs();
+ await report(args);
+
+ if (args.delete) {
+ await deleteDuplicates(args);
+ } else {
+ console.log(
+ 'Dry run only. Re-run with --danger-yes-delete to remove the duplicates.\n'
+ );
+ }
+}
+
+main()
+ .then(() => process.exit(0))
+ .catch((error) => {
+ console.error('Script failed:', error);
+ process.exit(1);
+ });
diff --git a/packages/queue/src/kafka.ts b/packages/queue/src/kafka.ts
index 1d54cbb61..f0a0c5870 100644
--- a/packages/queue/src/kafka.ts
+++ b/packages/queue/src/kafka.ts
@@ -78,28 +78,9 @@ const KAFKA_MIN_BYTES = KAFKA_MIN_MESSAGES * KAFKA_BYTES_PER_MESSAGE;
const KAFKA_MAX_BYTES_PER_PARTITION =
KAFKA_MAX_MESSAGES_PER_PARTITION * KAFKA_BYTES_PER_MESSAGE;
-const projectIdsEnv = (process.env.KAFKA_PROJECT_IDS || '').trim();
-const allowAllProjects = projectIdsEnv === '*';
-const projectIdAllowList = new Set<string>(
- projectIdsEnv && !allowAllProjects
- ? projectIdsEnv
- .split(',')
- .map((id) => id.trim())
- .filter(Boolean)
- : []
-);
-
export const isKafkaConfigured = (): boolean => KAFKA_BROKERS.length > 0;
-export const shouldUseKafka = (projectId: string): boolean => {
- if (!isKafkaConfigured()) {
- return false;
- }
- if (allowAllProjects) {
- return true;
- }
- return projectIdAllowList.has(projectId);
-};
+export const shouldUseKafka = (): boolean => isKafkaConfigured();
let kafka: Kafka | null = null;
const getKafka = (): Kafka => {