OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.
fix: improve kafka consumer
Carl-Gerhard Lindesvärd97d335d5fcba
diff --git a/apps/api/src/controllers/event.controller.ts b/apps/api/src/controllers/event.controller.tsindex 59a6d0f4a..cceb4b4e3 100644--- a/apps/api/src/controllers/event.controller.ts+++ b/apps/api/src/controllers/event.controller.ts@@ -2,7 +2,12 @@ import { generateId } from '@openpanel/common';import { parseUserAgent } from '@openpanel/common/server';import { getSalts } from '@openpanel/db';import { getGeoLocation } from '@openpanel/geo';-import { getEventsGroupQueueShard } from '@openpanel/queue';+import {+ type EventsQueuePayloadIncomingEvent,+ getEventsGroupQueueShard,+ produceIncomingEvent,+ shouldUseKafka,+} from '@openpanel/queue';import type { DeprecatedPostEventPayload } from '@openpanel/validation';import type { FastifyReply, FastifyRequest } from 'fastify';import { getStringHeaders, getTimestamp } from './track.controller';@@ -40,23 +45,29 @@ export async function postEvent(const groupId = uaInfo.isServer? `${projectId}:${request.body?.profileId ?? generateId()}`: deviceId;- await getEventsGroupQueueShard(groupId).add({- orderMs: new Date(timestamp).getTime(),- data: {- projectId,- headers,- event: {- ...request.body,- timestamp,- isTimestampFromThePast,- },- uaInfo,- geo,- deviceId,- sessionId: sessionId ?? '',+ const queueData: EventsQueuePayloadIncomingEvent['payload'] = {+ projectId,+ headers,+ event: {+ ...request.body,+ timestamp,+ isTimestampFromThePast,},- groupId,- });+ uaInfo,+ geo,+ deviceId,+ sessionId: sessionId ?? '',+ };++ if (shouldUseKafka()) {+ await produceIncomingEvent(queueData, groupId);+ } else {+ await getEventsGroupQueueShard(groupId).add({+ orderMs: new Date(timestamp).getTime(),+ data: queueData,+ groupId,+ });+ }reply.status(202).send('ok');}diff --git a/apps/api/src/controllers/track.controller.ts b/apps/api/src/controllers/track.controller.tsindex bf6b2d4e3..3a32f0fed 100644--- a/apps/api/src/controllers/track.controller.ts+++ b/apps/api/src/controllers/track.controller.ts@@ -218,7 +218,7 @@ async function handleTrack(const partitionKey = groupId || generateId();- if (shouldUseKafka(projectId)) {+ if (shouldUseKafka()) {promises.push(produceIncomingEvent(queueData, partitionKey));} else {promises.push(diff --git a/apps/public/content/docs/self-hosting/environment-variables.mdx b/apps/public/content/docs/self-hosting/environment-variables.mdxindex e14ab6c18..f53420eaf 100644--- a/apps/public/content/docs/self-hosting/environment-variables.mdx+++ b/apps/public/content/docs/self-hosting/environment-variables.mdx@@ -715,9 +715,9 @@ AUTO_BATCH_SIZE=100## Event Streaming (Kafka)-OpenPanel can optionally ingest events through a Kafka-compatible broker (Kafka itself, Redpanda, etc.) instead of (or alongside) the default Redis-backed events queue. This is opt-in per project and useful at high throughput, where many consumer instances can process partitions in parallel.+OpenPanel can optionally ingest events through a Kafka-compatible broker (Kafka itself, Redpanda, etc.) instead of the default Redis-backed events queue. This is useful at high throughput, where many consumer instances can process partitions in parallel.-Routing is decided per event at produce time: if the project's ID matches `KAFKA_PROJECT_IDS` and `KAFKA_BROKERS` is set, the event is produced to the Kafka topic; otherwise it stays on the default Redis-backed `events` queue. Set `ENABLED_QUEUES=events_kafka` on the worker(s) that should consume from Kafka.+Routing is all-or-nothing: if `KAFKA_BROKERS` is set, every event is produced to the Kafka topic; otherwise all events stay on the default Redis-backed `events` queue. Set `ENABLED_QUEUES=events_kafka` on the worker(s) that should consume from Kafka.<Callout>The client speaks the Kafka wire protocol via [kafkajs](https://kafka.js.org/), so any Kafka-compatible broker works. The bundled `docker-compose.yml` uses [Redpanda](https://redpanda.com/) as a lightweight, single-binary option, but you can point `KAFKA_BROKERS` at any Kafka cluster.@@ -729,7 +729,7 @@ The client speaks the Kafka wire protocol via [kafkajs](https://kafka.js.org/),**Required**: Yes (to enable the Kafka path)**Default**: None-Comma-separated list of Kafka broker addresses. When unset, the Kafka producer/consumer is disabled and all events fall back to the default Redis-backed queue.+Comma-separated list of Kafka broker addresses. When set, all events are produced to Kafka; when unset, the Kafka producer/consumer is disabled and all events fall back to the default Redis-backed queue.**Example**:```bash@@ -738,23 +738,6 @@ KAFKA_BROKERS=redpanda:9092KAFKA_BROKERS=kafka-1:9092,kafka-2:9092,kafka-3:9092```-### KAFKA_PROJECT_IDS--**Type**: `string` (comma-separated or `*`)-**Required**: No-**Default**: None (no projects route to Kafka)--Allow-list of project IDs whose events should be produced to Kafka. Use `*` to route every project's events through Kafka. If unset, no project uses Kafka even when `KAFKA_BROKERS` is configured.--**Example**:-```bash-# Route every project through Kafka-KAFKA_PROJECT_IDS=*--# Route only specific projects-KAFKA_PROJECT_IDS=proj_abc123,proj_def456-```-### KAFKA_EVENTS_TOPIC**Type**: `string`diff --git a/apps/worker/src/boot-workers.ts b/apps/worker/src/boot-workers.tsindex 5324fce9e..ec1631ac4 100644--- a/apps/worker/src/boot-workers.ts+++ b/apps/worker/src/boot-workers.ts@@ -9,6 +9,7 @@ import {gscQueue,importQueue,insightsQueue,+ isKafkaConfigured,miscQueue,notificationQueue,queueLogger,@@ -112,10 +113,15 @@ export function bootWorkers() {const workers: (Worker | GroupWorker<any>)[] = [];const extraStops: Array<() => Promise<unknown>> = [];- // Start event workers based on enabled queues+ // Start event workers based on enabled queues.+ // When Kafka is configured the producer routes every event to Kafka, so the+ // GroupMQ event shards would only poll an empty queue — skip them entirely+ // and let the Kafka consumer handle ingestion.const eventQueuesToStart: number[] = [];- if (enabledQueues.includes('events')) {+ if (isKafkaConfigured()) {+ logger.info('Kafka is configured, skipping GroupMQ event workers');+ } else if (enabledQueues.includes('events')) {// Start all event shardsfor (let i = 0; i < EVENTS_GROUP_QUEUES_SHARDS; i++) {eventQueuesToStart.push(i);@@ -169,8 +175,9 @@ export function bootWorkers() {logger.info({ concurrency }, `Started worker for ${queueName}`);}- // Start Kafka events consumer (parallel to groupmq events for allow-listed project IDs)- if (enabledQueues.includes('events_kafka') && process.env.KAFKA_BROKERS) {+ // Start Kafka events consumer. When Kafka is configured this fully replaces+ // the GroupMQ event workers (which are skipped above).+ if (enabledQueues.includes('events_kafka') && isKafkaConfigured()) {enableEventsHeartbeat();let handle: KafkaConsumerHandle | null = null;const startPromise = startKafkaEventsConsumer()diff --git a/apps/worker/src/jobs/events.incoming-event.ts b/apps/worker/src/jobs/events.incoming-event.tsindex 5035e3186..cf89391fe 100644--- a/apps/worker/src/jobs/events.incoming-event.ts+++ b/apps/worker/src/jobs/events.incoming-event.ts@@ -83,7 +83,11 @@ const parseRevenue = (revenue: unknown): number | undefined => {};export async function incomingEvent(- jobPayload: EventsQueuePayloadIncomingEvent['payload']+ jobPayload: EventsQueuePayloadIncomingEvent['payload'],+ // Kafka delivery coordinates, when the event came through the Kafka consumer.+ // Logged so a duplicate row in ClickHouse can be traced back to the exact+ // partition/offset that produced it.+ meta?: { partition: number; offset: string }) {const {geo,@@ -98,6 +102,9 @@ export async function incomingEvent(const reqId = headers['request-id'] ?? 'unknown';const logger = baseLogger.child({reqId,+ ...(meta+ ? { kafkaPartition: meta.partition, kafkaOffset: meta.offset }+ : {}),});const getProperty = (name: string): string | undefined => {// replace thing is just for older sdks when we didn't have `__`diff --git a/apps/worker/src/jobs/events.kafka-consumer.ts b/apps/worker/src/jobs/events.kafka-consumer.tsindex deca022ad..71e2f02d7 100644--- a/apps/worker/src/jobs/events.kafka-consumer.ts+++ b/apps/worker/src/jobs/events.kafka-consumer.ts@@ -6,6 +6,7 @@ import {kafkaLogger,type KafkaMessage,} from '@openpanel/queue';+import { kafkaReprocessedTotal } from '../metrics';import { logger } from '../utils/logger';import { markEventsActivity } from '../utils/worker-heartbeat';import { incomingEvent } from './events.incoming-event';@@ -19,6 +20,14 @@ export interface KafkaConsumerHandle {// comfortably under that even for slow handlers.const HEARTBEAT_EVERY = 16;+// Highest Kafka offset we have *resolved* (committed) per `topic-partition`,+// tracked across batches. Used purely for the reprocess detector below: if we+// ever see an offset at or below this watermark again, the message is being+// redelivered (at-least-once duplicate) outside of a rebalance — which is the+// signature of an offset-handling bug. Cleared on GROUP_JOIN so legitimate+// post-rebalance redelivery from the last committed offset doesn't trip it.+const resolvedHWM = new Map<string, number>();+export async function startKafkaEventsConsumer(): Promise<KafkaConsumerHandle> {const consumer = createKafkaEventsConsumer();await consumer.connect();@@ -29,6 +38,51 @@ export async function startKafkaEventsConsumer(): Promise<KafkaConsumerHandle> {consumer.on(consumer.events.HEARTBEAT, markEventsActivity);+ // ---- Lifecycle / rebalance ("re-election") visibility ----+ // We had no logging for partition reassignment before; without it a rebalance+ // storm (a common source of at-least-once duplicates) is invisible.+ consumer.on(consumer.events.GROUP_JOIN, ({ payload }) => {+ // A new assignment means partitions may have moved between members; reset+ // the reprocess watermarks so legitimate resume-from-committed-offset after+ // a rebalance is not flagged as a duplicate.+ resolvedHWM.clear();+ logger.info(+ {+ memberId: payload.memberId,+ groupId: payload.groupId,+ isLeader: payload.isLeader,+ memberAssignment: payload.memberAssignment,+ duration: payload.duration,+ },+ 'kafka consumer joined group (rebalance complete)'+ );+ });+ consumer.on(consumer.events.REBALANCING, ({ payload }) => {+ logger.warn(+ { memberId: payload.memberId, groupId: payload.groupId },+ 'kafka consumer rebalancing'+ );+ });+ consumer.on(consumer.events.CRASH, ({ payload }) => {+ logger.error(+ {+ err: payload.error,+ groupId: payload.groupId,+ restart: payload.restart,+ },+ 'kafka consumer crashed'+ );+ });+ consumer.on(consumer.events.DISCONNECT, () => {+ logger.warn('kafka consumer disconnected');+ });+ consumer.on(consumer.events.REQUEST_TIMEOUT, ({ payload }) => {+ logger.warn(+ { broker: payload.broker, clientId: payload.clientId },+ 'kafka consumer request timeout'+ );+ });+await consumer.run({partitionsConsumedConcurrently: KAFKA_PARTITIONS_CONCURRENT,eachBatchAutoResolve: false,@@ -43,6 +97,12 @@ export async function startKafkaEventsConsumer(): Promise<KafkaConsumerHandle> {return;}+ const pk = `${batch.topic}-${batch.partition}`;+ // Watermark from *previous* batches. Anything at or below this that we see+ // now is a redelivery. Captured before this batch so intra-batch+ // out-of-order processing (normal, see below) is never counted.+ const priorHWM = resolvedHWM.get(pk) ?? -1;+// Group by partition key (= deviceId or `${projectId}:${profileId}`).// Same-key messages stay serial so sessionBuffer/session-end-job state// can't race; different keys run in parallel via Promise.all.@@ -58,14 +118,41 @@ export async function startKafkaEventsConsumer(): Promise<KafkaConsumerHandle> {}}+ // Offsets that finished processing this batch. We resolve them AFTER all+ // groups complete, in strict ascending order (see the loop below).+ //+ // Why: KafkaJS `resolveOffset` is last-write-wins and the next fetch+ // starts from the last resolved offset (offsetManager.nextOffset). If we+ // resolved inside the concurrent per-key loop, a lower offset resolving+ // after a higher one would move the fetch position BACKWARDS and+ // re-deliver everything in between — the root cause of the duplicate+ // events. Resolving the contiguous ascending prefix at the end avoids+ // both duplicates (never regress) and loss (stop at the first gap).+ const processed = new Set<string>();+ let processedCount = 0;+await Promise.all([...groups.values()].map(async (msgs) => {- let processed = 0;for (const m of msgs) {if (!isRunning() || isStale()) {return;}+ // Reprocess detector: only fires for offsets already resolved in a+ // PRIOR batch (redelivery). Intra-batch out-of-order processing+ // across key-groups is expected and is not flagged.+ if (Number(m.offset) <= priorHWM) {+ kafkaReprocessedTotal.inc({ partition: String(batch.partition) });+ logger.warn(+ {+ partition: batch.partition,+ offset: m.offset,+ resolvedHighWaterMark: priorHWM,+ },+ 'kafka offset REPROCESSED — at-least-once duplicate (outside rebalance)'+ );+ }+if (m.value) {let payload: EventsQueuePayloadIncomingEvent['payload'] | null =null;@@ -81,7 +168,10 @@ export async function startKafkaEventsConsumer(): Promise<KafkaConsumerHandle> {}if (payload) {try {- await incomingEvent(payload);+ await incomingEvent(payload, {+ partition: batch.partition,+ offset: m.offset,+ });} catch (err) {// Match the previous eachMessage behaviour: log and ack.// At-most-once on handler exceptions; failures here would@@ -99,15 +189,28 @@ export async function startKafkaEventsConsumer(): Promise<KafkaConsumerHandle> {}}- resolveOffset(m.offset);- processed += 1;- if (processed % HEARTBEAT_EVERY === 0) {+ processed.add(m.offset);+ processedCount += 1;+ if (processedCount % HEARTBEAT_EVERY === 0) {await heartbeat();}}}));+ // Resolve in strict ascending offset order, stopping at the first offset+ // that did not finish (e.g. an isStale/isRunning early-return mid-batch).+ // batch.messages is already ordered by offset.+ let newHWM = priorHWM;+ for (const m of batch.messages) {+ if (!processed.has(m.offset)) {+ break;+ }+ resolveOffset(m.offset);+ newHWM = Math.max(newHWM, Number(m.offset));+ }+ resolvedHWM.set(pk, newHWM);+await heartbeat();markEventsActivity();},diff --git a/apps/worker/src/metrics.ts b/apps/worker/src/metrics.tsindex 00839945d..a197d9865 100644--- a/apps/worker/src/metrics.ts+++ b/apps/worker/src/metrics.ts@@ -25,6 +25,17 @@ export const eventsGroupJobDuration = new client.Histogram({register.registerMetric(eventsGroupJobDuration);+// Kafka event messages reprocessed (same offset redelivered outside a+// rebalance). Should stay ~0. A sustained non-zero rate means the consumer is+// re-delivering messages it already handled — an offset-handling/duplicate bug.+export const kafkaReprocessedTotal = new client.Counter({+ name: 'kafka_events_reprocessed_total',+ help: 'Kafka event messages reprocessed (offset redelivered outside a rebalance)',+ labelNames: ['partition'],+});++register.registerMetric(kafkaReprocessedTotal);+queues.forEach((queue) => {register.registerMetric(new client.Gauge({diff --git a/packages/db/package.json b/packages/db/package.jsonindex 634a264b1..abce690f7 100644--- a/packages/db/package.json+++ b/packages/db/package.json@@ -9,6 +9,7 @@"migrate:deploy:code": "pnpm with-env jiti ./code-migrations/migrate.ts","migrate:deploy:db": "pnpm with-env prisma migrate deploy","migrate:deploy": "pnpm migrate:deploy:db && pnpm migrate:deploy:code",+ "duplicate-events": "pnpm with-env jiti ./scripts/find-duplicate-events.ts","typecheck": "tsc --noEmit","with-env": "dotenv -e ../../.env -c --"},diff --git a/packages/db/scripts/find-duplicate-events.ts b/packages/db/scripts/find-duplicate-events.tsnew file mode 100644index 000000000..ef9370cc3--- /dev/null+++ b/packages/db/scripts/find-duplicate-events.ts@@ -0,0 +1,344 @@+/**+ * find-duplicate-events.ts+ *+ * Reports (and optionally deletes) duplicate rows in the ClickHouse `events`+ * table. A "duplicate" is two or more rows that are logically the same event+ * but have different `id`s — the signature of the event being processed more+ * than once (see the Kafka consumer offset-handling fix).+ *+ * Duplicates are matched by a content hash over the meaningful columns+ * (everything that identifies the event, excluding `id`), so genuinely+ * distinct events are never grouped together — only byte-identical rows.+ *+ * Usage (always dry-run unless --danger-yes-delete is passed):+ *+ * pnpm --filter @openpanel/db duplicate-events # last 24h, per day+ * pnpm --filter @openpanel/db duplicate-events --since=7d --bucket=day+ * pnpm --filter @openpanel/db duplicate-events --since=2026-06-01 --bucket=hour+ * pnpm --filter @openpanel/db duplicate-events --project=my-project+ * pnpm --filter @openpanel/db duplicate-events --since=48h --danger-yes-delete+ *+ * Flags:+ * --since=<24h|7d|ISO> Lookback window (default: 24h)+ * --bucket=<day|hour> Reporting granularity (default: day)+ * --project=<projectId> Restrict to a single project (default: all)+ * --limit=<n> Max example duplicate groups to print (default: 20)+ * --batch=<n> Ids per DELETE statement when deleting (default: 10000)+ * --danger-yes-delete Actually delete the extra rows. Keeps the lowest `id`+ * per duplicate group, deletes the rest. Without this+ * flag the script only reports.+ */+import {+ TABLE_NAMES,+ ch,+ getReplicatedTableName,+} from '../src/clickhouse/client';++// ---------------------------------------------------------------------------+// Arg parsing+// ---------------------------------------------------------------------------++type Bucket = 'day' | 'hour';++interface Args {+ since: Date;+ bucket: Bucket;+ project: string | null;+ limit: number;+ batch: number;+ delete: boolean;+}++function getArg(name: string): string | undefined {+ const prefix = `--${name}=`;+ const match = process.argv.find((a) => a.startsWith(prefix));+ return match ? match.slice(prefix.length) : undefined;+}++function hasFlag(name: string): boolean {+ return process.argv.includes(`--${name}`);+}++function parseSince(raw: string): Date {+ const relative = raw.match(/^(\d+)([hd])$/);+ if (relative) {+ const amount = Number.parseInt(relative[1]!, 10);+ const unitMs = relative[2] === 'h' ? 3600_000 : 86_400_000;+ return new Date(Date.now() - amount * unitMs);+ }+ const parsed = new Date(raw);+ if (Number.isNaN(parsed.getTime())) {+ throw new Error(+ `Invalid --since="${raw}". Use e.g. 24h, 7d, or an ISO date like 2026-06-01.`+ );+ }+ return parsed;+}++function parseArgs(): Args {+ const bucket = (getArg('bucket') ?? 'day') as Bucket;+ if (bucket !== 'day' && bucket !== 'hour') {+ throw new Error(`Invalid --bucket="${bucket}". Use day or hour.`);+ }+ return {+ since: parseSince(getArg('since') ?? '24h'),+ bucket,+ project: getArg('project') ?? null,+ limit: Number.parseInt(getArg('limit') ?? '20', 10),+ // Ids are inlined into `id IN (...)`. ~39 bytes/uuid, so 5000 ≈ 195KB stays+ // safely under ClickHouse's default max_query_size (256KB).+ batch: Number.parseInt(getArg('batch') ?? '5000', 10),+ delete: hasFlag('danger-yes-delete'),+ };+}++// ---------------------------------------------------------------------------+// SQL helpers+// ---------------------------------------------------------------------------++// Format a JS Date as a ClickHouse DateTime64(3) literal (UTC).+function chDateTime(date: Date): string {+ return date.toISOString().replace('T', ' ').replace('Z', '').slice(0, 23);+}++// Content hash identifying a logical event (everything but `id`). Properties+// are normalized to a canonical, order-independent "key=value" list so map+// ordering can never split an otherwise-identical pair. char(0)/char(1) are+// used as separators because they cannot appear in the textual values.+const DEDUP_KEY = `cityHash64(+ project_id, name, profile_id, device_id, session_id,+ toString(created_at),+ path, origin, referrer, referrer_name, referrer_type,+ country, city, region, os, os_version, browser, browser_version,+ device, brand, model, sdk_name, sdk_version,+ toString(revenue), toString(duration),+ arrayStringConcat(+ arraySort(+ arrayMap(+ (k, v) -> concat(k, char(0), v),+ mapKeys(properties), mapValues(properties)+ )+ ),+ char(1)+ )+)`;++function projectClause(project: string | null): string {+ return project ? ` AND project_id = '${project.replace(/'/g, "''")}'` : '';+}++async function query<T>(sql: string): Promise<T[]> {+ return ch+ .query({ query: sql, format: 'JSONEachRow' })+ .then((res) => res.json<T>());+}++// ---------------------------------------------------------------------------+// Report+// ---------------------------------------------------------------------------++async function report(args: Args): Promise<void> {+ const bucketFn = args.bucket === 'day' ? 'toStartOfDay' : 'toStartOfHour';+ const sinceLiteral = chDateTime(args.since);+ const where = `created_at >= '${sinceLiteral}'${projectClause(args.project)}`;++ console.log(+ `\nScanning ${TABLE_NAMES.events} since ${sinceLiteral} UTC` ++ (args.project ? ` (project=${args.project})` : '') ++ `, bucketed by ${args.bucket}\n`+ );++ const rows = await query<{+ bucket: string;+ total_rows: string;+ unique_rows: string;+ duplicate_rows: string;+ }>(`+ SELECT+ ${bucketFn}(created_at) AS bucket,+ count() AS total_rows,+ uniqExact(${DEDUP_KEY}) AS unique_rows,+ count() - uniqExact(${DEDUP_KEY}) AS duplicate_rows+ FROM ${TABLE_NAMES.events}+ WHERE ${where}+ GROUP BY bucket+ ORDER BY bucket+ `);++ if (rows.length === 0) {+ console.log('No events found in this window.');+ return;+ }++ let totalRows = 0;+ let totalDupes = 0;+ console.table(+ rows.map((r) => {+ const total = Number(r.total_rows);+ const dupes = Number(r.duplicate_rows);+ totalRows += total;+ totalDupes += dupes;+ return {+ bucket: r.bucket,+ total: total.toLocaleString(),+ duplicates: dupes.toLocaleString(),+ pct: total > 0 ? `${((dupes / total) * 100).toFixed(2)}%` : '0%',+ };+ })+ );+ console.log(+ `\nTotal: ${totalRows.toLocaleString()} rows, ` ++ `${totalDupes.toLocaleString()} duplicate rows ` ++ `(${totalRows > 0 ? ((totalDupes / totalRows) * 100).toFixed(2) : '0'}%)\n`+ );++ // Top example groups, so it's clear *what* is duplicating.+ // Note: SELECT aliases must NOT shadow real column names — ClickHouse+ // resolves identifiers in WHERE/GROUP BY against SELECT aliases first, so an+ // alias named `project_id`/`created_at`/etc. (all referenced inside DEDUP_KEY+ // in GROUP BY, and in WHERE) turns into an aggregate-in-GROUP-BY/WHERE error.+ // Hence the de-shadowed alias names below.+ const examples = await query<{+ proj: string;+ event_name: string;+ profile: string;+ first_seen: string;+ occurrences: string;+ ids: string[];+ }>(`+ SELECT+ any(project_id) AS proj,+ any(name) AS event_name,+ any(profile_id) AS profile,+ toString(min(created_at)) AS first_seen,+ count() AS occurrences,+ arraySlice(groupArray(id), 1, 5) AS ids+ FROM ${TABLE_NAMES.events}+ WHERE ${where}+ GROUP BY ${DEDUP_KEY}+ HAVING occurrences > 1+ ORDER BY occurrences DESC+ LIMIT ${args.limit}+ `);++ if (examples.length > 0) {+ console.log(`Top ${examples.length} duplicate groups:`);+ console.table(+ examples.map((e) => ({+ project: e.proj,+ name: e.event_name,+ profile: e.profile,+ created_at: e.first_seen,+ copies: e.occurrences,+ sample_ids: e.ids.join(', '),+ }))+ );+ }+}++// ---------------------------------------------------------------------------+// Delete+// ---------------------------------------------------------------------------++function* bucketRanges(+ since: Date,+ bucket: Bucket+): Generator<{ start: Date; end: Date }> {+ const stepMs = bucket === 'day' ? 86_400_000 : 3600_000;+ // Align the first bucket to the start of the day/hour.+ const start = new Date(since);+ if (bucket === 'day') {+ start.setUTCHours(0, 0, 0, 0);+ } else {+ start.setUTCMinutes(0, 0, 0);+ }+ const now = Date.now();+ for (let t = start.getTime(); t < now; t += stepMs) {+ yield { start: new Date(t), end: new Date(t + stepMs) };+ }+}++async function deleteDuplicates(args: Args): Promise<void> {+ const target = getReplicatedTableName(TABLE_NAMES.events);+ console.log(+ `\n⚠️ DELETE MODE — keeping the lowest id per group, deleting the rest.\n` ++ ` Target: ${target}\n` ++ ` Note: ClickHouse mutations run asynchronously; rows disappear once the\n` ++ ` mutation completes server-side.\n`+ );++ let grandTotal = 0;+ for (const range of bucketRanges(args.since, args.bucket)) {+ const startLiteral = chDateTime(range.start);+ const endLiteral = chDateTime(range.end);+ const where =+ `created_at >= '${startLiteral}' AND created_at < '${endLiteral}'` ++ projectClause(args.project);++ // Extra rows = every row in a duplicate group except the lowest id.+ const extras = await query<{ id: string }>(`+ SELECT id FROM (+ SELECT id, row_number() OVER (+ PARTITION BY ${DEDUP_KEY} ORDER BY id ASC+ ) AS rn+ FROM ${TABLE_NAMES.events}+ WHERE ${where}+ ) WHERE rn > 1+ `);++ if (extras.length === 0) {+ continue;+ }++ const ids = extras.map((e) => e.id);+ console.log(+ `${startLiteral} → ${ids.length.toLocaleString()} duplicate rows to delete`+ );+ grandTotal += ids.length;++ for (let i = 0; i < ids.length; i += args.batch) {+ const slice = ids.slice(i, i + args.batch);+ const idList = slice.map((id) => `'${id}'`).join(',');+ // Scope the mutation to this bucket's time range so it only touches the+ // relevant monthly partition (and the primary index, when --project is+ // set) instead of scanning all history — `id` alone is not in the sort+ // key, so an unbounded `id IN (...)` mutates every partition.+ await ch.command({+ query:+ `ALTER TABLE ${target} DELETE WHERE ` ++ `created_at >= '${startLiteral}' AND created_at < '${endLiteral}'` ++ `${projectClause(args.project)} AND id IN (${idList})`,+ });+ // Avoid piling up mutations.+ await new Promise((resolve) => setTimeout(resolve, 200));+ }+ }++ console.log(+ `\nSubmitted deletes for ${grandTotal.toLocaleString()} duplicate rows.\n` ++ `Watch system.mutations for completion:\n` ++ ` SELECT * FROM system.mutations WHERE is_done = 0\n`+ );+}++// ---------------------------------------------------------------------------++async function main(): Promise<void> {+ const args = parseArgs();+ await report(args);++ if (args.delete) {+ await deleteDuplicates(args);+ } else {+ console.log(+ 'Dry run only. Re-run with --danger-yes-delete to remove the duplicates.\n'+ );+ }+}++main()+ .then(() => process.exit(0))+ .catch((error) => {+ console.error('Script failed:', error);+ process.exit(1);+ });diff --git a/packages/queue/src/kafka.ts b/packages/queue/src/kafka.tsindex 1d54cbb61..f0a0c5870 100644--- a/packages/queue/src/kafka.ts+++ b/packages/queue/src/kafka.ts@@ -78,28 +78,9 @@ const KAFKA_MIN_BYTES = KAFKA_MIN_MESSAGES * KAFKA_BYTES_PER_MESSAGE;const KAFKA_MAX_BYTES_PER_PARTITION =KAFKA_MAX_MESSAGES_PER_PARTITION * KAFKA_BYTES_PER_MESSAGE;-const projectIdsEnv = (process.env.KAFKA_PROJECT_IDS || '').trim();-const allowAllProjects = projectIdsEnv === '*';-const projectIdAllowList = new Set<string>(- projectIdsEnv && !allowAllProjects- ? projectIdsEnv- .split(',')- .map((id) => id.trim())- .filter(Boolean)- : []-);-export const isKafkaConfigured = (): boolean => KAFKA_BROKERS.length > 0;-export const shouldUseKafka = (projectId: string): boolean => {- if (!isKafkaConfigured()) {- return false;- }- if (allowAllProjects) {- return true;- }- return projectIdAllowList.has(projectId);-};+export const shouldUseKafka = (): boolean => isKafkaConfigured();let kafka: Kafka | null = null;const getKafka = (): Kafka => {