Skip to content

OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.

ralph(P1-002): Producer-minted event ids: /track mints the id, createEvent uses it

ralph8b8c2aa74a30
diff --git a/apps/api/src/controllers/event.controller.ts b/apps/api/src/controllers/event.controller.ts
index 2fa2902aa..6ccf85110 100644
--- a/apps/api/src/controllers/event.controller.ts
+++ b/apps/api/src/controllers/event.controller.ts
@@ -8,6 +8,7 @@ import {
} from '@openpanel/queue';
import type { DeprecatedPostEventPayload } from '@openpanel/validation';
import type { FastifyReply, FastifyRequest } from 'fastify';
+import { v4 as uuid } from 'uuid';
import { getStringHeaders, getTimestamp } from './track.controller';
import { applyBotSuspicion } from '@/bots/suspicion';
import { getDeviceId } from '@/utils/ids';
@@ -59,6 +60,8 @@ export async function postEvent(
? `${projectId}:${request.body?.profileId ?? generateId()}`
: deviceId;
const queueData: EventsQueuePayloadIncomingEvent['payload'] = {
+ // See track.controller: the producer mints the event id.
+ id: uuid(),
projectId,
headers,
event: {
diff --git a/apps/api/src/controllers/track.controller.ts b/apps/api/src/controllers/track.controller.ts
index a79bc4ae1..eed7d403e 100644
--- a/apps/api/src/controllers/track.controller.ts
+++ b/apps/api/src/controllers/track.controller.ts
@@ -32,6 +32,7 @@ import type {
} from '@openpanel/validation';
import type { FastifyReply, FastifyRequest } from 'fastify';
import { assocPath, pathOr, pick } from 'ramda';
+import { v4 as uuid } from 'uuid';
import { applyBotSuspicion, stripBotProperties } from '@/bots/suspicion';
import { HttpError } from '@/utils/errors';
import { getDeviceId } from '@/utils/ids';
@@ -250,6 +251,9 @@ async function handleTrack(
}
const queueData: EventsQueuePayloadIncomingEvent['payload'] = {
+ // The id of the ClickHouse row this event becomes, minted here so a
+ // redelivered Kafka message is a duplicate we can recognise.
+ id: uuid(),
projectId,
headers,
event: {
diff --git a/apps/worker/src/jobs/events.incoming-event.ts b/apps/worker/src/jobs/events.incoming-event.ts
index 8d13de7dd..8b9471f57 100644
--- a/apps/worker/src/jobs/events.incoming-event.ts
+++ b/apps/worker/src/jobs/events.incoming-event.ts
@@ -1,6 +1,10 @@
import { getTime, isSameDomain, parsePath } from '@openpanel/common';
import { getReferrerWithQuery, parseReferrer } from '@openpanel/common/server';
-import type { IServiceCreateEventPayload, IServiceEvent } from '@openpanel/db';
+import type {
+ IServiceCreateEventPayload,
+ IServiceCreateEventPayloadWithId,
+ IServiceEvent,
+} from '@openpanel/db';
import {
checkNotificationRulesForEvent,
createEvent,
@@ -55,7 +59,7 @@ async function markFirstEvent(projectId: string, logger: ILogger) {
}
async function createEventAndNotify(
- payload: IServiceCreateEventPayload,
+ payload: IServiceCreateEventPayloadWithId,
logger: ILogger,
projectId: string
) {
@@ -109,6 +113,10 @@ export async function incomingEvent(
deviceId,
sessionId,
uaInfo,
+ // Producer-minted; a redelivered message re-creates the same row id.
+ // Absent on messages produced before the field existed — createEvent then
+ // falls back to a fresh uuid.
+ id: eventId,
} = jobPayload;
const properties = body.properties ?? {};
const reqId = headers['request-id'] ?? 'unknown';
@@ -212,7 +220,11 @@ export async function incomingEvent(
latitude: session?.latitude ?? baseEvent.latitude,
};
- return createEventAndNotify(payload as IServiceEvent, logger, projectId);
+ return createEventAndNotify(
+ { ...(payload as IServiceEvent), id: eventId },
+ logger,
+ projectId
+ );
}
if (await isEventExcludedByProjectFilter(baseEvent, projectId)) {
@@ -248,6 +260,8 @@ export async function incomingEvent(
if (session?.kind === 'new' || session?.kind === 'boundary') {
sessionsStarted.inc({ kind: session.kind });
+ // No `id` here: session_start is a second, derived row and must not share
+ // the producer-minted id of the event that triggered it.
await createEventAndNotify(
{
...baseEvent,
@@ -277,5 +291,9 @@ export async function incomingEvent(
} as Partial<IServiceCreateEventPayload>)
: baseEvent;
- return createEventAndNotify(finalPayload, logger, projectId);
+ return createEventAndNotify(
+ { ...finalPayload, id: eventId },
+ logger,
+ projectId
+ );
}
diff --git a/apps/worker/src/jobs/events.incoming-events.test.ts b/apps/worker/src/jobs/events.incoming-events.test.ts
index abb22e080..38d62d5a9 100644
--- a/apps/worker/src/jobs/events.incoming-events.test.ts
+++ b/apps/worker/src/jobs/events.incoming-events.test.ts
@@ -35,6 +35,8 @@ vi.mock('@openpanel/db', async () => {
const projectId = 'test-project';
const deviceId = 'device-123';
const newSessionId = 'a1b2c3d4-e5f6-4789-a012-345678901234';
+// What the producer (/track) would have minted for one request.
+const producedEventId = '11111111-2222-4333-8444-555555555555';
const geo = {
country: 'US',
city: 'New York',
@@ -318,6 +320,65 @@ describe('incomingEvent', () => {
});
});
+ it('re-creates the same row id when Kafka redelivers the message', async () => {
+ const session = makeSession({ id: newSessionId });
+ vi.mocked(sessionBuffer.ingest)
+ .mockResolvedValueOnce({ kind: 'new', current: session })
+ .mockResolvedValueOnce({ kind: 'extend', current: session });
+
+ // Same message, delivered twice: identical payload, identical offset.
+ const jobData = buildJobData({ id: producedEventId });
+ const delivery = { partition: 3, offset: '42' };
+ await incomingEvent(jobData, delivery);
+ await incomingEvent(jobData, delivery);
+
+ const eventCalls = (createEvent as Mock).mock.calls.filter(
+ ([a]) => a?.name === 'test_event'
+ );
+ expect(eventCalls).toHaveLength(2);
+ expect(eventCalls[0]![0].id).toBe(producedEventId);
+ expect(eventCalls[1]![0].id).toBe(producedEventId);
+
+ // The derived session_start row must NOT reuse the event's id.
+ const sessionStartCalls = (createEvent as Mock).mock.calls.filter(
+ ([a]) => a?.name === 'session_start'
+ );
+ expect(sessionStartCalls).toHaveLength(1);
+ expect(sessionStartCalls[0]![0].id).toBeUndefined();
+ });
+
+ it('carries the producer-minted id on server-side events', async () => {
+ vi.mocked(sessionBuffer.getExistingSession).mockResolvedValueOnce(null);
+
+ await incomingEvent(
+ buildJobData({
+ id: producedEventId,
+ event: {
+ name: 'server_event',
+ timestamp: new Date().toISOString(),
+ isTimestampFromThePast: false,
+ profileId: 'profile-123',
+ },
+ uaInfo: uaInfoServer,
+ deviceId: '',
+ sessionId: '',
+ })
+ );
+
+ expect((createEvent as Mock).mock.calls[0]![0].id).toBe(producedEventId);
+ });
+
+ it('leaves the id undefined for a payload produced without one', async () => {
+ vi.mocked(sessionBuffer.ingest).mockResolvedValueOnce({
+ kind: 'extend',
+ current: makeSession(),
+ });
+
+ await incomingEvent(buildJobData());
+
+ expect((createEvent as Mock).mock.calls[0]![0].id).toBeUndefined();
+ });
+
it('emits session_start only once across 3 rapid events (new → extend → extend)', async () => {
const session = makeSession({ id: newSessionId });
vi.mocked(sessionBuffer.ingest)
@@ -325,9 +386,36 @@ describe('incomingEvent', () => {
.mockResolvedValueOnce({ kind: 'extend', current: session })
.mockResolvedValueOnce({ kind: 'extend', current: session });
- await incomingEvent(buildJobData({ event: { name: 'e1', timestamp: new Date().toISOString(), isTimestampFromThePast: false, properties: { __path: 'https://example.com/test' } } }));
- await incomingEvent(buildJobData({ event: { name: 'e2', timestamp: new Date().toISOString(), isTimestampFromThePast: false, properties: { __path: 'https://example.com/test' } } }));
- await incomingEvent(buildJobData({ event: { name: 'e3', timestamp: new Date().toISOString(), isTimestampFromThePast: false, properties: { __path: 'https://example.com/test' } } }));
+ await incomingEvent(
+ buildJobData({
+ event: {
+ name: 'e1',
+ timestamp: new Date().toISOString(),
+ isTimestampFromThePast: false,
+ properties: { __path: 'https://example.com/test' },
+ },
+ })
+ );
+ await incomingEvent(
+ buildJobData({
+ event: {
+ name: 'e2',
+ timestamp: new Date().toISOString(),
+ isTimestampFromThePast: false,
+ properties: { __path: 'https://example.com/test' },
+ },
+ })
+ );
+ await incomingEvent(
+ buildJobData({
+ event: {
+ name: 'e3',
+ timestamp: new Date().toISOString(),
+ isTimestampFromThePast: false,
+ properties: { __path: 'https://example.com/test' },
+ },
+ })
+ );
const sessionStartCalls = (createEvent as Mock).mock.calls.filter(
([a]) => a?.name === 'session_start'
diff --git a/packages/db/src/services/create-event-id.test.ts b/packages/db/src/services/create-event-id.test.ts
new file mode 100644
index 000000000..8b10aa425
--- /dev/null
+++ b/packages/db/src/services/create-event-id.test.ts
@@ -0,0 +1,72 @@
+import { describe, expect, it, vi } from 'vitest';
+
+const add = vi.fn();
+
+vi.mock('../buffers', () => ({
+ eventBuffer: { add },
+ botBuffer: { add: vi.fn() },
+}));
+
+const { createEvent } = await import('./event.service');
+
+const PRODUCED_EVENT_ID = '11111111-2222-4333-8444-555555555555';
+const UUID_V4 =
+ /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/;
+
+const payload = {
+ name: 'test_event',
+ deviceId: '',
+ profileId: '',
+ projectId: 'test-project',
+ sessionId: '',
+ properties: {},
+ createdAt: new Date('2026-01-01T00:00:00.000Z'),
+ country: '',
+ city: '',
+ region: '',
+ longitude: undefined,
+ latitude: undefined,
+ os: '',
+ osVersion: '',
+ browser: '',
+ browserVersion: '',
+ device: '',
+ brand: '',
+ model: '',
+ duration: 0,
+ path: '',
+ origin: '',
+ referrer: '',
+ referrerName: '',
+ referrerType: '',
+ sdkName: '',
+ sdkVersion: '',
+ groups: [],
+};
+
+describe('createEvent id', () => {
+ it('uses the producer-minted id for the ClickHouse row', async () => {
+ add.mockClear();
+
+ const first = await createEvent({ ...payload, id: PRODUCED_EVENT_ID });
+ // The same message redelivered by Kafka.
+ const second = await createEvent({ ...payload, id: PRODUCED_EVENT_ID });
+
+ expect(add).toHaveBeenCalledTimes(2);
+ expect(add.mock.calls[0]![0].id).toBe(PRODUCED_EVENT_ID);
+ expect(add.mock.calls[1]![0].id).toBe(PRODUCED_EVENT_ID);
+ expect(first.document.id).toBe(second.document.id);
+ });
+
+ it('falls back to a generated uuid when the payload carries no id', async () => {
+ add.mockClear();
+
+ await createEvent({ ...payload });
+ await createEvent({ ...payload });
+
+ const [firstId, secondId] = add.mock.calls.map(([event]) => event.id);
+ expect(firstId).toMatch(UUID_V4);
+ expect(secondId).toMatch(UUID_V4);
+ expect(firstId).not.toBe(secondId);
+ });
+});
diff --git a/packages/db/src/services/event.service.ts b/packages/db/src/services/event.service.ts
index f81cbfdec..907607158 100644
--- a/packages/db/src/services/event.service.ts
+++ b/packages/db/src/services/event.service.ts
@@ -196,6 +196,13 @@ export type IServiceCreateEventPayload = Omit<
IServiceEvent,
'id' | 'importedAt' | 'profile' | 'meta'
>;
+/**
+ * A create payload whose id was already minted upstream (the ingest producer).
+ * Reusing it means a redelivered Kafka message lands as an identical-id row.
+ */
+export type IServiceCreateEventPayloadWithId = IServiceCreateEventPayload & {
+ id?: string;
+};
export type IServiceImportedEventPayload = Omit<
IServiceEvent,
'profile' | 'meta'
@@ -371,13 +378,13 @@ export async function getEvents(
* else (session_start, session_end) the session-row update is correctly a
* no-op anyway.
*/
-export async function createEvent(payload: IServiceCreateEventPayload) {
+export async function createEvent(payload: IServiceCreateEventPayloadWithId) {
if (!payload.profileId && payload.deviceId) {
payload.profileId = payload.deviceId;
}
const event: IClickhouseEvent = {
- id: uuid(),
+ id: payload.id ?? uuid(),
name: payload.name,
device_id: payload.deviceId,
profile_id: payload.profileId ? String(payload.profileId) : '',
diff --git a/packages/queue/src/queues.ts b/packages/queue/src/queues.ts
index 4a4060342..9e66b2721 100644
--- a/packages/queue/src/queues.ts
+++ b/packages/queue/src/queues.ts
@@ -34,6 +34,11 @@ const guardQueue = <
export interface EventsQueuePayloadIncomingEvent {
type: 'incomingEvent';
payload: {
+ // Minted by the producer (/track, /event) so a Kafka redelivery becomes an
+ // identical-id row instead of a new one. Optional on the wire: a V1
+ // producer omits it and a V1 consumer ignores it, so the topic payload
+ // stays readable across a rolling deploy.
+ id?: string;
projectId: string;
event: ITrackPayload & {
timestamp: string | number;