OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.
ralph(P1-002): Producer-minted event ids: /track mints the id, createEvent uses it
ralph8b8c2aa74a30
diff --git a/apps/api/src/controllers/event.controller.ts b/apps/api/src/controllers/event.controller.tsindex 2fa2902aa..6ccf85110 100644--- a/apps/api/src/controllers/event.controller.ts+++ b/apps/api/src/controllers/event.controller.ts@@ -8,6 +8,7 @@ import {} from '@openpanel/queue';import type { DeprecatedPostEventPayload } from '@openpanel/validation';import type { FastifyReply, FastifyRequest } from 'fastify';+import { v4 as uuid } from 'uuid';import { getStringHeaders, getTimestamp } from './track.controller';import { applyBotSuspicion } from '@/bots/suspicion';import { getDeviceId } from '@/utils/ids';@@ -59,6 +60,8 @@ export async function postEvent(? `${projectId}:${request.body?.profileId ?? generateId()}`: deviceId;const queueData: EventsQueuePayloadIncomingEvent['payload'] = {+ // See track.controller: the producer mints the event id.+ id: uuid(),projectId,headers,event: {diff --git a/apps/api/src/controllers/track.controller.ts b/apps/api/src/controllers/track.controller.tsindex a79bc4ae1..eed7d403e 100644--- a/apps/api/src/controllers/track.controller.ts+++ b/apps/api/src/controllers/track.controller.ts@@ -32,6 +32,7 @@ import type {} from '@openpanel/validation';import type { FastifyReply, FastifyRequest } from 'fastify';import { assocPath, pathOr, pick } from 'ramda';+import { v4 as uuid } from 'uuid';import { applyBotSuspicion, stripBotProperties } from '@/bots/suspicion';import { HttpError } from '@/utils/errors';import { getDeviceId } from '@/utils/ids';@@ -250,6 +251,9 @@ async function handleTrack(}const queueData: EventsQueuePayloadIncomingEvent['payload'] = {+ // The id of the ClickHouse row this event becomes, minted here so a+ // redelivered Kafka message is a duplicate we can recognise.+ id: uuid(),projectId,headers,event: {diff --git a/apps/worker/src/jobs/events.incoming-event.ts b/apps/worker/src/jobs/events.incoming-event.tsindex 8d13de7dd..8b9471f57 100644--- a/apps/worker/src/jobs/events.incoming-event.ts+++ b/apps/worker/src/jobs/events.incoming-event.ts@@ -1,6 +1,10 @@import { getTime, isSameDomain, parsePath } from '@openpanel/common';import { getReferrerWithQuery, parseReferrer } from '@openpanel/common/server';-import type { IServiceCreateEventPayload, IServiceEvent } from '@openpanel/db';+import type {+ IServiceCreateEventPayload,+ IServiceCreateEventPayloadWithId,+ IServiceEvent,+} from '@openpanel/db';import {checkNotificationRulesForEvent,createEvent,@@ -55,7 +59,7 @@ async function markFirstEvent(projectId: string, logger: ILogger) {}async function createEventAndNotify(- payload: IServiceCreateEventPayload,+ payload: IServiceCreateEventPayloadWithId,logger: ILogger,projectId: string) {@@ -109,6 +113,10 @@ export async function incomingEvent(deviceId,sessionId,uaInfo,+ // Producer-minted; a redelivered message re-creates the same row id.+ // Absent on messages produced before the field existed — createEvent then+ // falls back to a fresh uuid.+ id: eventId,} = jobPayload;const properties = body.properties ?? {};const reqId = headers['request-id'] ?? 'unknown';@@ -212,7 +220,11 @@ export async function incomingEvent(latitude: session?.latitude ?? baseEvent.latitude,};- return createEventAndNotify(payload as IServiceEvent, logger, projectId);+ return createEventAndNotify(+ { ...(payload as IServiceEvent), id: eventId },+ logger,+ projectId+ );}if (await isEventExcludedByProjectFilter(baseEvent, projectId)) {@@ -248,6 +260,8 @@ export async function incomingEvent(if (session?.kind === 'new' || session?.kind === 'boundary') {sessionsStarted.inc({ kind: session.kind });+ // No `id` here: session_start is a second, derived row and must not share+ // the producer-minted id of the event that triggered it.await createEventAndNotify({...baseEvent,@@ -277,5 +291,9 @@ export async function incomingEvent(} as Partial<IServiceCreateEventPayload>): baseEvent;- return createEventAndNotify(finalPayload, logger, projectId);+ return createEventAndNotify(+ { ...finalPayload, id: eventId },+ logger,+ projectId+ );}diff --git a/apps/worker/src/jobs/events.incoming-events.test.ts b/apps/worker/src/jobs/events.incoming-events.test.tsindex abb22e080..38d62d5a9 100644--- a/apps/worker/src/jobs/events.incoming-events.test.ts+++ b/apps/worker/src/jobs/events.incoming-events.test.ts@@ -35,6 +35,8 @@ vi.mock('@openpanel/db', async () => {const projectId = 'test-project';const deviceId = 'device-123';const newSessionId = 'a1b2c3d4-e5f6-4789-a012-345678901234';+// What the producer (/track) would have minted for one request.+const producedEventId = '11111111-2222-4333-8444-555555555555';const geo = {country: 'US',city: 'New York',@@ -318,6 +320,65 @@ describe('incomingEvent', () => {});});+ it('re-creates the same row id when Kafka redelivers the message', async () => {+ const session = makeSession({ id: newSessionId });+ vi.mocked(sessionBuffer.ingest)+ .mockResolvedValueOnce({ kind: 'new', current: session })+ .mockResolvedValueOnce({ kind: 'extend', current: session });++ // Same message, delivered twice: identical payload, identical offset.+ const jobData = buildJobData({ id: producedEventId });+ const delivery = { partition: 3, offset: '42' };+ await incomingEvent(jobData, delivery);+ await incomingEvent(jobData, delivery);++ const eventCalls = (createEvent as Mock).mock.calls.filter(+ ([a]) => a?.name === 'test_event'+ );+ expect(eventCalls).toHaveLength(2);+ expect(eventCalls[0]![0].id).toBe(producedEventId);+ expect(eventCalls[1]![0].id).toBe(producedEventId);++ // The derived session_start row must NOT reuse the event's id.+ const sessionStartCalls = (createEvent as Mock).mock.calls.filter(+ ([a]) => a?.name === 'session_start'+ );+ expect(sessionStartCalls).toHaveLength(1);+ expect(sessionStartCalls[0]![0].id).toBeUndefined();+ });++ it('carries the producer-minted id on server-side events', async () => {+ vi.mocked(sessionBuffer.getExistingSession).mockResolvedValueOnce(null);++ await incomingEvent(+ buildJobData({+ id: producedEventId,+ event: {+ name: 'server_event',+ timestamp: new Date().toISOString(),+ isTimestampFromThePast: false,+ profileId: 'profile-123',+ },+ uaInfo: uaInfoServer,+ deviceId: '',+ sessionId: '',+ })+ );++ expect((createEvent as Mock).mock.calls[0]![0].id).toBe(producedEventId);+ });++ it('leaves the id undefined for a payload produced without one', async () => {+ vi.mocked(sessionBuffer.ingest).mockResolvedValueOnce({+ kind: 'extend',+ current: makeSession(),+ });++ await incomingEvent(buildJobData());++ expect((createEvent as Mock).mock.calls[0]![0].id).toBeUndefined();+ });+it('emits session_start only once across 3 rapid events (new → extend → extend)', async () => {const session = makeSession({ id: newSessionId });vi.mocked(sessionBuffer.ingest)@@ -325,9 +386,36 @@ describe('incomingEvent', () => {.mockResolvedValueOnce({ kind: 'extend', current: session }).mockResolvedValueOnce({ kind: 'extend', current: session });- await incomingEvent(buildJobData({ event: { name: 'e1', timestamp: new Date().toISOString(), isTimestampFromThePast: false, properties: { __path: 'https://example.com/test' } } }));- await incomingEvent(buildJobData({ event: { name: 'e2', timestamp: new Date().toISOString(), isTimestampFromThePast: false, properties: { __path: 'https://example.com/test' } } }));- await incomingEvent(buildJobData({ event: { name: 'e3', timestamp: new Date().toISOString(), isTimestampFromThePast: false, properties: { __path: 'https://example.com/test' } } }));+ await incomingEvent(+ buildJobData({+ event: {+ name: 'e1',+ timestamp: new Date().toISOString(),+ isTimestampFromThePast: false,+ properties: { __path: 'https://example.com/test' },+ },+ })+ );+ await incomingEvent(+ buildJobData({+ event: {+ name: 'e2',+ timestamp: new Date().toISOString(),+ isTimestampFromThePast: false,+ properties: { __path: 'https://example.com/test' },+ },+ })+ );+ await incomingEvent(+ buildJobData({+ event: {+ name: 'e3',+ timestamp: new Date().toISOString(),+ isTimestampFromThePast: false,+ properties: { __path: 'https://example.com/test' },+ },+ })+ );const sessionStartCalls = (createEvent as Mock).mock.calls.filter(([a]) => a?.name === 'session_start'diff --git a/packages/db/src/services/create-event-id.test.ts b/packages/db/src/services/create-event-id.test.tsnew file mode 100644index 000000000..8b10aa425--- /dev/null+++ b/packages/db/src/services/create-event-id.test.ts@@ -0,0 +1,72 @@+import { describe, expect, it, vi } from 'vitest';++const add = vi.fn();++vi.mock('../buffers', () => ({+ eventBuffer: { add },+ botBuffer: { add: vi.fn() },+}));++const { createEvent } = await import('./event.service');++const PRODUCED_EVENT_ID = '11111111-2222-4333-8444-555555555555';+const UUID_V4 =+ /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/;++const payload = {+ name: 'test_event',+ deviceId: '',+ profileId: '',+ projectId: 'test-project',+ sessionId: '',+ properties: {},+ createdAt: new Date('2026-01-01T00:00:00.000Z'),+ country: '',+ city: '',+ region: '',+ longitude: undefined,+ latitude: undefined,+ os: '',+ osVersion: '',+ browser: '',+ browserVersion: '',+ device: '',+ brand: '',+ model: '',+ duration: 0,+ path: '',+ origin: '',+ referrer: '',+ referrerName: '',+ referrerType: '',+ sdkName: '',+ sdkVersion: '',+ groups: [],+};++describe('createEvent id', () => {+ it('uses the producer-minted id for the ClickHouse row', async () => {+ add.mockClear();++ const first = await createEvent({ ...payload, id: PRODUCED_EVENT_ID });+ // The same message redelivered by Kafka.+ const second = await createEvent({ ...payload, id: PRODUCED_EVENT_ID });++ expect(add).toHaveBeenCalledTimes(2);+ expect(add.mock.calls[0]![0].id).toBe(PRODUCED_EVENT_ID);+ expect(add.mock.calls[1]![0].id).toBe(PRODUCED_EVENT_ID);+ expect(first.document.id).toBe(second.document.id);+ });++ it('falls back to a generated uuid when the payload carries no id', async () => {+ add.mockClear();++ await createEvent({ ...payload });+ await createEvent({ ...payload });++ const [firstId, secondId] = add.mock.calls.map(([event]) => event.id);+ expect(firstId).toMatch(UUID_V4);+ expect(secondId).toMatch(UUID_V4);+ expect(firstId).not.toBe(secondId);+ });+});diff --git a/packages/db/src/services/event.service.ts b/packages/db/src/services/event.service.tsindex f81cbfdec..907607158 100644--- a/packages/db/src/services/event.service.ts+++ b/packages/db/src/services/event.service.ts@@ -196,6 +196,13 @@ export type IServiceCreateEventPayload = Omit<IServiceEvent,'id' | 'importedAt' | 'profile' | 'meta'>;+/**+ * A create payload whose id was already minted upstream (the ingest producer).+ * Reusing it means a redelivered Kafka message lands as an identical-id row.+ */+export type IServiceCreateEventPayloadWithId = IServiceCreateEventPayload & {+ id?: string;+};export type IServiceImportedEventPayload = Omit<IServiceEvent,'profile' | 'meta'@@ -371,13 +378,13 @@ export async function getEvents(* else (session_start, session_end) the session-row update is correctly a* no-op anyway.*/-export async function createEvent(payload: IServiceCreateEventPayload) {+export async function createEvent(payload: IServiceCreateEventPayloadWithId) {if (!payload.profileId && payload.deviceId) {payload.profileId = payload.deviceId;}const event: IClickhouseEvent = {- id: uuid(),+ id: payload.id ?? uuid(),name: payload.name,device_id: payload.deviceId,profile_id: payload.profileId ? String(payload.profileId) : '',diff --git a/packages/queue/src/queues.ts b/packages/queue/src/queues.tsindex 4a4060342..9e66b2721 100644--- a/packages/queue/src/queues.ts+++ b/packages/queue/src/queues.ts@@ -34,6 +34,11 @@ const guardQueue = <export interface EventsQueuePayloadIncomingEvent {type: 'incomingEvent';payload: {+ // Minted by the producer (/track, /event) so a Kafka redelivery becomes an+ // identical-id row instead of a new one. Optional on the wire: a V1+ // producer omits it and a V1 consumer ignores it, so the topic payload+ // stays readable across a rolling deploy.+ id?: string;projectId: string;event: ITrackPayload & {timestamp: string | number;