Skip to content

OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.

ralph(M8-003): kafka consumer + incoming-event + create-session-end into ingest/session

ralph3e785893af64
diff --git a/apps/worker/src/jobs/events.incoming-event.ts b/apps/worker/src/jobs/events.incoming-event.ts
index cb0b9d338..ed4adeacb 100644
--- a/apps/worker/src/jobs/events.incoming-event.ts
+++ b/apps/worker/src/jobs/events.incoming-event.ts
@@ -1,302 +1,40 @@
-import { getTime, isSameDomain, parsePath } from '@openpanel/common';
+// Dissolved into @openpanel/core's ingest module (M8-003): the whole handler
+// — session lifecycle decisions, session_start emission, referrer inheritance
+// and the producer-minted event id — moved to
+// packages/core/src/modules/ingest/src/incoming-event-handler.ts. This file
+// stays (DELEGATE PATTERN) because it is what binds core's injected
+// dependencies to V1's world: this worker's registry counters, this worker's
+// pino, and `enqueueSessionEndV2`, which speaks @openpanel/queue's BullMQ
+// wire shape and therefore cannot live in core.
import {
- getReferrerWithQuery,
- type ILogger,
- parseReferrer,
+ type IncomingEventDelivery,
+ type IncomingEventDeps,
+ type IncomingEventPayload,
+ incomingEvent as incomingEventCore,
+ loadIncomingEventDeps,
} from '@openpanel/core';
-import type {
- IServiceCreateEventPayload,
- IServiceCreateEventPayloadWithId,
- IServiceEvent,
-} from '@openpanel/db';
-import {
- checkNotificationRulesForEvent,
- createEvent,
- db,
- getProjectByIdCached,
- matchEvent,
- sessionBuffer,
-} from '@openpanel/db';
-import type { EventsQueuePayloadIncomingEvent } from '@openpanel/queue';
-import { anyPass, isEmpty, isNil, mergeDeepRight, omit, reject } from 'ramda';
import { sessionEndsEnqueued, sessionsStarted } from '@/metrics';
import { logger as baseLogger } from '@/utils/logger';
import { enqueueSessionEndV2 } from '@/utils/session-handler';
-const GLOBAL_PROPERTIES = ['__path', '__referrer', '__timestamp', '__revenue'];
-
-// Strip empty/nullish from B, then deep-merge over A.
-const merge = <A, B>(a: Partial<A>, b: Partial<B>): A & B =>
- mergeDeepRight(a, reject(anyPass([isEmpty, isNil]))(b)) as A & B;
-
-async function isEventExcludedByProjectFilter(
- payload: IServiceCreateEventPayload,
- projectId: string
-): Promise<boolean> {
- const project = await getProjectByIdCached(projectId);
- const eventExcludeFilters = (project?.filters ?? []).filter(
- (f) => f.type === 'event'
- );
- if (eventExcludeFilters.length === 0) {
- return false;
- }
- return eventExcludeFilters.some((filter) => matchEvent(payload, filter));
-}
-
-/**
- * Records the project's first-ever event timestamp exactly once. The cached
- * project read makes this a no-op on every event after the first; the
- * conditional update keeps concurrent workers idempotent.
- */
-async function markFirstEvent(projectId: string, logger: ILogger) {
- const project = await getProjectByIdCached(projectId);
- if (!project || project.firstEventAt) {
- return;
- }
- await db.project.updateMany({
- where: { id: projectId, firstEventAt: null },
- data: { firstEventAt: new Date() },
+let deps: Promise<IncomingEventDeps> | null = null;
+
+// Resolved once per process: the loader dynamically imports @openpanel/db's
+// buffers, prisma client and notification service, and this is the hot path.
+function getDeps(): Promise<IncomingEventDeps> {
+ deps ??= loadIncomingEventDeps(baseLogger, enqueueSessionEndV2, {
+ // V1's own registry, so this worker's /metrics body is unchanged.
+ metrics: {
+ sessionStarted: (kind) => sessionsStarted.inc({ kind }),
+ sessionEndEnqueued: (source) => sessionEndsEnqueued.inc({ source }),
+ },
});
- await getProjectByIdCached.clear(projectId);
- logger.info({ projectId }, 'Project received its first event');
+ return deps;
}
-async function createEventAndNotify(
- payload: IServiceCreateEventPayloadWithId,
- logger: ILogger,
- projectId: string
-) {
- const isExcluded = await isEventExcludedByProjectFilter(payload, projectId);
- if (isExcluded) {
- logger.info(
- { event: payload.name, projectId },
- 'Event excluded by project filter'
- );
- return null;
- }
-
- logger.info({ event: payload }, 'Creating event');
- const [event] = await Promise.all([
- createEvent(payload),
- checkNotificationRulesForEvent(payload).catch(() => null),
- ]);
- // Only after the event is accepted — recording the first event before a
- // failed createEvent would leave the activation checklist claiming data
- // arrived that was never persisted.
- await markFirstEvent(projectId, logger).catch(() => null);
- return event;
-}
-
-const parseRevenue = (revenue: unknown): number | undefined => {
- if (!revenue) {
- return undefined;
- }
- if (typeof revenue === 'number') {
- return revenue;
- }
- if (typeof revenue === 'string') {
- const parsed = Number.parseFloat(revenue);
- return Number.isNaN(parsed) ? undefined : parsed;
- }
- return undefined;
-};
-
export async function incomingEvent(
- jobPayload: EventsQueuePayloadIncomingEvent['payload'],
- // Kafka delivery coordinates, when the event came through the Kafka consumer.
- // Logged so a duplicate row in ClickHouse can be traced back to the exact
- // partition/offset that produced it.
- meta?: { partition: number; offset: string }
+ jobPayload: IncomingEventPayload,
+ meta?: IncomingEventDelivery
) {
- const {
- geo,
- event: body,
- headers,
- projectId,
- deviceId,
- sessionId,
- uaInfo,
- // Producer-minted; a redelivered message re-creates the same row id.
- // Absent on messages produced before the field existed — createEvent then
- // falls back to a fresh uuid.
- id: eventId,
- } = jobPayload;
- const properties = body.properties ?? {};
- const reqId = headers['request-id'] ?? 'unknown';
- const logger = baseLogger.child({
- reqId,
- ...(meta
- ? { kafkaPartition: meta.partition, kafkaOffset: meta.offset }
- : {}),
- });
- const getProperty = (name: string): string | undefined => {
- // replace thing is just for older sdks when we didn't have `__`
- // remove when kiddokitchen app (24.09.02) is not used anymore
- return (
- ((properties[name] || properties[name.replace('__', '')]) as
- | string
- | null
- | undefined) ?? undefined
- );
- };
-
- const profileId = body.profileId ? String(body.profileId) : '';
- const createdAt = new Date(body.timestamp);
- const isTimestampFromThePast = body.isTimestampFromThePast;
- const url = getProperty('__path');
- const { path, hash, query, origin } = parsePath(url);
- const referrer = isSameDomain(getProperty('__referrer'), url)
- ? null
- : parseReferrer(getProperty('__referrer'));
- const utmReferrer = getReferrerWithQuery(query);
- const sdkName = headers['openpanel-sdk-name'];
- const sdkVersion = headers['openpanel-sdk-version'];
-
- const baseEvent: IServiceCreateEventPayload = {
- name: body.name,
- profileId,
- projectId,
- deviceId,
- sessionId,
- properties: omit(GLOBAL_PROPERTIES, {
- ...properties,
- __hash: hash,
- __query: query,
- }),
- groups: body.groups ?? [],
- createdAt,
- duration: 0,
- sdkName,
- sdkVersion,
- city: geo.city,
- country: geo.country,
- region: geo.region,
- longitude: geo.longitude,
- latitude: geo.latitude,
- path,
- origin,
- referrer: referrer?.url || '',
- referrerName: utmReferrer?.name || referrer?.name || referrer?.url,
- referrerType: utmReferrer?.type || referrer?.type || '',
- os: uaInfo.os,
- osVersion: uaInfo.osVersion,
- browser: uaInfo.browser,
- browserVersion: uaInfo.browserVersion,
- device: uaInfo.device,
- brand: uaInfo.brand,
- model: uaInfo.model,
- revenue:
- body.name === 'revenue' && '__revenue' in properties
- ? parseRevenue(properties.__revenue)
- : undefined,
- };
-
- // Server-side and "timestamp from the past" events ride alongside an
- // existing client session if there is one — they don't open / close
- // sessions of their own.
- if (uaInfo.isServer || isTimestampFromThePast) {
- const session =
- profileId && !isTimestampFromThePast
- ? await sessionBuffer.getExistingSession({ profileId, projectId })
- : null;
-
- const payload = {
- ...baseEvent,
- deviceId: session?.device_id ?? '',
- sessionId: session?.id ?? '',
- referrer: session?.referrer ?? undefined,
- referrerName: session?.referrer_name ?? undefined,
- referrerType: session?.referrer_type ?? undefined,
- path: session?.exit_path ?? baseEvent.path,
- origin: session?.exit_origin ?? baseEvent.origin,
- os: session?.os ?? baseEvent.os,
- osVersion: session?.os_version ?? baseEvent.osVersion,
- browserVersion: session?.browser_version ?? baseEvent.browserVersion,
- browser: session?.browser ?? baseEvent.browser,
- device: session?.device ?? baseEvent.device,
- brand: session?.brand ?? baseEvent.brand,
- model: session?.model ?? baseEvent.model,
- city: session?.city ?? baseEvent.city,
- country: session?.country ?? baseEvent.country,
- region: session?.region ?? baseEvent.region,
- longitude: session?.longitude ?? baseEvent.longitude,
- latitude: session?.latitude ?? baseEvent.latitude,
- };
-
- return createEventAndNotify(
- { ...(payload as IServiceEvent), id: eventId },
- logger,
- projectId
- );
- }
-
- if (await isEventExcludedByProjectFilter(baseEvent, projectId)) {
- logger.info(
- { event: baseEvent.name, projectId },
- 'Skipping session_start and event (excluded by project filter)'
- );
- return null;
- }
-
- // The single source of truth for session lifecycle. Reads the current
- // session, decides extend/new/boundary, writes back. The returned
- // `current` is the canonical session — use its referrer fields for
- // inheritance, just like the previous behavior.
- const session = await sessionBuffer.ingest(baseEvent);
-
- if (session?.kind === 'boundary') {
- // Close the old session in a separate job (one Redis-buffered insert
- // for the session_end event + notification rule check). Idempotent via
- // BullMQ jobId dedup.
- await enqueueSessionEndV2({
- payload: baseEvent,
- closedSession: session.closed,
- })
- .then(() => sessionEndsEnqueued.inc({ source: 'boundary' }))
- .catch((error) => {
- logger.error(
- { err: error, deviceId, sessionId: session.closed.id },
- 'Error enqueueing session_end on boundary'
- );
- });
- }
-
- if (session?.kind === 'new' || session?.kind === 'boundary') {
- sessionsStarted.inc({ kind: session.kind });
- // No `id` here: session_start is a second, derived row and must not share
- // the producer-minted id of the event that triggered it.
- await createEventAndNotify(
- {
- ...baseEvent,
- name: 'session_start',
- createdAt: new Date(getTime(baseEvent.createdAt) - 100),
- },
- logger,
- projectId
- ).catch((error) => {
- logger.error(
- { err: error, event: baseEvent },
- 'Error creating session start event'
- );
- throw error;
- });
- }
-
- // Inherit referrer fields from the canonical session for the actual event.
- // For 'extend' this preserves the original session's referrer across
- // mid-session events; for 'new' / 'boundary' it's the event's own referrer
- // (which `ingest` just stored on the fresh session).
- const finalPayload: IServiceCreateEventPayload = session
- ? merge(baseEvent, {
- referrer: session.current.referrer,
- referrerName: session.current.referrer_name,
- referrerType: session.current.referrer_type,
- } as Partial<IServiceCreateEventPayload>)
- : baseEvent;
-
- return createEventAndNotify(
- { ...finalPayload, id: eventId },
- logger,
- projectId
- );
+ return incomingEventCore(jobPayload, await getDeps(), meta);
}
diff --git a/apps/worker/src/jobs/events.incoming-events.test.ts b/apps/worker/src/jobs/events.incoming-events.test.ts
deleted file mode 100644
index 38d62d5a9..000000000
--- a/apps/worker/src/jobs/events.incoming-events.test.ts
+++ /dev/null
@@ -1,426 +0,0 @@
-import {
- createEvent,
- formatClickhouseDate,
- type IClickhouseSession,
- sessionBuffer,
-} from '@openpanel/db';
-import {
- type EventsQueuePayloadIncomingEvent,
- sessionsQueue,
-} from '@openpanel/queue';
-import type { Job } from 'bullmq';
-import { beforeEach, describe, expect, it, type Mock, vi } from 'vitest';
-import { incomingEvent } from './events.incoming-event';
-
-vi.mock('@openpanel/queue');
-vi.mock('@openpanel/db', async () => {
- const actual = await vi.importActual('@openpanel/db');
- return {
- ...actual,
- createEvent: vi.fn(),
- checkNotificationRulesForEvent: vi.fn().mockResolvedValue(true),
- getProjectByIdCached: vi.fn().mockResolvedValue({ filters: [] }),
- matchEvent: vi.fn().mockReturnValue(false),
- sessionBuffer: {
- // name/getBufferSize keep metrics.ts happy (it registers a per-buffer
- // gauge at import); the tests only drive getExistingSession/ingest.
- name: 'session',
- getBufferSize: vi.fn().mockResolvedValue(0),
- getExistingSession: vi.fn(),
- ingest: vi.fn(),
- },
- };
-});
-
-const projectId = 'test-project';
-const deviceId = 'device-123';
-const newSessionId = 'a1b2c3d4-e5f6-4789-a012-345678901234';
-// What the producer (/track) would have minted for one request.
-const producedEventId = '11111111-2222-4333-8444-555555555555';
-const geo = {
- country: 'US',
- city: 'New York',
- region: 'NY',
- longitude: 0,
- latitude: 0,
-};
-
-const uaInfo: EventsQueuePayloadIncomingEvent['payload']['uaInfo'] = {
- isServer: false,
- device: 'desktop',
- os: 'Windows',
- osVersion: '10',
- browser: 'Chrome',
- browserVersion: '91.0.4472.124',
- brand: '',
- model: '',
-};
-
-const uaInfoServer: EventsQueuePayloadIncomingEvent['payload']['uaInfo'] = {
- isServer: true,
- device: 'server',
- os: '',
- osVersion: '',
- browser: '',
- browserVersion: '',
- brand: '',
- model: '',
-};
-
-function makeSession(
- overrides: Partial<IClickhouseSession> = {}
-): IClickhouseSession {
- const now = new Date();
- return {
- id: 'session-existing',
- project_id: projectId,
- device_id: deviceId,
- profile_id: '',
- event_count: 1,
- screen_view_count: 0,
- entry_path: '/',
- entry_origin: 'https://example.com',
- exit_path: '/',
- exit_origin: 'https://example.com',
- created_at: formatClickhouseDate(now),
- ended_at: formatClickhouseDate(now),
- os: 'Windows',
- os_version: '10',
- browser: 'Chrome',
- browser_version: '91.0.4472.124',
- device: 'desktop',
- brand: '',
- model: '',
- country: 'US',
- region: 'NY',
- city: 'New York',
- longitude: 0,
- latitude: 0,
- duration: 0,
- referrer: '',
- referrer_name: '',
- referrer_type: '',
- is_bounce: true,
- utm_term: '',
- utm_source: '',
- utm_campaign: '',
- utm_content: '',
- utm_medium: '',
- revenue: 0,
- sign: 1,
- version: 1,
- groups: [],
- ...overrides,
- } satisfies IClickhouseSession;
-}
-
-function buildJobData(
- overrides: Partial<EventsQueuePayloadIncomingEvent['payload']> = {}
-): EventsQueuePayloadIncomingEvent['payload'] {
- return {
- geo,
- event: {
- name: 'test_event',
- timestamp: new Date().toISOString(),
- isTimestampFromThePast: false,
- properties: { __path: 'https://example.com/test' },
- },
- uaInfo,
- headers: {
- 'request-id': '123',
- 'user-agent': 'Mozilla/5.0',
- 'openpanel-sdk-name': 'web',
- 'openpanel-sdk-version': '1.0.0',
- },
- projectId,
- deviceId,
- sessionId: newSessionId,
- ...overrides,
- };
-}
-
-describe('incomingEvent', () => {
- beforeEach(() => {
- vi.clearAllMocks();
- (createEvent as Mock).mockImplementation((event) => event);
- });
-
- it('emits session_start when ingest returns kind="new"', async () => {
- vi.mocked(sessionBuffer.ingest).mockResolvedValueOnce({
- kind: 'new',
- current: makeSession({ id: newSessionId }),
- });
-
- await incomingEvent(buildJobData());
-
- expect(sessionsQueue.add).not.toHaveBeenCalled();
- const calls = (createEvent as Mock).mock.calls;
- expect(calls).toHaveLength(2);
- expect(calls[0]![0].name).toBe('session_start');
- expect(calls[1]![0].name).toBe('test_event');
- });
-
- it('skips session_start when ingest returns kind="extend"', async () => {
- vi.mocked(sessionBuffer.ingest).mockResolvedValueOnce({
- kind: 'extend',
- current: makeSession(),
- });
-
- await incomingEvent(buildJobData());
-
- expect(sessionsQueue.add).not.toHaveBeenCalled();
- const sessionStartCalls = (createEvent as Mock).mock.calls.filter(
- ([arg]) => arg?.name === 'session_start'
- );
- expect(sessionStartCalls).toHaveLength(0);
- });
-
- it('closes old session and emits session_start when ingest returns kind="boundary"', async () => {
- const closed = makeSession({ id: 'old-session-id' });
- const current = makeSession({ id: newSessionId });
- vi.mocked(sessionBuffer.ingest).mockResolvedValueOnce({
- kind: 'boundary',
- current,
- closed,
- });
-
- const spy = vi
- .spyOn(sessionsQueue, 'add')
- .mockResolvedValue({} as Job);
-
- await incomingEvent(buildJobData());
-
- expect(spy).toHaveBeenCalledTimes(1);
- const [, payload, opts] = spy.mock.calls[0]!;
- expect((payload as any).type).toBe('createSessionEnd');
- expect((payload as any).payload.sessionId).toBe('old-session-id');
- expect((payload as any).snapshot.id).toBe('old-session-id');
- expect(opts?.jobId).toBe('sessionEnd:v2:old-session-id');
-
- const calls = (createEvent as Mock).mock.calls;
- expect(calls.filter(([a]) => a?.name === 'session_start')).toHaveLength(1);
- expect(calls.filter(([a]) => a?.name === 'test_event')).toHaveLength(1);
- });
-
- it('inherits referrer from current session on the actual event', async () => {
- vi.mocked(sessionBuffer.ingest).mockResolvedValueOnce({
- kind: 'extend',
- current: makeSession({
- referrer: 'https://google.com',
- referrer_name: 'Google',
- referrer_type: 'search',
- }),
- });
-
- await incomingEvent(buildJobData());
-
- const testEventCall = (createEvent as Mock).mock.calls.find(
- ([a]) => a?.name === 'test_event'
- );
- expect(testEventCall![0].referrer).toBe('https://google.com');
- expect(testEventCall![0].referrerName).toBe('Google');
- expect(testEventCall![0].referrerType).toBe('search');
- });
-
- it('handles server events with existing profile session', async () => {
- const timestamp = new Date();
- const jobData = buildJobData({
- event: {
- name: 'server_event',
- timestamp: timestamp.toISOString(),
- properties: { custom_property: 'test_value' },
- profileId: 'profile-123',
- isTimestampFromThePast: false,
- },
- uaInfo: uaInfoServer,
- deviceId: '',
- sessionId: '',
- headers: {
- 'user-agent': 'OpenPanel Server/1.0',
- 'openpanel-sdk-name': 'server',
- 'openpanel-sdk-version': '1.0.0',
- 'request-id': '123',
- },
- });
-
- vi.mocked(sessionBuffer.getExistingSession).mockResolvedValueOnce(
- makeSession({
- id: 'last-session-456',
- device_id: 'last-device-123',
- profile_id: 'profile-123',
- os: 'iOS',
- os_version: '15.0',
- browser: 'Safari',
- browser_version: '15.0',
- device: 'mobile',
- brand: 'Apple',
- model: 'iPhone',
- country: 'CA',
- region: 'ON',
- city: 'Toronto',
- entry_path: '/last-path',
- entry_origin: 'https://example.com',
- exit_path: '/last-path',
- exit_origin: 'https://example.com',
- referrer: 'https://google.com',
- referrer_name: 'Google',
- referrer_type: 'search',
- is_bounce: false,
- event_count: 0,
- screen_view_count: 0,
- })
- );
-
- await incomingEvent(jobData);
-
- expect(sessionBuffer.ingest).not.toHaveBeenCalled();
- expect(sessionsQueue.add).not.toHaveBeenCalled();
- expect((createEvent as Mock).mock.calls[0]![0]).toMatchObject({
- name: 'server_event',
- deviceId: 'last-device-123',
- sessionId: 'last-session-456',
- profileId: 'profile-123',
- city: 'Toronto',
- country: 'CA',
- referrer: 'https://google.com',
- });
- });
-
- it('handles server events without any active session', async () => {
- vi.mocked(sessionBuffer.getExistingSession).mockResolvedValueOnce(null);
-
- await incomingEvent(
- buildJobData({
- event: {
- name: 'server_event',
- timestamp: new Date().toISOString(),
- properties: { custom_property: 'test_value' },
- profileId: 'profile-123',
- isTimestampFromThePast: false,
- },
- uaInfo: uaInfoServer,
- deviceId: '',
- sessionId: '',
- headers: {
- 'user-agent': 'OpenPanel Server/1.0',
- 'openpanel-sdk-name': 'server',
- 'openpanel-sdk-version': '1.0.0',
- 'request-id': '123',
- },
- })
- );
-
- expect(sessionBuffer.ingest).not.toHaveBeenCalled();
- expect(sessionsQueue.add).not.toHaveBeenCalled();
- expect((createEvent as Mock).mock.calls[0]![0]).toMatchObject({
- name: 'server_event',
- deviceId: '',
- sessionId: '',
- profileId: 'profile-123',
- });
- });
-
- it('re-creates the same row id when Kafka redelivers the message', async () => {
- const session = makeSession({ id: newSessionId });
- vi.mocked(sessionBuffer.ingest)
- .mockResolvedValueOnce({ kind: 'new', current: session })
- .mockResolvedValueOnce({ kind: 'extend', current: session });
-
- // Same message, delivered twice: identical payload, identical offset.
- const jobData = buildJobData({ id: producedEventId });
- const delivery = { partition: 3, offset: '42' };
- await incomingEvent(jobData, delivery);
- await incomingEvent(jobData, delivery);
-
- const eventCalls = (createEvent as Mock).mock.calls.filter(
- ([a]) => a?.name === 'test_event'
- );
- expect(eventCalls).toHaveLength(2);
- expect(eventCalls[0]![0].id).toBe(producedEventId);
- expect(eventCalls[1]![0].id).toBe(producedEventId);
-
- // The derived session_start row must NOT reuse the event's id.
- const sessionStartCalls = (createEvent as Mock).mock.calls.filter(
- ([a]) => a?.name === 'session_start'
- );
- expect(sessionStartCalls).toHaveLength(1);
- expect(sessionStartCalls[0]![0].id).toBeUndefined();
- });
-
- it('carries the producer-minted id on server-side events', async () => {
- vi.mocked(sessionBuffer.getExistingSession).mockResolvedValueOnce(null);
-
- await incomingEvent(
- buildJobData({
- id: producedEventId,
- event: {
- name: 'server_event',
- timestamp: new Date().toISOString(),
- isTimestampFromThePast: false,
- profileId: 'profile-123',
- },
- uaInfo: uaInfoServer,
- deviceId: '',
- sessionId: '',
- })
- );
-
- expect((createEvent as Mock).mock.calls[0]![0].id).toBe(producedEventId);
- });
-
- it('leaves the id undefined for a payload produced without one', async () => {
- vi.mocked(sessionBuffer.ingest).mockResolvedValueOnce({
- kind: 'extend',
- current: makeSession(),
- });
-
- await incomingEvent(buildJobData());
-
- expect((createEvent as Mock).mock.calls[0]![0].id).toBeUndefined();
- });
-
- it('emits session_start only once across 3 rapid events (new → extend → extend)', async () => {
- const session = makeSession({ id: newSessionId });
- vi.mocked(sessionBuffer.ingest)
- .mockResolvedValueOnce({ kind: 'new', current: session })
- .mockResolvedValueOnce({ kind: 'extend', current: session })
- .mockResolvedValueOnce({ kind: 'extend', current: session });
-
- await incomingEvent(
- buildJobData({
- event: {
- name: 'e1',
- timestamp: new Date().toISOString(),
- isTimestampFromThePast: false,
- properties: { __path: 'https://example.com/test' },
- },
- })
- );
- await incomingEvent(
- buildJobData({
- event: {
- name: 'e2',
- timestamp: new Date().toISOString(),
- isTimestampFromThePast: false,
- properties: { __path: 'https://example.com/test' },
- },
- })
- );
- await incomingEvent(
- buildJobData({
- event: {
- name: 'e3',
- timestamp: new Date().toISOString(),
- isTimestampFromThePast: false,
- properties: { __path: 'https://example.com/test' },
- },
- })
- );
-
- const sessionStartCalls = (createEvent as Mock).mock.calls.filter(
- ([a]) => a?.name === 'session_start'
- );
- expect(sessionStartCalls).toHaveLength(1);
- expect(sessionsQueue.add).not.toHaveBeenCalled();
- });
-});
diff --git a/apps/worker/src/jobs/events.kafka-consumer.ts b/apps/worker/src/jobs/events.kafka-consumer.ts
index 0454f7e01..bde98df53 100644
--- a/apps/worker/src/jobs/events.kafka-consumer.ts
+++ b/apps/worker/src/jobs/events.kafka-consumer.ts
@@ -1,14 +1,18 @@
+// Dissolved into @openpanel/core's ingest module (M8-003): the batch handler
+// (per-key serial groups, the ascending contiguous-prefix offset walk,
+// at-least-once, bounded retry, DLQ) is
+// packages/core/src/modules/ingest/src/consumer.ts. This file stays (DELEGATE
+// PATTERN) and supplies the halves core cannot reach: @openpanel/queue's
+// kafkajs client and its topic / consumer-group / retry constants, this
+// worker's registry counters, and the events heartbeat.
+import { startKafkaEventsConsumer as startKafkaEventsConsumerCore } from '@openpanel/core';
import {
createKafkaEventsConsumer,
- type DeadLetterMessage,
- type EachBatchPayload,
- type EventsQueuePayloadIncomingEvent,
KAFKA_EVENTS_TOPIC,
KAFKA_HANDLER_MAX_ATTEMPTS,
KAFKA_HANDLER_RETRY_INITIAL_MS,
KAFKA_HANDLER_RETRY_MAX_MS,
KAFKA_PARTITIONS_CONCURRENT,
- type KafkaMessage,
kafkaLogger,
produceDeadLetterEvent,
} from '@openpanel/queue';
@@ -22,385 +26,32 @@ import { logger } from '../utils/logger';
import { markEventsActivity } from '../utils/worker-heartbeat';
import { incomingEvent } from './events.incoming-event';
-export interface KafkaConsumerHandle {
- stop: () => Promise<void>;
-}
-
-// Heartbeat every N messages within a per-key group. The default kafkajs
-// sessionTimeout is 30s — calling heartbeat every 16 messages keeps us
-// comfortably under that even for slow handlers.
-const HEARTBEAT_EVERY = 16;
-
-const RETRY_BACKOFF_FACTOR = 2;
-
-/** Why a message was dead-lettered. A label value — keep the set small. */
-type DeadLetterReason = 'parse_error' | 'handler_error';
-
-interface ConsumerLogger {
- info: (obj: object, msg?: string) => void;
- warn: (obj: object, msg?: string) => void;
- error: (obj: object, msg?: string) => void;
-}
-
-interface ConsumerMetrics {
- reprocessed: (partition: string) => void;
- handlerFailed: (partition: string) => void;
- deadLettered: (partition: string, reason: DeadLetterReason) => void;
- deadLetterFailed: (partition: string) => void;
-}
-
-export interface EventsBatchHandlerDeps {
- handleEvent: (
- payload: EventsQueuePayloadIncomingEvent['payload'],
- meta: { partition: number; offset: string }
- ) => Promise<unknown>;
- sendToDeadLetter: (message: DeadLetterMessage) => Promise<void>;
- logger: ConsumerLogger;
- metrics: ConsumerMetrics;
- onActivity: () => void;
- topic: string;
- maxAttempts: number;
- initialRetryMs: number;
- maxRetryMs: number;
- sleep: (ms: number) => Promise<void>;
-}
-
-const defaultSleep = (ms: number): Promise<void> =>
- new Promise((resolve) => setTimeout(resolve, ms));
-
-const errorMessage = (err: unknown): string =>
- err instanceof Error ? err.message : String(err);
-
-const defaultDeps = (): EventsBatchHandlerDeps => ({
- handleEvent: incomingEvent,
- sendToDeadLetter: produceDeadLetterEvent,
- logger,
- metrics: {
- reprocessed: (partition) => kafkaReprocessedTotal.inc({ partition }),
- handlerFailed: (partition) => kafkaHandlerFailuresTotal.inc({ partition }),
- deadLettered: (partition, reason) =>
- kafkaDeadLetteredTotal.inc({ partition, reason }),
- deadLetterFailed: (partition) =>
- kafkaDeadLetterFailedTotal.inc({ partition }),
- },
- onActivity: markEventsActivity,
- topic: KAFKA_EVENTS_TOPIC,
- maxAttempts: KAFKA_HANDLER_MAX_ATTEMPTS,
- initialRetryMs: KAFKA_HANDLER_RETRY_INITIAL_MS,
- maxRetryMs: KAFKA_HANDLER_RETRY_MAX_MS,
- sleep: defaultSleep,
-});
-
-export interface EventsBatchHandler {
- eachBatch: (payload: EachBatchPayload) => Promise<void>;
- /** Called on GROUP_JOIN: a new assignment invalidates the watermarks. */
- resetWatermarks: () => void;
-}
-
-/**
- * The `eachBatch` handler, with its dependencies injected so the offset walk,
- * the per-key serial groups and the failure paths are testable without a
- * broker.
- *
- * Delivery contract (ADR-004): at-least-once. An offset is resolved only once
- * its message has been handled, dead-lettered, or deliberately skipped —
- * never merely because it failed.
- */
-export function createEventsBatchHandler(
- overrides: Partial<EventsBatchHandlerDeps> = {}
-): EventsBatchHandler {
- const deps: EventsBatchHandlerDeps = { ...defaultDeps(), ...overrides };
-
- // Highest Kafka offset we have *resolved* (committed) per `topic-partition`,
- // tracked across batches. Used purely for the reprocess detector below: if we
- // ever see an offset at or below this watermark again, the message is being
- // redelivered (at-least-once duplicate) outside of a rebalance — which is the
- // signature of an offset-handling bug. Cleared on GROUP_JOIN so legitimate
- // post-rebalance redelivery from the last committed offset doesn't trip it.
- const resolvedHWM = new Map<string, number>();
-
- const deadLetter = async (
- message: KafkaMessage,
- partition: number,
- reason: DeadLetterReason,
- err: unknown
- ): Promise<boolean> => {
- const label = String(partition);
- try {
- await deps.sendToDeadLetter({
- key: message.key,
- value: message.value,
- headers: message.headers,
- topic: deps.topic,
- partition,
- offset: message.offset,
- reason,
- error: errorMessage(err),
- });
- } catch (dlqErr) {
- // The event is not lost: leaving the offset unresolved stops the
- // contiguous-prefix walk here, so the broker redelivers this message.
- deps.metrics.deadLetterFailed(label);
- deps.logger.error(
- { err: dlqErr, partition, offset: message.offset, reason },
- 'kafka dead-letter produce failed — offset left unresolved for redelivery'
- );
- return false;
- }
- deps.metrics.deadLettered(label, reason);
- deps.logger.error(
- { err, partition, offset: message.offset, reason },
- 'kafka message dead-lettered'
- );
- return true;
- };
-
- const runHandlerWithRetry = async (
- payload: EventsQueuePayloadIncomingEvent['payload'],
- partition: number,
- offset: string
- ): Promise<{ ok: true } | { ok: false; err: unknown }> => {
- const label = String(partition);
- let delayMs = deps.initialRetryMs;
- let lastErr: unknown;
-
- for (let attempt = 1; attempt <= deps.maxAttempts; attempt++) {
- try {
- await deps.handleEvent(payload, { partition, offset });
- return { ok: true };
- } catch (err) {
- lastErr = err;
- deps.metrics.handlerFailed(label);
- deps.logger.warn(
- {
- err,
- partition,
- offset,
- projectId: payload.projectId,
- attempt,
- maxAttempts: deps.maxAttempts,
- },
- 'kafka incomingEvent handler failed'
- );
- if (attempt < deps.maxAttempts) {
- await deps.sleep(delayMs);
- delayMs = Math.min(delayMs * RETRY_BACKOFF_FACTOR, deps.maxRetryMs);
- }
- }
- }
-
- return { ok: false, err: lastErr };
- };
-
- /** @returns true when the offset may be resolved. */
- const processMessage = async (
- message: KafkaMessage,
- partition: number
- ): Promise<boolean> => {
- let payload: EventsQueuePayloadIncomingEvent['payload'];
- try {
- if (!message.value) {
- throw new Error('kafka message has no value');
- }
- payload = JSON.parse(
- message.value.toString()
- ) as EventsQueuePayloadIncomingEvent['payload'];
- } catch (err) {
- deps.logger.error(
- { err, partition, offset: message.offset },
- 'kafka message parse failed'
- );
- // Deterministic: retrying identical bytes cannot succeed.
- return deadLetter(message, partition, 'parse_error', err);
- }
-
- const result = await runHandlerWithRetry(
- payload,
- partition,
- message.offset
- );
- if (result.ok) {
- return true;
- }
- return deadLetter(message, partition, 'handler_error', result.err);
- };
-
- const eachBatch = async ({
- batch,
- resolveOffset,
- heartbeat,
- isRunning,
- isStale,
- }: EachBatchPayload): Promise<void> => {
- if (batch.messages.length === 0) {
- return;
- }
-
- const pk = `${batch.topic}-${batch.partition}`;
- // Watermark from *previous* batches. Anything at or below this that we see
- // now is a redelivery. Captured before this batch so intra-batch
- // out-of-order processing (normal, see below) is never counted.
- const priorHWM = resolvedHWM.get(pk) ?? -1;
-
- // Group by partition key (= deviceId or `${projectId}:${profileId}`).
- // Same-key messages stay serial so sessionBuffer/session-end-job state
- // can't race; different keys run in parallel via Promise.all.
- // Keyless messages get their own singleton group.
- const groups = new Map<string, KafkaMessage[]>();
- for (const m of batch.messages) {
- const key = m.key ? m.key.toString() : `__no_key__:${m.offset}`;
- const arr = groups.get(key);
- if (arr) {
- arr.push(m);
- } else {
- groups.set(key, [m]);
- }
- }
+export type { KafkaConsumerHandle } from '@openpanel/core';
- // Offsets that finished processing this batch. We resolve them AFTER all
- // groups complete, in strict ascending order (see the loop below).
- //
- // Why: KafkaJS `resolveOffset` is last-write-wins and the next fetch
- // starts from the last resolved offset (offsetManager.nextOffset). If we
- // resolved inside the concurrent per-key loop, a lower offset resolving
- // after a higher one would move the fetch position BACKWARDS and
- // re-deliver everything in between — the root cause of the duplicate
- // events. Resolving the contiguous ascending prefix at the end avoids
- // both duplicates (never regress) and loss (stop at the first gap).
- const processed = new Set<string>();
- let processedCount = 0;
-
- await Promise.all(
- [...groups.values()].map(async (msgs) => {
- for (const m of msgs) {
- if (!isRunning() || isStale()) {
- return;
- }
-
- // Reprocess detector: only fires for offsets already resolved in a
- // PRIOR batch (redelivery). Intra-batch out-of-order processing
- // across key-groups is expected and is not flagged.
- if (Number(m.offset) <= priorHWM) {
- deps.metrics.reprocessed(String(batch.partition));
- deps.logger.warn(
- {
- partition: batch.partition,
- offset: m.offset,
- resolvedHighWaterMark: priorHWM,
- },
- 'kafka offset REPROCESSED — at-least-once duplicate (outside rebalance)'
- );
- }
-
- const resolvable = await processMessage(m, batch.partition);
- if (!resolvable) {
- // Stop this key-group: continuing past an unresolvable offset only
- // adds duplicates, since the whole tail is redelivered anyway.
- return;
- }
-
- processed.add(m.offset);
- processedCount += 1;
- if (processedCount % HEARTBEAT_EVERY === 0) {
- await heartbeat();
- }
- }
- })
- );
-
- // Resolve in strict ascending offset order, stopping at the first offset
- // that did not finish (e.g. an isStale/isRunning early-return mid-batch).
- // batch.messages is already ordered by offset.
- let newHWM = priorHWM;
- for (const m of batch.messages) {
- if (!processed.has(m.offset)) {
- break;
- }
- resolveOffset(m.offset);
- newHWM = Math.max(newHWM, Number(m.offset));
- }
- resolvedHWM.set(pk, newHWM);
-
- await heartbeat();
- deps.onActivity();
- };
-
- return { eachBatch, resetWatermarks: () => resolvedHWM.clear() };
-}
-
-export async function startKafkaEventsConsumer(): Promise<KafkaConsumerHandle> {
- const consumer = createKafkaEventsConsumer();
- const handler = createEventsBatchHandler();
- await consumer.connect();
- await consumer.subscribe({
+export function startKafkaEventsConsumer() {
+ return startKafkaEventsConsumerCore({
+ createConsumer: createKafkaEventsConsumer,
+ logger,
+ kafkaLogger,
topic: KAFKA_EVENTS_TOPIC,
- fromBeginning: false,
- });
-
- consumer.on(consumer.events.HEARTBEAT, markEventsActivity);
-
- // ---- Lifecycle / rebalance ("re-election") visibility ----
- // We had no logging for partition reassignment before; without it a rebalance
- // storm (a common source of at-least-once duplicates) is invisible.
- consumer.on(consumer.events.GROUP_JOIN, ({ payload }) => {
- // A new assignment means partitions may have moved between members; reset
- // the reprocess watermarks so legitimate resume-from-committed-offset after
- // a rebalance is not flagged as a duplicate.
- handler.resetWatermarks();
- logger.info(
- {
- memberId: payload.memberId,
- groupId: payload.groupId,
- isLeader: payload.isLeader,
- memberAssignment: payload.memberAssignment,
- duration: payload.duration,
- },
- 'kafka consumer joined group (rebalance complete)'
- );
- });
- consumer.on(consumer.events.REBALANCING, ({ payload }) => {
- logger.warn(
- { memberId: payload.memberId, groupId: payload.groupId },
- 'kafka consumer rebalancing'
- );
- });
- consumer.on(consumer.events.CRASH, ({ payload }) => {
- logger.error(
- {
- err: payload.error,
- groupId: payload.groupId,
- restart: payload.restart,
- },
- 'kafka consumer crashed'
- );
- });
- consumer.on(consumer.events.DISCONNECT, () => {
- logger.warn('kafka consumer disconnected');
- });
- consumer.on(consumer.events.REQUEST_TIMEOUT, ({ payload }) => {
- logger.warn(
- { broker: payload.broker, clientId: payload.clientId },
- 'kafka consumer request timeout'
- );
- });
-
- await consumer.run({
partitionsConsumedConcurrently: KAFKA_PARTITIONS_CONCURRENT,
- eachBatchAutoResolve: false,
- eachBatch: handler.eachBatch,
- });
-
- kafkaLogger.info(
- {
- topic: KAFKA_EVENTS_TOPIC,
- partitionsConsumedConcurrently: KAFKA_PARTITIONS_CONCURRENT,
- },
- 'kafka events consumer running'
- );
-
- return {
- stop: async () => {
- await consumer.disconnect();
+ batch: {
+ handleEvent: incomingEvent,
+ sendToDeadLetter: produceDeadLetterEvent,
+ logger,
+ metrics: {
+ reprocessed: (partition) => kafkaReprocessedTotal.inc({ partition }),
+ handlerFailed: (partition) =>
+ kafkaHandlerFailuresTotal.inc({ partition }),
+ deadLettered: (partition, reason) =>
+ kafkaDeadLetteredTotal.inc({ partition, reason }),
+ deadLetterFailed: (partition) =>
+ kafkaDeadLetterFailedTotal.inc({ partition }),
+ },
+ onActivity: markEventsActivity,
+ maxAttempts: KAFKA_HANDLER_MAX_ATTEMPTS,
+ initialRetryMs: KAFKA_HANDLER_RETRY_INITIAL_MS,
+ maxRetryMs: KAFKA_HANDLER_RETRY_MAX_MS,
},
- };
+ });
}
diff --git a/packages/core/package.json b/packages/core/package.json
index fa3a1d6a4..88934ae44 100644
--- a/packages/core/package.json
+++ b/packages/core/package.json
@@ -52,6 +52,7 @@
"fast-json-stable-hash": "^1.0.3",
"ipaddr.js": "^2.5.0",
"js-yaml": "^4.1.0",
+ "kafkajs": "^2.2.4",
"lru-cache": "^11.2.4",
"mathjs": "^12.3.2",
"nanoid": "^5.1.6",
diff --git a/packages/core/src/context.ts b/packages/core/src/context.ts
index 1f1f7f4c5..16824e9a9 100644
--- a/packages/core/src/context.ts
+++ b/packages/core/src/context.ts
@@ -52,8 +52,9 @@ export interface AppDeps {
/**
* The events transport. `produceIncomingEvent` still lives in
* @openpanel/queue, which imports core for its logger, so core cannot
- * import it back (M8-002); main.ts supplies it and M8-003 replaces it with
- * core's own Kafka producer.
+ * import it back (M8-002); main.ts supplies it. It stays injected until
+ * @openpanel/queue dies at P9 — M8-003 moved the consumer, not the
+ * producer, and the consumer takes its kafkajs client the same way.
*/
produceIncomingEvent: IncomingEventProducer;
logger: Logger;
diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts
index 79253497d..82177897a 100644
--- a/packages/core/src/index.ts
+++ b/packages/core/src/index.ts
@@ -1151,6 +1151,34 @@ export {
summarizeBotSignals,
validateIngestRequest,
} from './modules/ingest/ingest.service';
+// The Kafka consumer and its per-message handler (M8-003). apps/worker's
+// events.kafka-consumer.ts / events.incoming-event.ts are thin delegates that
+// hand in the kafkajs client, the topic/group names and their own registry's
+// counters — none of which this package spells, so they stay byte-identical.
+export type {
+ ConsumerLogger,
+ ConsumerMetrics,
+ DeadLetterMessage,
+ DeadLetterReason,
+ EventsBatchHandler,
+ EventsBatchHandlerDeps,
+ EventsConsumerDeps,
+ KafkaConsumerHandle,
+} from './modules/ingest/src/consumer';
+export {
+ createEventsBatchHandler,
+ startKafkaEventsConsumer,
+} from './modules/ingest/src/consumer';
+export type {
+ IncomingEventDelivery,
+ IncomingEventDeps,
+ IncomingEventMetrics,
+ IncomingEventSessions,
+} from './modules/ingest/src/incoming-event-handler';
+export {
+ incomingEvent,
+ loadIncomingEventDeps,
+} from './modules/ingest/src/incoming-event-handler';
// Dissolved from @openpanel/db's services/insights* + referrer-spikes.service
// (M5-001) — apps/worker's insight job files and packages/trpc's insight
// router call these directly, the same way V1 reaches every other dissolved
diff --git a/apps/worker/src/jobs/events.kafka-consumer.test.ts b/packages/core/src/modules/ingest/src/consumer.test.ts
similarity index 77%
rename from apps/worker/src/jobs/events.kafka-consumer.test.ts
rename to packages/core/src/modules/ingest/src/consumer.test.ts
index 4c0d6c9d6..f4645d4cf 100644
--- a/apps/worker/src/jobs/events.kafka-consumer.test.ts
+++ b/packages/core/src/modules/ingest/src/consumer.test.ts
@@ -3,32 +3,19 @@
* An offset is resolved once its message has been handled, dead-lettered, or
* deliberately skipped — never merely because it failed.
*
- * The batch handler is exercised directly with injected dependencies; the
- * broker, the metrics registry and `incomingEvent` are all out of the picture.
+ * Ported from apps/worker/src/jobs/events.kafka-consumer.test.ts (M8-003)
+ * with the assertions unchanged. The batch handler is exercised directly with
+ * injected dependencies; the broker, the metrics registry and `incomingEvent`
+ * are all out of the picture — which is why no `mock.module` appears here.
*/
-import type {
- DeadLetterMessage,
- EachBatchPayload,
- KafkaMessage,
-} from '@openpanel/queue';
-import { beforeEach, describe, expect, it, vi } from 'vitest';
-
-vi.mock('../metrics', () => ({
- kafkaReprocessedTotal: { inc: vi.fn() },
- kafkaHandlerFailuresTotal: { inc: vi.fn() },
- kafkaDeadLetteredTotal: { inc: vi.fn() },
- kafkaDeadLetterFailedTotal: { inc: vi.fn() },
-}));
-vi.mock('./events.incoming-event', () => ({ incomingEvent: vi.fn() }));
-vi.mock('../utils/logger', () => ({
- logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() },
-}));
-
+import { describe, expect, mock, test } from 'bun:test';
+import type { EachBatchPayload, KafkaMessage } from 'kafkajs';
import {
createEventsBatchHandler,
+ type DeadLetterMessage,
type EventsBatchHandlerDeps,
-} from './events.kafka-consumer';
+} from './consumer';
const PARTITION = 7;
const TOPIC = 'events';
@@ -57,8 +44,8 @@ function makeBatch(
messages: KafkaMessage[],
options: { isRunning?: () => boolean; isStale?: () => boolean } = {}
) {
- const resolveOffset = vi.fn();
- const heartbeat = vi.fn().mockResolvedValue(undefined);
+ const resolveOffset = mock((_offset: string) => undefined);
+ const heartbeat = mock(async () => undefined);
const payload = {
batch: { topic: TOPIC, partition: PARTITION, messages },
resolveOffset,
@@ -69,44 +56,56 @@ function makeBatch(
return { payload, resolveOffset, heartbeat };
}
-const resolvedOffsets = (resolveOffset: ReturnType<typeof vi.fn>): string[] =>
+interface MockCalls {
+ mock: { calls: unknown[][] };
+}
+
+const resolvedOffsets = (resolveOffset: MockCalls): string[] =>
resolveOffset.mock.calls.map(([offset]) => offset as string);
function makeDeps(overrides: Partial<EventsBatchHandlerDeps> = {}) {
- const metrics = {
- reprocessed: vi.fn(),
- handlerFailed: vi.fn(),
- deadLettered: vi.fn(),
- deadLetterFailed: vi.fn(),
+ const metrics: EventsBatchHandlerDeps['metrics'] = {
+ reprocessed: mock(() => undefined),
+ handlerFailed: mock(() => undefined),
+ deadLettered: mock(() => undefined),
+ deadLetterFailed: mock(() => undefined),
};
- const deps: Partial<EventsBatchHandlerDeps> = {
- handleEvent: vi.fn().mockResolvedValue(undefined),
- sendToDeadLetter: vi.fn().mockResolvedValue(undefined),
- logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() },
- onActivity: vi.fn(),
+ const handleEvent = mock<EventsBatchHandlerDeps['handleEvent']>(
+ async () => undefined
+ );
+ const sendToDeadLetter = mock<EventsBatchHandlerDeps['sendToDeadLetter']>(
+ async () => undefined
+ );
+ const sleep = mock(async (_ms: number) => undefined);
+ const deps: EventsBatchHandlerDeps = {
+ handleEvent,
+ sendToDeadLetter,
+ logger: {
+ info: mock(() => undefined),
+ warn: mock(() => undefined),
+ error: mock(() => undefined),
+ },
+ onActivity: mock(() => undefined),
topic: TOPIC,
maxAttempts: 3,
initialRetryMs: 10,
maxRetryMs: 25,
- sleep: vi.fn().mockResolvedValue(undefined),
+ sleep,
...overrides,
metrics: { ...metrics, ...overrides.metrics },
};
return {
deps,
metrics: deps.metrics as typeof metrics,
- handleEvent: deps.handleEvent as ReturnType<typeof vi.fn>,
- sendToDeadLetter: deps.sendToDeadLetter as ReturnType<typeof vi.fn>,
- sleep: deps.sleep as ReturnType<typeof vi.fn>,
+ handleEvent,
+ sendToDeadLetter: (overrides.sendToDeadLetter ??
+ sendToDeadLetter) as typeof sendToDeadLetter,
+ sleep,
};
}
-beforeEach(() => {
- vi.clearAllMocks();
-});
-
describe('offset walk', () => {
- it('resolves the whole contiguous prefix in ascending order', async () => {
+ test('resolves the whole contiguous prefix in ascending order', async () => {
const { deps } = makeDeps();
const handler = createEventsBatchHandler(deps);
// Two interleaved key-groups: they run concurrently, so completion order
@@ -123,7 +122,7 @@ describe('offset walk', () => {
expect(resolvedOffsets(resolveOffset)).toEqual(['0', '1', '2', '3']);
});
- it('stops at the first gap, even when later offsets did finish', async () => {
+ test('stops at the first gap, even when later offsets did finish', async () => {
// Keys 'a' (0, 2, 4) and 'b' (1, 3, 5) run concurrently. 'a' is held on
// its first message until 'b' has finished all of its own, then the
// consumer stops — so 3 and 5 are processed while 2 never runs. Resolving
@@ -158,15 +157,13 @@ describe('offset walk', () => {
await handler.eachBatch(payload);
- const handled = handleEvent.mock.calls.map(
- ([p]) => (p as { projectId: string }).projectId
- );
+ const handled = handleEvent.mock.calls.map(([p]) => p.projectId);
expect(handled).toContain('project-5');
expect(handled).not.toContain('project-2');
expect(resolvedOffsets(resolveOffset)).toEqual(['0', '1']);
});
- it('resolves nothing when the batch is stale before the first message', async () => {
+ test('resolves nothing when the batch is stale before the first message', async () => {
const { deps, handleEvent } = makeDeps();
const handler = createEventsBatchHandler(deps);
const { payload, resolveOffset } = makeBatch(
@@ -182,7 +179,7 @@ describe('offset walk', () => {
});
describe('kill consumer mid-batch', () => {
- it('loses no event: every unresolved offset is left for redelivery', async () => {
+ test('loses no event: every unresolved offset is left for redelivery', async () => {
const KILL_AFTER = 3;
const TOTAL = 8;
let handled = 0;
@@ -215,11 +212,11 @@ describe('kill consumer mid-batch', () => {
});
describe('handler failure', () => {
- it('retries with bounded backoff before giving up', async () => {
+ test('retries with bounded backoff before giving up', async () => {
const { deps, handleEvent, sleep, sendToDeadLetter } = makeDeps({
maxAttempts: 4,
});
- handleEvent.mockRejectedValue(new Error('boom'));
+ handleEvent.mockImplementation(() => Promise.reject(new Error('boom')));
const handler = createEventsBatchHandler(deps);
const { payload } = makeBatch([message(0, 'a')]);
@@ -231,11 +228,15 @@ describe('handler failure', () => {
expect(sendToDeadLetter).toHaveBeenCalledTimes(1);
});
- it('stops retrying as soon as the handler succeeds', async () => {
+ test('stops retrying as soon as the handler succeeds', async () => {
const { deps, handleEvent, sendToDeadLetter, metrics } = makeDeps();
- handleEvent
- .mockRejectedValueOnce(new Error('transient'))
- .mockResolvedValue(undefined);
+ let attempts = 0;
+ handleEvent.mockImplementation(() => {
+ attempts += 1;
+ return attempts === 1
+ ? Promise.reject(new Error('transient'))
+ : Promise.resolve(undefined);
+ });
const handler = createEventsBatchHandler(deps);
const { payload, resolveOffset } = makeBatch([message(0, 'a')]);
@@ -247,7 +248,7 @@ describe('handler failure', () => {
expect(resolvedOffsets(resolveOffset)).toEqual(['0']);
});
- it('dead-letters the raw message, counts it, and keeps the partition moving', async () => {
+ test('dead-letters the raw message, counts it, and keeps the partition moving', async () => {
const poison = 'poison-payload';
const { deps, handleEvent, sendToDeadLetter, metrics } = makeDeps();
handleEvent.mockImplementation(async (p: { projectId: string }) => {
@@ -282,9 +283,9 @@ describe('handler failure', () => {
expect(resolvedOffsets(resolveOffset)).toEqual(['0', '1', '2']);
});
- it('leaves the offset unresolved when the dead-letter produce fails', async () => {
+ test('leaves the offset unresolved when the dead-letter produce fails', async () => {
const { deps, handleEvent, metrics } = makeDeps({
- sendToDeadLetter: vi.fn().mockRejectedValue(new Error('broker down')),
+ sendToDeadLetter: mock(() => Promise.reject(new Error('broker down'))),
});
handleEvent.mockImplementation(async (p: { projectId: string }) => {
if (p.projectId === 'project-1') {
@@ -307,7 +308,7 @@ describe('handler failure', () => {
});
describe('unparseable messages', () => {
- it('dead-letters instead of dropping, without retrying', async () => {
+ test('dead-letters instead of dropping, without retrying', async () => {
const { deps, handleEvent, sendToDeadLetter, metrics, sleep } = makeDeps();
const handler = createEventsBatchHandler(deps);
const { payload, resolveOffset } = makeBatch([
@@ -330,7 +331,7 @@ describe('unparseable messages', () => {
expect(resolvedOffsets(resolveOffset)).toEqual(['0']);
});
- it('dead-letters a message with no value at all', async () => {
+ test('dead-letters a message with no value at all', async () => {
const { deps, sendToDeadLetter, metrics } = makeDeps();
const handler = createEventsBatchHandler(deps);
const { payload, resolveOffset } = makeBatch([message(0, 'a', '')]);
@@ -347,7 +348,7 @@ describe('unparseable messages', () => {
});
describe('reprocess counter', () => {
- it('does not fire for intra-batch out-of-order processing', async () => {
+ test('does not fire for intra-batch out-of-order processing', async () => {
const { deps, metrics } = makeDeps();
const handler = createEventsBatchHandler(deps);
const { payload } = makeBatch([
@@ -361,7 +362,7 @@ describe('reprocess counter', () => {
expect(metrics.reprocessed).not.toHaveBeenCalled();
});
- it('fires once per redelivered offset in a later batch', async () => {
+ test('fires once per redelivered offset in a later batch', async () => {
const { deps, metrics } = makeDeps();
const handler = createEventsBatchHandler(deps);
const messages = [message(0, 'a'), message(1, 'a')];
@@ -374,7 +375,7 @@ describe('reprocess counter', () => {
expect(metrics.reprocessed).toHaveBeenCalledWith(String(PARTITION));
});
- it('does not fire after a rebalance clears the watermarks', async () => {
+ test('does not fire after a rebalance clears the watermarks', async () => {
const { deps, metrics } = makeDeps();
const handler = createEventsBatchHandler(deps);
const messages = [message(0, 'a'), message(1, 'a')];
@@ -386,12 +387,12 @@ describe('reprocess counter', () => {
expect(metrics.reprocessed).not.toHaveBeenCalled();
});
- it('does not fire for a dead-lettered offset re-seen only within the batch', async () => {
+ test('does not fire for a dead-lettered offset re-seen only within the batch', async () => {
// A dead-lettered offset is resolved, so it must move the watermark just
// like a handled one — otherwise the next batch would flag it as a
// reprocess.
const { deps, handleEvent, metrics } = makeDeps();
- handleEvent.mockRejectedValue(new Error('boom'));
+ handleEvent.mockImplementation(() => Promise.reject(new Error('boom')));
const handler = createEventsBatchHandler(deps);
await handler.eachBatch(makeBatch([message(0, 'a')]).payload);
diff --git a/packages/core/src/modules/ingest/src/consumer.ts b/packages/core/src/modules/ingest/src/consumer.ts
new file mode 100644
index 000000000..7408baa29
--- /dev/null
+++ b/packages/core/src/modules/ingest/src/consumer.ts
@@ -0,0 +1,416 @@
+// Ported from apps/worker/src/jobs/events.kafka-consumer.ts (M8-003). The
+// delivery contract is ADR-004's and is unchanged line for line: per-key
+// serial groups, the ascending contiguous-prefix offset walk, at-least-once,
+// bounded in-consumer retry, and a dead-letter topic on exhaustion.
+//
+// The kafkajs client, the topic/consumer-group names, the DLQ producer and
+// the retry bounds are all INJECTED. They still live in @openpanel/queue,
+// which imports @openpanel/core for its logger, so core cannot import it back
+// without a package cycle (see ingest.service.ts's header). Injection is also
+// what keeps those names byte-identical: this file never spells one.
+
+import type {
+ Consumer,
+ EachBatchPayload,
+ IHeaders,
+ KafkaMessage,
+} from 'kafkajs';
+import type { IncomingEventPayload } from './incoming-event';
+
+export interface KafkaConsumerHandle {
+ stop: () => Promise<void>;
+}
+
+// Heartbeat every N messages within a per-key group. The default kafkajs
+// sessionTimeout is 30s — calling heartbeat every 16 messages keeps us
+// comfortably under that even for slow handlers.
+const HEARTBEAT_EVERY = 16;
+
+const RETRY_BACKOFF_FACTOR = 2;
+
+/** Why a message was dead-lettered. A label value — keep the set small. */
+export type DeadLetterReason = 'parse_error' | 'handler_error';
+
+/**
+ * What the consumer hands to the dead-letter producer. Structurally identical
+ * to @openpanel/queue's `DeadLetterMessage`: the value stays the producer's
+ * original bytes, so a DLQ message can be replayed onto the events topic
+ * unchanged.
+ */
+export interface DeadLetterMessage {
+ key: Buffer | null;
+ value: Buffer | null;
+ headers?: IHeaders;
+ topic: string;
+ partition: number;
+ offset: string;
+ reason: string;
+ error: string;
+}
+
+export interface ConsumerLogger {
+ info: (obj: object, msg?: string) => void;
+ warn: (obj: object, msg?: string) => void;
+ error: (obj: object, msg?: string) => void;
+}
+
+export interface ConsumerMetrics {
+ reprocessed: (partition: string) => void;
+ handlerFailed: (partition: string) => void;
+ deadLettered: (partition: string, reason: DeadLetterReason) => void;
+ deadLetterFailed: (partition: string) => void;
+}
+
+export interface EventsBatchHandlerDeps {
+ handleEvent: (
+ payload: IncomingEventPayload,
+ meta: { partition: number; offset: string }
+ ) => Promise<unknown>;
+ sendToDeadLetter: (message: DeadLetterMessage) => Promise<void>;
+ logger: ConsumerLogger;
+ metrics: ConsumerMetrics;
+ onActivity: () => void;
+ topic: string;
+ maxAttempts: number;
+ initialRetryMs: number;
+ maxRetryMs: number;
+ /** Injected by the tests; production uses the timer below. */
+ sleep?: (ms: number) => Promise<void>;
+}
+
+const defaultSleep = (ms: number): Promise<void> =>
+ new Promise((resolve) => setTimeout(resolve, ms));
+
+const errorMessage = (err: unknown): string =>
+ err instanceof Error ? err.message : String(err);
+
+export interface EventsBatchHandler {
+ eachBatch: (payload: EachBatchPayload) => Promise<void>;
+ /** Called on GROUP_JOIN: a new assignment invalidates the watermarks. */
+ resetWatermarks: () => void;
+}
+
+/**
+ * The `eachBatch` handler, with its dependencies injected so the offset walk,
+ * the per-key serial groups and the failure paths are testable without a
+ * broker.
+ *
+ * Delivery contract (ADR-004): at-least-once. An offset is resolved only once
+ * its message has been handled, dead-lettered, or deliberately skipped —
+ * never merely because it failed.
+ */
+export function createEventsBatchHandler(
+ deps: EventsBatchHandlerDeps
+): EventsBatchHandler {
+ const sleep = deps.sleep ?? defaultSleep;
+
+ // Highest Kafka offset we have *resolved* (committed) per `topic-partition`,
+ // tracked across batches. Used purely for the reprocess detector below: if we
+ // ever see an offset at or below this watermark again, the message is being
+ // redelivered (at-least-once duplicate) outside of a rebalance — which is the
+ // signature of an offset-handling bug. Cleared on GROUP_JOIN so legitimate
+ // post-rebalance redelivery from the last committed offset doesn't trip it.
+ const resolvedHWM = new Map<string, number>();
+
+ const deadLetter = async (
+ message: KafkaMessage,
+ partition: number,
+ reason: DeadLetterReason,
+ err: unknown
+ ): Promise<boolean> => {
+ const label = String(partition);
+ try {
+ await deps.sendToDeadLetter({
+ key: message.key,
+ value: message.value,
+ headers: message.headers,
+ topic: deps.topic,
+ partition,
+ offset: message.offset,
+ reason,
+ error: errorMessage(err),
+ });
+ } catch (dlqErr) {
+ // The event is not lost: leaving the offset unresolved stops the
+ // contiguous-prefix walk here, so the broker redelivers this message.
+ deps.metrics.deadLetterFailed(label);
+ deps.logger.error(
+ { err: dlqErr, partition, offset: message.offset, reason },
+ 'kafka dead-letter produce failed — offset left unresolved for redelivery'
+ );
+ return false;
+ }
+ deps.metrics.deadLettered(label, reason);
+ deps.logger.error(
+ { err, partition, offset: message.offset, reason },
+ 'kafka message dead-lettered'
+ );
+ return true;
+ };
+
+ const runHandlerWithRetry = async (
+ payload: IncomingEventPayload,
+ partition: number,
+ offset: string
+ ): Promise<{ ok: true } | { ok: false; err: unknown }> => {
+ const label = String(partition);
+ let delayMs = deps.initialRetryMs;
+ let lastErr: unknown;
+
+ for (let attempt = 1; attempt <= deps.maxAttempts; attempt++) {
+ try {
+ await deps.handleEvent(payload, { partition, offset });
+ return { ok: true };
+ } catch (err) {
+ lastErr = err;
+ deps.metrics.handlerFailed(label);
+ deps.logger.warn(
+ {
+ err,
+ partition,
+ offset,
+ projectId: payload.projectId,
+ attempt,
+ maxAttempts: deps.maxAttempts,
+ },
+ 'kafka incomingEvent handler failed'
+ );
+ if (attempt < deps.maxAttempts) {
+ await sleep(delayMs);
+ delayMs = Math.min(delayMs * RETRY_BACKOFF_FACTOR, deps.maxRetryMs);
+ }
+ }
+ }
+
+ return { ok: false, err: lastErr };
+ };
+
+ /** @returns true when the offset may be resolved. */
+ const processMessage = async (
+ message: KafkaMessage,
+ partition: number
+ ): Promise<boolean> => {
+ let payload: IncomingEventPayload;
+ try {
+ if (!message.value) {
+ throw new Error('kafka message has no value');
+ }
+ payload = JSON.parse(message.value.toString()) as IncomingEventPayload;
+ } catch (err) {
+ deps.logger.error(
+ { err, partition, offset: message.offset },
+ 'kafka message parse failed'
+ );
+ // Deterministic: retrying identical bytes cannot succeed.
+ return deadLetter(message, partition, 'parse_error', err);
+ }
+
+ const result = await runHandlerWithRetry(
+ payload,
+ partition,
+ message.offset
+ );
+ if (result.ok) {
+ return true;
+ }
+ return deadLetter(message, partition, 'handler_error', result.err);
+ };
+
+ const eachBatch = async ({
+ batch,
+ resolveOffset,
+ heartbeat,
+ isRunning,
+ isStale,
+ }: EachBatchPayload): Promise<void> => {
+ if (batch.messages.length === 0) {
+ return;
+ }
+
+ const pk = `${batch.topic}-${batch.partition}`;
+ // Watermark from *previous* batches. Anything at or below this that we see
+ // now is a redelivery. Captured before this batch so intra-batch
+ // out-of-order processing (normal, see below) is never counted.
+ const priorHWM = resolvedHWM.get(pk) ?? -1;
+
+ // Group by partition key (= deviceId or `${projectId}:${profileId}`).
+ // Same-key messages stay serial so sessionBuffer/session-end-job state
+ // can't race; different keys run in parallel via Promise.all.
+ // Keyless messages get their own singleton group.
+ const groups = new Map<string, KafkaMessage[]>();
+ for (const m of batch.messages) {
+ const key = m.key ? m.key.toString() : `__no_key__:${m.offset}`;
+ const arr = groups.get(key);
+ if (arr) {
+ arr.push(m);
+ } else {
+ groups.set(key, [m]);
+ }
+ }
+
+ // Offsets that finished processing this batch. We resolve them AFTER all
+ // groups complete, in strict ascending order (see the loop below).
+ //
+ // Why: KafkaJS `resolveOffset` is last-write-wins and the next fetch
+ // starts from the last resolved offset (offsetManager.nextOffset). If we
+ // resolved inside the concurrent per-key loop, a lower offset resolving
+ // after a higher one would move the fetch position BACKWARDS and
+ // re-deliver everything in between — the root cause of the duplicate
+ // events. Resolving the contiguous ascending prefix at the end avoids
+ // both duplicates (never regress) and loss (stop at the first gap).
+ const processed = new Set<string>();
+ let processedCount = 0;
+
+ await Promise.all(
+ [...groups.values()].map(async (msgs) => {
+ for (const m of msgs) {
+ if (!isRunning() || isStale()) {
+ return;
+ }
+
+ // Reprocess detector: only fires for offsets already resolved in a
+ // PRIOR batch (redelivery). Intra-batch out-of-order processing
+ // across key-groups is expected and is not flagged.
+ if (Number(m.offset) <= priorHWM) {
+ deps.metrics.reprocessed(String(batch.partition));
+ deps.logger.warn(
+ {
+ partition: batch.partition,
+ offset: m.offset,
+ resolvedHighWaterMark: priorHWM,
+ },
+ 'kafka offset REPROCESSED — at-least-once duplicate (outside rebalance)'
+ );
+ }
+
+ const resolvable = await processMessage(m, batch.partition);
+ if (!resolvable) {
+ // Stop this key-group: continuing past an unresolvable offset only
+ // adds duplicates, since the whole tail is redelivered anyway.
+ return;
+ }
+
+ processed.add(m.offset);
+ processedCount += 1;
+ if (processedCount % HEARTBEAT_EVERY === 0) {
+ await heartbeat();
+ }
+ }
+ })
+ );
+
+ // Resolve in strict ascending offset order, stopping at the first offset
+ // that did not finish (e.g. an isStale/isRunning early-return mid-batch).
+ // batch.messages is already ordered by offset.
+ let newHWM = priorHWM;
+ for (const m of batch.messages) {
+ if (!processed.has(m.offset)) {
+ break;
+ }
+ resolveOffset(m.offset);
+ newHWM = Math.max(newHWM, Number(m.offset));
+ }
+ resolvedHWM.set(pk, newHWM);
+
+ await heartbeat();
+ deps.onActivity();
+ };
+
+ return { eachBatch, resetWatermarks: () => resolvedHWM.clear() };
+}
+
+/** The broker-facing half, supplied by whoever owns the kafkajs client. */
+export interface EventsConsumerDeps {
+ /** Already configured with the group id, timeouts and fetch bounds. */
+ createConsumer: () => Consumer;
+ /** Lifecycle / rebalance visibility. */
+ logger: ConsumerLogger;
+ /** The "consumer running" line, on the kafka-scoped logger. */
+ kafkaLogger: ConsumerLogger;
+ topic: string;
+ partitionsConsumedConcurrently: number;
+ batch: Omit<EventsBatchHandlerDeps, 'topic'>;
+}
+
+export async function startKafkaEventsConsumer(
+ deps: EventsConsumerDeps
+): Promise<KafkaConsumerHandle> {
+ const { logger } = deps;
+ const consumer = deps.createConsumer();
+ const handler = createEventsBatchHandler({
+ ...deps.batch,
+ topic: deps.topic,
+ });
+ await consumer.connect();
+ await consumer.subscribe({
+ topic: deps.topic,
+ fromBeginning: false,
+ });
+
+ consumer.on(consumer.events.HEARTBEAT, deps.batch.onActivity);
+
+ // ---- Lifecycle / rebalance ("re-election") visibility ----
+ // We had no logging for partition reassignment before; without it a rebalance
+ // storm (a common source of at-least-once duplicates) is invisible.
+ consumer.on(consumer.events.GROUP_JOIN, ({ payload }) => {
+ // A new assignment means partitions may have moved between members; reset
+ // the reprocess watermarks so legitimate resume-from-committed-offset after
+ // a rebalance is not flagged as a duplicate.
+ handler.resetWatermarks();
+ logger.info(
+ {
+ memberId: payload.memberId,
+ groupId: payload.groupId,
+ isLeader: payload.isLeader,
+ memberAssignment: payload.memberAssignment,
+ duration: payload.duration,
+ },
+ 'kafka consumer joined group (rebalance complete)'
+ );
+ });
+ consumer.on(consumer.events.REBALANCING, ({ payload }) => {
+ logger.warn(
+ { memberId: payload.memberId, groupId: payload.groupId },
+ 'kafka consumer rebalancing'
+ );
+ });
+ consumer.on(consumer.events.CRASH, ({ payload }) => {
+ logger.error(
+ {
+ err: payload.error,
+ groupId: payload.groupId,
+ restart: payload.restart,
+ },
+ 'kafka consumer crashed'
+ );
+ });
+ consumer.on(consumer.events.DISCONNECT, () => {
+ logger.warn({}, 'kafka consumer disconnected');
+ });
+ consumer.on(consumer.events.REQUEST_TIMEOUT, ({ payload }) => {
+ logger.warn(
+ { broker: payload.broker, clientId: payload.clientId },
+ 'kafka consumer request timeout'
+ );
+ });
+
+ await consumer.run({
+ partitionsConsumedConcurrently: deps.partitionsConsumedConcurrently,
+ eachBatchAutoResolve: false,
+ eachBatch: handler.eachBatch,
+ });
+
+ deps.kafkaLogger.info(
+ {
+ topic: deps.topic,
+ partitionsConsumedConcurrently: deps.partitionsConsumedConcurrently,
+ },
+ 'kafka events consumer running'
+ );
+
+ return {
+ stop: async () => {
+ await consumer.disconnect();
+ },
+ };
+}
diff --git a/packages/core/src/modules/ingest/src/incoming-event-handler.test.ts b/packages/core/src/modules/ingest/src/incoming-event-handler.test.ts
new file mode 100644
index 000000000..3a99cf1c9
--- /dev/null
+++ b/packages/core/src/modules/ingest/src/incoming-event-handler.test.ts
@@ -0,0 +1,456 @@
+// Ported from apps/worker/src/jobs/events.incoming-events.test.ts (M8-003)
+// with every assertion unchanged. What changed is the seam: V1 reached the
+// session buffer, `createEvent` and the sessions queue through
+// `vi.mock('@openpanel/db')` / `vi.mock('@openpanel/queue')`; here they are
+// recording doubles handed in as `IncomingEventDeps`, so the assertions are
+// on calls the code under test actually made and no `mock.module` is needed.
+//
+// The session_end enqueue is asserted through the real `sessionEndJobPayload`
+// / `sessionEndEnqueueOptions` — the same functions V1's producer
+// (`enqueueSessionEndV2`) calls — so the jobId and the `{payload, snapshot}`
+// mapping are still pinned here, not re-implemented by the test.
+
+import { beforeEach, describe, expect, mock, test } from 'bun:test';
+import { formatClickhouseDate } from '../../event/src/dates';
+import type { IClickhouseSession } from '../../session/session.service';
+import {
+ sessionEndEnqueueOptions,
+ sessionEndJobPayload,
+} from '../../session/src/session-end';
+import type { IncomingEventPayload } from './incoming-event';
+import {
+ type IncomingEventDeps,
+ incomingEvent,
+} from './incoming-event-handler';
+
+const projectId = 'test-project';
+const deviceId = 'device-123';
+const newSessionId = 'a1b2c3d4-e5f6-4789-a012-345678901234';
+// What the producer (/track) would have minted for one request.
+const producedEventId = '11111111-2222-4333-8444-555555555555';
+const geo = {
+ country: 'US',
+ city: 'New York',
+ region: 'NY',
+ longitude: 0,
+ latitude: 0,
+};
+
+const uaInfo: IncomingEventPayload['uaInfo'] = {
+ isServer: false,
+ device: 'desktop',
+ os: 'Windows',
+ osVersion: '10',
+ browser: 'Chrome',
+ browserVersion: '91.0.4472.124',
+ brand: '',
+ model: '',
+};
+
+const uaInfoServer: IncomingEventPayload['uaInfo'] = {
+ isServer: true,
+ device: 'server',
+ os: '',
+ osVersion: '',
+ browser: '',
+ browserVersion: '',
+ brand: '',
+ model: '',
+};
+
+function makeSession(
+ overrides: Partial<IClickhouseSession> = {}
+): IClickhouseSession {
+ const now = new Date();
+ return {
+ id: 'session-existing',
+ project_id: projectId,
+ device_id: deviceId,
+ profile_id: '',
+ event_count: 1,
+ screen_view_count: 0,
+ entry_path: '/',
+ entry_origin: 'https://example.com',
+ exit_path: '/',
+ exit_origin: 'https://example.com',
+ created_at: formatClickhouseDate(now),
+ ended_at: formatClickhouseDate(now),
+ os: 'Windows',
+ os_version: '10',
+ browser: 'Chrome',
+ browser_version: '91.0.4472.124',
+ device: 'desktop',
+ brand: '',
+ model: '',
+ country: 'US',
+ region: 'NY',
+ city: 'New York',
+ longitude: 0,
+ latitude: 0,
+ duration: 0,
+ referrer: '',
+ referrer_name: '',
+ referrer_type: '',
+ is_bounce: true,
+ utm_term: '',
+ utm_source: '',
+ utm_campaign: '',
+ utm_content: '',
+ utm_medium: '',
+ revenue: 0,
+ sign: 1,
+ version: 1,
+ groups: [],
+ ...overrides,
+ } satisfies IClickhouseSession;
+}
+
+function buildJobData(
+ overrides: Partial<IncomingEventPayload> = {}
+): IncomingEventPayload {
+ return {
+ geo,
+ event: {
+ name: 'test_event',
+ timestamp: new Date().toISOString(),
+ isTimestampFromThePast: false,
+ properties: { __path: 'https://example.com/test' },
+ },
+ uaInfo,
+ headers: {
+ 'request-id': '123',
+ 'user-agent': 'Mozilla/5.0',
+ 'openpanel-sdk-name': 'web',
+ 'openpanel-sdk-version': '1.0.0',
+ },
+ projectId,
+ deviceId,
+ sessionId: newSessionId,
+ ...overrides,
+ };
+}
+
+const noopLogger = (() => {
+ const noop = () => undefined;
+ const logger = {
+ fatal: noop,
+ error: noop,
+ warn: noop,
+ info: noop,
+ debug: noop,
+ trace: noop,
+ child: () => logger,
+ };
+ return logger;
+})();
+
+function makeDeps() {
+ const createEvent = mock<IncomingEventDeps['createEvent']>(
+ async (event) => event
+ );
+ const getExistingSession = mock<
+ IncomingEventDeps['sessions']['getExistingSession']
+ >(async () => null);
+ const ingest = mock<IncomingEventDeps['sessions']['ingest']>(
+ async () => null
+ );
+ const enqueueSessionEnd = mock<IncomingEventDeps['enqueueSessionEnd']>(
+ async () => undefined
+ );
+ const deps: IncomingEventDeps = {
+ logger: noopLogger,
+ sessions: { getExistingSession, ingest },
+ createEvent,
+ checkNotificationRulesForEvent: mock(async () => true),
+ projects: {
+ getCached: mock(async () => ({ firstEventAt: new Date(), filters: [] })),
+ markFirstEvent: mock(async () => undefined),
+ },
+ enqueueSessionEnd,
+ metrics: {
+ sessionStarted: mock(() => undefined),
+ sessionEndEnqueued: mock(() => undefined),
+ },
+ };
+ return { deps, createEvent, getExistingSession, ingest, enqueueSessionEnd };
+}
+
+/** Every `createEvent` call's first argument, in order. */
+const createdEvents = (createEvent: {
+ mock: { calls: unknown[][] };
+}): Record<string, unknown>[] =>
+ createEvent.mock.calls.map(([arg]) => arg as Record<string, unknown>);
+
+let ctx: ReturnType<typeof makeDeps>;
+
+beforeEach(() => {
+ ctx = makeDeps();
+});
+
+describe('incomingEvent', () => {
+ test('emits session_start when ingest returns kind="new"', async () => {
+ const { deps, createEvent, ingest, enqueueSessionEnd } = ctx;
+ ingest.mockResolvedValueOnce({
+ kind: 'new',
+ current: makeSession({ id: newSessionId }),
+ });
+
+ await incomingEvent(buildJobData(), deps);
+
+ expect(enqueueSessionEnd).not.toHaveBeenCalled();
+ const calls = createdEvents(createEvent);
+ expect(calls).toHaveLength(2);
+ expect(calls[0]!.name).toBe('session_start');
+ expect(calls[1]!.name).toBe('test_event');
+ });
+
+ test('skips session_start when ingest returns kind="extend"', async () => {
+ const { deps, createEvent, ingest, enqueueSessionEnd } = ctx;
+ ingest.mockResolvedValueOnce({ kind: 'extend', current: makeSession() });
+
+ await incomingEvent(buildJobData(), deps);
+
+ expect(enqueueSessionEnd).not.toHaveBeenCalled();
+ const sessionStartCalls = createdEvents(createEvent).filter(
+ (arg) => arg.name === 'session_start'
+ );
+ expect(sessionStartCalls).toHaveLength(0);
+ });
+
+ test('closes old session and emits session_start when ingest returns kind="boundary"', async () => {
+ const { deps, createEvent, ingest, enqueueSessionEnd } = ctx;
+ const closed = makeSession({ id: 'old-session-id' });
+ const current = makeSession({ id: newSessionId });
+ ingest.mockResolvedValueOnce({ kind: 'boundary', current, closed });
+
+ await incomingEvent(buildJobData(), deps);
+
+ expect(enqueueSessionEnd).toHaveBeenCalledTimes(1);
+ const input = enqueueSessionEnd.mock.calls[0]![0];
+ // The wire payload V1's producer builds from this input, through the same
+ // core functions it calls.
+ const { event, snapshot } = sessionEndJobPayload(input);
+ expect(event.sessionId).toBe('old-session-id');
+ expect(snapshot.id).toBe('old-session-id');
+ expect(sessionEndEnqueueOptions(input.closedSession.id).jobId).toBe(
+ 'sessionEnd:v2:old-session-id'
+ );
+
+ const calls = createdEvents(createEvent);
+ expect(calls.filter((a) => a.name === 'session_start')).toHaveLength(1);
+ expect(calls.filter((a) => a.name === 'test_event')).toHaveLength(1);
+ });
+
+ test('inherits referrer from current session on the actual event', async () => {
+ const { deps, createEvent, ingest } = ctx;
+ ingest.mockResolvedValueOnce({
+ kind: 'extend',
+ current: makeSession({
+ referrer: 'https://google.com',
+ referrer_name: 'Google',
+ referrer_type: 'search',
+ }),
+ });
+
+ await incomingEvent(buildJobData(), deps);
+
+ const testEventCall = createdEvents(createEvent).find(
+ (a) => a.name === 'test_event'
+ );
+ expect(testEventCall!.referrer).toBe('https://google.com');
+ expect(testEventCall!.referrerName).toBe('Google');
+ expect(testEventCall!.referrerType).toBe('search');
+ });
+
+ test('handles server events with existing profile session', async () => {
+ const { deps, createEvent, ingest, getExistingSession, enqueueSessionEnd } =
+ ctx;
+ const timestamp = new Date();
+ const jobData = buildJobData({
+ event: {
+ name: 'server_event',
+ timestamp: timestamp.toISOString(),
+ properties: { custom_property: 'test_value' },
+ profileId: 'profile-123',
+ isTimestampFromThePast: false,
+ },
+ uaInfo: uaInfoServer,
+ deviceId: '',
+ sessionId: '',
+ headers: {
+ 'user-agent': 'OpenPanel Server/1.0',
+ 'openpanel-sdk-name': 'server',
+ 'openpanel-sdk-version': '1.0.0',
+ 'request-id': '123',
+ },
+ });
+
+ getExistingSession.mockResolvedValueOnce(
+ makeSession({
+ id: 'last-session-456',
+ device_id: 'last-device-123',
+ profile_id: 'profile-123',
+ os: 'iOS',
+ os_version: '15.0',
+ browser: 'Safari',
+ browser_version: '15.0',
+ device: 'mobile',
+ brand: 'Apple',
+ model: 'iPhone',
+ country: 'CA',
+ region: 'ON',
+ city: 'Toronto',
+ entry_path: '/last-path',
+ entry_origin: 'https://example.com',
+ exit_path: '/last-path',
+ exit_origin: 'https://example.com',
+ referrer: 'https://google.com',
+ referrer_name: 'Google',
+ referrer_type: 'search',
+ is_bounce: false,
+ event_count: 0,
+ screen_view_count: 0,
+ })
+ );
+
+ await incomingEvent(jobData, deps);
+
+ expect(ingest).not.toHaveBeenCalled();
+ expect(enqueueSessionEnd).not.toHaveBeenCalled();
+ expect(createdEvents(createEvent)[0]!).toMatchObject({
+ name: 'server_event',
+ deviceId: 'last-device-123',
+ sessionId: 'last-session-456',
+ profileId: 'profile-123',
+ city: 'Toronto',
+ country: 'CA',
+ referrer: 'https://google.com',
+ });
+ });
+
+ test('handles server events without any active session', async () => {
+ const { deps, createEvent, ingest, getExistingSession, enqueueSessionEnd } =
+ ctx;
+ getExistingSession.mockResolvedValueOnce(null);
+
+ await incomingEvent(
+ buildJobData({
+ event: {
+ name: 'server_event',
+ timestamp: new Date().toISOString(),
+ properties: { custom_property: 'test_value' },
+ profileId: 'profile-123',
+ isTimestampFromThePast: false,
+ },
+ uaInfo: uaInfoServer,
+ deviceId: '',
+ sessionId: '',
+ headers: {
+ 'user-agent': 'OpenPanel Server/1.0',
+ 'openpanel-sdk-name': 'server',
+ 'openpanel-sdk-version': '1.0.0',
+ 'request-id': '123',
+ },
+ }),
+ deps
+ );
+
+ expect(ingest).not.toHaveBeenCalled();
+ expect(enqueueSessionEnd).not.toHaveBeenCalled();
+ expect(createdEvents(createEvent)[0]!).toMatchObject({
+ name: 'server_event',
+ deviceId: '',
+ sessionId: '',
+ profileId: 'profile-123',
+ });
+ });
+
+ test('re-creates the same row id when Kafka redelivers the message', async () => {
+ const { deps, createEvent, ingest } = ctx;
+ const session = makeSession({ id: newSessionId });
+ ingest
+ .mockResolvedValueOnce({ kind: 'new', current: session })
+ .mockResolvedValueOnce({ kind: 'extend', current: session });
+
+ // Same message, delivered twice: identical payload, identical offset.
+ const jobData = buildJobData({ id: producedEventId });
+ const delivery = { partition: 3, offset: '42' };
+ await incomingEvent(jobData, deps, delivery);
+ await incomingEvent(jobData, deps, delivery);
+
+ const eventCalls = createdEvents(createEvent).filter(
+ (a) => a.name === 'test_event'
+ );
+ expect(eventCalls).toHaveLength(2);
+ expect(eventCalls[0]!.id).toBe(producedEventId);
+ expect(eventCalls[1]!.id).toBe(producedEventId);
+
+ // The derived session_start row must NOT reuse the event's id.
+ const sessionStartCalls = createdEvents(createEvent).filter(
+ (a) => a.name === 'session_start'
+ );
+ expect(sessionStartCalls).toHaveLength(1);
+ expect(sessionStartCalls[0]!.id).toBeUndefined();
+ });
+
+ test('carries the producer-minted id on server-side events', async () => {
+ const { deps, createEvent, getExistingSession } = ctx;
+ getExistingSession.mockResolvedValueOnce(null);
+
+ await incomingEvent(
+ buildJobData({
+ id: producedEventId,
+ event: {
+ name: 'server_event',
+ timestamp: new Date().toISOString(),
+ isTimestampFromThePast: false,
+ profileId: 'profile-123',
+ },
+ uaInfo: uaInfoServer,
+ deviceId: '',
+ sessionId: '',
+ }),
+ deps
+ );
+
+ expect(createdEvents(createEvent)[0]!.id).toBe(producedEventId);
+ });
+
+ test('leaves the id undefined for a payload produced without one', async () => {
+ const { deps, createEvent, ingest } = ctx;
+ ingest.mockResolvedValueOnce({ kind: 'extend', current: makeSession() });
+
+ await incomingEvent(buildJobData(), deps);
+
+ expect(createdEvents(createEvent)[0]!.id).toBeUndefined();
+ });
+
+ test('emits session_start only once across 3 rapid events (new → extend → extend)', async () => {
+ const { deps, createEvent, ingest, enqueueSessionEnd } = ctx;
+ const session = makeSession({ id: newSessionId });
+ ingest
+ .mockResolvedValueOnce({ kind: 'new', current: session })
+ .mockResolvedValueOnce({ kind: 'extend', current: session })
+ .mockResolvedValueOnce({ kind: 'extend', current: session });
+
+ for (const name of ['e1', 'e2', 'e3']) {
+ await incomingEvent(
+ buildJobData({
+ event: {
+ name,
+ timestamp: new Date().toISOString(),
+ isTimestampFromThePast: false,
+ properties: { __path: 'https://example.com/test' },
+ },
+ }),
+ deps
+ );
+ }
+
+ const sessionStartCalls = createdEvents(createEvent).filter(
+ (a) => a.name === 'session_start'
+ );
+ expect(sessionStartCalls).toHaveLength(1);
+ expect(enqueueSessionEnd).not.toHaveBeenCalled();
+ });
+});
diff --git a/packages/core/src/modules/ingest/src/incoming-event-handler.ts b/packages/core/src/modules/ingest/src/incoming-event-handler.ts
new file mode 100644
index 000000000..d841c0992
--- /dev/null
+++ b/packages/core/src/modules/ingest/src/incoming-event-handler.ts
@@ -0,0 +1,427 @@
+// Ported from apps/worker/src/jobs/events.incoming-event.ts (M8-003): the
+// Kafka consumer's per-message handler — the step that turns one topic
+// payload into session lifecycle decisions and ClickHouse rows. V1's job file
+// stays (DELEGATE PATTERN) and calls this until apps/worker dies (P9).
+//
+// Everything that touches Redis, Postgres, ClickHouse or a queue is INJECTED
+// (`IncomingEventDeps`), the way session-end.ts does it: the tests then drive
+// the real code with recording doubles instead of `mock.module`, and V1 keeps
+// handing in its own buffers, counters and `sessionsQueue` producer so its
+// behaviour and its `/metrics` body are unchanged.
+//
+// createEvent is a static import for the reason session.service.ts's header
+// gives: a dynamic edge here closes a dynamic-import cycle that panics
+// rolldown when apps/worker bundles the workspace.
+
+import { getTime, isSameDomain, parsePath } from '@openpanel/common';
+import type { IProjectFilters } from '@openpanel/validation';
+import { anyPass, isEmpty, isNil, mergeDeepRight, omit, reject } from 'ramda';
+import type { SessionIngestResult } from '../../../buffers/session-buffer';
+import type { Logger } from '../../../logger';
+import {
+ getReferrerWithQuery,
+ parseReferrer,
+} from '../../../shared/parse-referrer';
+import {
+ createEvent,
+ type IServiceCreateEventPayload,
+ type IServiceCreateEventPayloadWithId,
+ type IServiceEvent,
+} from '../../event/event.service';
+import { matchEvent } from '../../notification/notification.service';
+// Static, not lazy: a dynamic edge into project.service becomes its own
+// rolldown chunk, which stops `hashPassword` (and with it the @node-rs/argon2
+// native addon, external in apps/worker's bundle) from being shaken out of
+// the worker build.
+import { getProjectByIdCached } from '../../project/project.service';
+import type { IClickhouseSession } from '../../session/session.service';
+import { sessionEndsEnqueued } from '../../session/src/session.metrics';
+import type { EnqueueSessionEndInput } from '../../session/src/session-end';
+import type { IncomingEventPayload } from './incoming-event';
+import { sessionsStarted } from './ingest.metrics';
+
+const GLOBAL_PROPERTIES = ['__path', '__referrer', '__timestamp', '__revenue'];
+
+/** How far before the event `session_start` is stamped, so it sorts first. */
+const SESSION_START_OFFSET_MS = 100;
+
+// Strip empty/nullish from B, then deep-merge over A.
+const merge = <A, B>(a: Partial<A>, b: Partial<B>): A & B =>
+ mergeDeepRight(a, reject(anyPass([isEmpty, isNil]))(b)) as A & B;
+
+/** Kafka delivery coordinates, when the event came through the consumer. */
+export interface IncomingEventDelivery {
+ partition: number;
+ offset: string;
+}
+
+/** The project fields the filter and first-event checks read. */
+interface IncomingEventProject {
+ firstEventAt?: Date | null;
+ filters?: IProjectFilters[] | null;
+}
+
+/** The session-buffer subset this handler drives. */
+export interface IncomingEventSessions {
+ getExistingSession(options: {
+ projectId: string;
+ profileId: string;
+ }): Promise<IClickhouseSession | null>;
+ ingest(
+ payload: IServiceCreateEventPayload
+ ): Promise<SessionIngestResult | null>;
+}
+
+/**
+ * Injected so V1's worker keeps incrementing its own registry's counters
+ * while core's registry owns the V2 ones (ingest.metrics.ts).
+ */
+export interface IncomingEventMetrics {
+ sessionStarted(kind: 'new' | 'boundary'): void;
+ sessionEndEnqueued(source: 'boundary'): void;
+}
+
+export interface IncomingEventDeps {
+ /** Base logger; the handler child()s it per message. */
+ logger: Logger;
+ sessions: IncomingEventSessions;
+ createEvent(
+ payload: IServiceCreateEventPayloadWithId
+ ): Promise<{ document: unknown } | unknown>;
+ checkNotificationRulesForEvent(
+ payload: IServiceCreateEventPayload
+ ): Promise<unknown>;
+ projects: {
+ getCached(projectId: string): Promise<IncomingEventProject | null>;
+ /** Conditional update + cache clear; idempotent across workers. */
+ markFirstEvent(projectId: string): Promise<unknown>;
+ };
+ enqueueSessionEnd(input: EnqueueSessionEndInput): Promise<unknown>;
+ metrics: IncomingEventMetrics;
+}
+
+/**
+ * Binds `IncomingEventDeps` to the real implementations. db/ch access stays
+ * lazy — see event.service.ts's header.
+ */
+export async function loadIncomingEventDeps(
+ logger: Logger,
+ enqueueSessionEnd: (input: EnqueueSessionEndInput) => Promise<unknown>,
+ overrides: Partial<IncomingEventDeps> = {}
+): Promise<IncomingEventDeps> {
+ const [{ sessionBuffer }, { db }, notifications] = await Promise.all([
+ import('@openpanel/db/src/buffers'),
+ import('@openpanel/db/src/prisma-client'),
+ import('@openpanel/db/src/services/notification.service'),
+ ]);
+ return {
+ logger,
+ sessions: sessionBuffer,
+ createEvent,
+ checkNotificationRulesForEvent:
+ notifications.checkNotificationRulesForEvent,
+ projects: {
+ getCached: getProjectByIdCached,
+ markFirstEvent: async (projectId) => {
+ await db.project.updateMany({
+ where: { id: projectId, firstEventAt: null },
+ data: { firstEventAt: new Date() },
+ });
+ await getProjectByIdCached.clear(projectId);
+ },
+ },
+ enqueueSessionEnd,
+ metrics: {
+ sessionStarted: (kind) => sessionsStarted.inc({ kind }),
+ sessionEndEnqueued: (source) => sessionEndsEnqueued.inc({ source }),
+ },
+ ...overrides,
+ };
+}
+
+async function isEventExcludedByProjectFilter(
+ payload: IServiceCreateEventPayload,
+ projectId: string,
+ deps: IncomingEventDeps
+): Promise<boolean> {
+ const project = await deps.projects.getCached(projectId);
+ const eventExcludeFilters = (project?.filters ?? []).filter(
+ (f) => f.type === 'event'
+ );
+ if (eventExcludeFilters.length === 0) {
+ return false;
+ }
+ return eventExcludeFilters.some((filter) => matchEvent(payload, filter));
+}
+
+/**
+ * Records the project's first-ever event timestamp exactly once. The cached
+ * project read makes this a no-op on every event after the first; the
+ * conditional update keeps concurrent workers idempotent.
+ */
+async function markFirstEvent(
+ projectId: string,
+ logger: Logger,
+ deps: IncomingEventDeps
+) {
+ const project = await deps.projects.getCached(projectId);
+ if (!project || project.firstEventAt) {
+ return;
+ }
+ await deps.projects.markFirstEvent(projectId);
+ logger.info({ projectId }, 'Project received its first event');
+}
+
+async function createEventAndNotify(
+ payload: IServiceCreateEventPayloadWithId,
+ logger: Logger,
+ projectId: string,
+ deps: IncomingEventDeps
+) {
+ const isExcluded = await isEventExcludedByProjectFilter(
+ payload,
+ projectId,
+ deps
+ );
+ if (isExcluded) {
+ logger.info(
+ { event: payload.name, projectId },
+ 'Event excluded by project filter'
+ );
+ return null;
+ }
+
+ logger.info({ event: payload }, 'Creating event');
+ const [event] = await Promise.all([
+ deps.createEvent(payload),
+ deps.checkNotificationRulesForEvent(payload).catch(() => null),
+ ]);
+ // Only after the event is accepted — recording the first event before a
+ // failed createEvent would leave the activation checklist claiming data
+ // arrived that was never persisted.
+ await markFirstEvent(projectId, logger, deps).catch(() => null);
+ return event;
+}
+
+const parseRevenue = (revenue: unknown): number | undefined => {
+ if (!revenue) {
+ return undefined;
+ }
+ if (typeof revenue === 'number') {
+ return revenue;
+ }
+ if (typeof revenue === 'string') {
+ const parsed = Number.parseFloat(revenue);
+ return Number.isNaN(parsed) ? undefined : parsed;
+ }
+ return undefined;
+};
+
+export async function incomingEvent(
+ jobPayload: IncomingEventPayload,
+ deps: IncomingEventDeps,
+ // Logged so a duplicate row in ClickHouse can be traced back to the exact
+ // partition/offset that produced it.
+ meta?: IncomingEventDelivery
+) {
+ const {
+ geo,
+ event: body,
+ headers,
+ projectId,
+ deviceId,
+ sessionId,
+ uaInfo,
+ // Producer-minted; a redelivered message re-creates the same row id.
+ // Absent on messages produced before the field existed — createEvent then
+ // falls back to a fresh uuid.
+ id: eventId,
+ } = jobPayload;
+ const properties: Record<string, unknown> = body.properties ?? {};
+ const reqId = headers['request-id'] ?? 'unknown';
+ const logger = deps.logger.child({
+ reqId,
+ ...(meta
+ ? { kafkaPartition: meta.partition, kafkaOffset: meta.offset }
+ : {}),
+ });
+ const getProperty = (name: string): string | undefined => {
+ // replace thing is just for older sdks when we didn't have `__`
+ // remove when kiddokitchen app (24.09.02) is not used anymore
+ return (
+ ((properties[name] || properties[name.replace('__', '')]) as
+ | string
+ | null
+ | undefined) ?? undefined
+ );
+ };
+
+ const profileId = body.profileId ? String(body.profileId) : '';
+ const createdAt = new Date(body.timestamp);
+ const isTimestampFromThePast = body.isTimestampFromThePast;
+ const url = getProperty('__path');
+ const { path, hash, query, origin } = parsePath(url);
+ const referrer = isSameDomain(getProperty('__referrer'), url)
+ ? null
+ : parseReferrer(getProperty('__referrer'));
+ const utmReferrer = getReferrerWithQuery(query);
+ // Typed as a plain record so `omit`'s key set is `string`, not the two
+ // literal keys the spread would otherwise infer.
+ const eventProperties: Record<string, unknown> = {
+ ...properties,
+ __hash: hash,
+ __query: query,
+ };
+ const sdkName = headers['openpanel-sdk-name'];
+ const sdkVersion = headers['openpanel-sdk-version'];
+
+ const baseEvent: IServiceCreateEventPayload = {
+ name: body.name,
+ profileId,
+ projectId,
+ deviceId,
+ sessionId,
+ properties: omit(GLOBAL_PROPERTIES, eventProperties),
+ groups: body.groups ?? [],
+ createdAt,
+ duration: 0,
+ sdkName,
+ sdkVersion,
+ city: geo.city,
+ country: geo.country,
+ region: geo.region,
+ longitude: geo.longitude,
+ latitude: geo.latitude,
+ path,
+ origin,
+ referrer: referrer?.url || '',
+ referrerName: utmReferrer?.name || referrer?.name || referrer?.url,
+ referrerType: utmReferrer?.type || referrer?.type || '',
+ os: uaInfo.os,
+ osVersion: uaInfo.osVersion,
+ browser: uaInfo.browser,
+ browserVersion: uaInfo.browserVersion,
+ device: uaInfo.device,
+ brand: uaInfo.brand,
+ model: uaInfo.model,
+ revenue:
+ body.name === 'revenue' && '__revenue' in properties
+ ? parseRevenue(properties.__revenue)
+ : undefined,
+ };
+
+ // Server-side and "timestamp from the past" events ride alongside an
+ // existing client session if there is one — they don't open / close
+ // sessions of their own.
+ if (uaInfo.isServer || isTimestampFromThePast) {
+ const session =
+ profileId && !isTimestampFromThePast
+ ? await deps.sessions.getExistingSession({ profileId, projectId })
+ : null;
+
+ const payload = {
+ ...baseEvent,
+ deviceId: session?.device_id ?? '',
+ sessionId: session?.id ?? '',
+ referrer: session?.referrer ?? undefined,
+ referrerName: session?.referrer_name ?? undefined,
+ referrerType: session?.referrer_type ?? undefined,
+ path: session?.exit_path ?? baseEvent.path,
+ origin: session?.exit_origin ?? baseEvent.origin,
+ os: session?.os ?? baseEvent.os,
+ osVersion: session?.os_version ?? baseEvent.osVersion,
+ browserVersion: session?.browser_version ?? baseEvent.browserVersion,
+ browser: session?.browser ?? baseEvent.browser,
+ device: session?.device ?? baseEvent.device,
+ brand: session?.brand ?? baseEvent.brand,
+ model: session?.model ?? baseEvent.model,
+ city: session?.city ?? baseEvent.city,
+ country: session?.country ?? baseEvent.country,
+ region: session?.region ?? baseEvent.region,
+ longitude: session?.longitude ?? baseEvent.longitude,
+ latitude: session?.latitude ?? baseEvent.latitude,
+ };
+
+ return createEventAndNotify(
+ { ...(payload as IServiceEvent), id: eventId },
+ logger,
+ projectId,
+ deps
+ );
+ }
+
+ if (await isEventExcludedByProjectFilter(baseEvent, projectId, deps)) {
+ logger.info(
+ { event: baseEvent.name, projectId },
+ 'Skipping session_start and event (excluded by project filter)'
+ );
+ return null;
+ }
+
+ // The single source of truth for session lifecycle. Reads the current
+ // session, decides extend/new/boundary, writes back. The returned
+ // `current` is the canonical session — use its referrer fields for
+ // inheritance, just like the previous behavior.
+ const session = await deps.sessions.ingest(baseEvent);
+
+ if (session?.kind === 'boundary') {
+ // Close the old session in a separate job (one Redis-buffered insert
+ // for the session_end event + notification rule check). Idempotent via
+ // BullMQ jobId dedup.
+ await deps
+ .enqueueSessionEnd({
+ payload: baseEvent,
+ closedSession: session.closed,
+ })
+ .then(() => deps.metrics.sessionEndEnqueued('boundary'))
+ .catch((error) => {
+ logger.error(
+ { err: error, deviceId, sessionId: session.closed.id },
+ 'Error enqueueing session_end on boundary'
+ );
+ });
+ }
+
+ if (session?.kind === 'new' || session?.kind === 'boundary') {
+ deps.metrics.sessionStarted(session.kind);
+ // No `id` here: session_start is a second, derived row and must not share
+ // the producer-minted id of the event that triggered it.
+ await createEventAndNotify(
+ {
+ ...baseEvent,
+ name: 'session_start',
+ createdAt: new Date(
+ getTime(baseEvent.createdAt) - SESSION_START_OFFSET_MS
+ ),
+ },
+ logger,
+ projectId,
+ deps
+ ).catch((error) => {
+ logger.error(
+ { err: error, event: baseEvent },
+ 'Error creating session start event'
+ );
+ throw error;
+ });
+ }
+
+ // Inherit referrer fields from the canonical session for the actual event.
+ // For 'extend' this preserves the original session's referrer across
+ // mid-session events; for 'new' / 'boundary' it's the event's own referrer
+ // (which `ingest` just stored on the fresh session).
+ const finalPayload: IServiceCreateEventPayload = session
+ ? merge(baseEvent, {
+ referrer: session.current.referrer,
+ referrerName: session.current.referrer_name,
+ referrerType: session.current.referrer_type,
+ } as Partial<IServiceCreateEventPayload>)
+ : baseEvent;
+
+ return createEventAndNotify(
+ { ...finalPayload, id: eventId },
+ logger,
+ projectId,
+ deps
+ );
+}
diff --git a/packages/core/src/modules/ingest/src/ingest.metrics.ts b/packages/core/src/modules/ingest/src/ingest.metrics.ts
new file mode 100644
index 000000000..2c4762133
--- /dev/null
+++ b/packages/core/src/modules/ingest/src/ingest.metrics.ts
@@ -0,0 +1,54 @@
+// The ingest-path collectors, moved from apps/worker/src/metrics.ts (M8-003)
+// onto core's one registry. Names, labels and help strings are V1's; the
+// worker-side copies die with apps/worker (P9). Until then V1's consumer and
+// incoming-event delegate inject their own counters (both take `metrics` as a
+// dependency), so V1's `/metrics` body is unchanged and these are the V2
+// bindings.
+
+import client from 'prom-client';
+import { registry } from '../../../metrics';
+
+// Kafka event messages reprocessed (same offset redelivered outside a
+// rebalance). Should stay ~0. A sustained non-zero rate means the consumer is
+// re-delivering messages it already handled — an offset-handling/duplicate bug.
+export const kafkaReprocessedTotal = new client.Counter({
+ name: 'kafka_events_reprocessed_total',
+ help: 'Kafka event messages reprocessed (offset redelivered outside a rebalance)',
+ labelNames: ['partition'],
+ registers: [registry],
+});
+
+// Handler exceptions on the Kafka ingest path, counted per attempt — retries
+// included, so the series shows real failure pressure and not just the
+// messages that ran out of attempts.
+export const kafkaHandlerFailuresTotal = new client.Counter({
+ name: 'kafka_events_handler_failures_total',
+ help: 'Kafka event handler exceptions (each attempt, retries included)',
+ labelNames: ['partition'],
+ registers: [registry],
+});
+
+// Messages written to the dead-letter topic after their attempts were
+// exhausted (or after they failed to parse). One increment per message.
+export const kafkaDeadLetteredTotal = new client.Counter({
+ name: 'kafka_events_dead_lettered_total',
+ help: 'Kafka event messages produced to the dead-letter topic',
+ labelNames: ['partition', 'reason'],
+ registers: [registry],
+});
+
+// The dead-letter produce itself failed. The offset is then left unresolved
+// and the message is redelivered, so this is a stuck partition, not a loss.
+export const kafkaDeadLetterFailedTotal = new client.Counter({
+ name: 'kafka_events_dead_letter_failed_total',
+ help: 'Failed attempts to produce a Kafka event message to the dead-letter topic',
+ labelNames: ['partition'],
+ registers: [registry],
+});
+
+export const sessionsStarted = new client.Counter({
+ name: 'sessions_started_total',
+ help: 'session_start events emitted, by lifecycle kind',
+ labelNames: ['kind'], // 'new' | 'boundary'
+ registers: [registry],
+});
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index 33abe4199..d1e8bc1f1 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -1169,6 +1169,9 @@ importers:
js-yaml:
specifier: ^4.1.0
version: 4.1.1
+ kafkajs:
+ specifier: ^2.2.4
+ version: 2.2.4
lru-cache:
specifier: ^11.2.4
version: 11.5.1