OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.
ralph(M8-003): kafka consumer + incoming-event + create-session-end into ingest/session
ralph3e785893af64
diff --git a/apps/worker/src/jobs/events.incoming-event.ts b/apps/worker/src/jobs/events.incoming-event.tsindex cb0b9d338..ed4adeacb 100644--- a/apps/worker/src/jobs/events.incoming-event.ts+++ b/apps/worker/src/jobs/events.incoming-event.ts@@ -1,302 +1,40 @@-import { getTime, isSameDomain, parsePath } from '@openpanel/common';+// Dissolved into @openpanel/core's ingest module (M8-003): the whole handler+// — session lifecycle decisions, session_start emission, referrer inheritance+// and the producer-minted event id — moved to+// packages/core/src/modules/ingest/src/incoming-event-handler.ts. This file+// stays (DELEGATE PATTERN) because it is what binds core's injected+// dependencies to V1's world: this worker's registry counters, this worker's+// pino, and `enqueueSessionEndV2`, which speaks @openpanel/queue's BullMQ+// wire shape and therefore cannot live in core.import {- getReferrerWithQuery,- type ILogger,- parseReferrer,+ type IncomingEventDelivery,+ type IncomingEventDeps,+ type IncomingEventPayload,+ incomingEvent as incomingEventCore,+ loadIncomingEventDeps,} from '@openpanel/core';-import type {- IServiceCreateEventPayload,- IServiceCreateEventPayloadWithId,- IServiceEvent,-} from '@openpanel/db';-import {- checkNotificationRulesForEvent,- createEvent,- db,- getProjectByIdCached,- matchEvent,- sessionBuffer,-} from '@openpanel/db';-import type { EventsQueuePayloadIncomingEvent } from '@openpanel/queue';-import { anyPass, isEmpty, isNil, mergeDeepRight, omit, reject } from 'ramda';import { sessionEndsEnqueued, sessionsStarted } from '@/metrics';import { logger as baseLogger } from '@/utils/logger';import { enqueueSessionEndV2 } from '@/utils/session-handler';-const GLOBAL_PROPERTIES = ['__path', '__referrer', '__timestamp', '__revenue'];--// Strip empty/nullish from B, then deep-merge over A.-const merge = <A, B>(a: Partial<A>, b: Partial<B>): A & B =>- mergeDeepRight(a, reject(anyPass([isEmpty, isNil]))(b)) as A & B;--async function isEventExcludedByProjectFilter(- payload: IServiceCreateEventPayload,- projectId: string-): Promise<boolean> {- const project = await getProjectByIdCached(projectId);- const eventExcludeFilters = (project?.filters ?? []).filter(- (f) => f.type === 'event'- );- if (eventExcludeFilters.length === 0) {- return false;- }- return eventExcludeFilters.some((filter) => matchEvent(payload, filter));-}--/**- * Records the project's first-ever event timestamp exactly once. The cached- * project read makes this a no-op on every event after the first; the- * conditional update keeps concurrent workers idempotent.- */-async function markFirstEvent(projectId: string, logger: ILogger) {- const project = await getProjectByIdCached(projectId);- if (!project || project.firstEventAt) {- return;- }- await db.project.updateMany({- where: { id: projectId, firstEventAt: null },- data: { firstEventAt: new Date() },+let deps: Promise<IncomingEventDeps> | null = null;++// Resolved once per process: the loader dynamically imports @openpanel/db's+// buffers, prisma client and notification service, and this is the hot path.+function getDeps(): Promise<IncomingEventDeps> {+ deps ??= loadIncomingEventDeps(baseLogger, enqueueSessionEndV2, {+ // V1's own registry, so this worker's /metrics body is unchanged.+ metrics: {+ sessionStarted: (kind) => sessionsStarted.inc({ kind }),+ sessionEndEnqueued: (source) => sessionEndsEnqueued.inc({ source }),+ },});- await getProjectByIdCached.clear(projectId);- logger.info({ projectId }, 'Project received its first event');+ return deps;}-async function createEventAndNotify(- payload: IServiceCreateEventPayloadWithId,- logger: ILogger,- projectId: string-) {- const isExcluded = await isEventExcludedByProjectFilter(payload, projectId);- if (isExcluded) {- logger.info(- { event: payload.name, projectId },- 'Event excluded by project filter'- );- return null;- }-- logger.info({ event: payload }, 'Creating event');- const [event] = await Promise.all([- createEvent(payload),- checkNotificationRulesForEvent(payload).catch(() => null),- ]);- // Only after the event is accepted — recording the first event before a- // failed createEvent would leave the activation checklist claiming data- // arrived that was never persisted.- await markFirstEvent(projectId, logger).catch(() => null);- return event;-}--const parseRevenue = (revenue: unknown): number | undefined => {- if (!revenue) {- return undefined;- }- if (typeof revenue === 'number') {- return revenue;- }- if (typeof revenue === 'string') {- const parsed = Number.parseFloat(revenue);- return Number.isNaN(parsed) ? undefined : parsed;- }- return undefined;-};-export async function incomingEvent(- jobPayload: EventsQueuePayloadIncomingEvent['payload'],- // Kafka delivery coordinates, when the event came through the Kafka consumer.- // Logged so a duplicate row in ClickHouse can be traced back to the exact- // partition/offset that produced it.- meta?: { partition: number; offset: string }+ jobPayload: IncomingEventPayload,+ meta?: IncomingEventDelivery) {- const {- geo,- event: body,- headers,- projectId,- deviceId,- sessionId,- uaInfo,- // Producer-minted; a redelivered message re-creates the same row id.- // Absent on messages produced before the field existed — createEvent then- // falls back to a fresh uuid.- id: eventId,- } = jobPayload;- const properties = body.properties ?? {};- const reqId = headers['request-id'] ?? 'unknown';- const logger = baseLogger.child({- reqId,- ...(meta- ? { kafkaPartition: meta.partition, kafkaOffset: meta.offset }- : {}),- });- const getProperty = (name: string): string | undefined => {- // replace thing is just for older sdks when we didn't have `__`- // remove when kiddokitchen app (24.09.02) is not used anymore- return (- ((properties[name] || properties[name.replace('__', '')]) as- | string- | null- | undefined) ?? undefined- );- };-- const profileId = body.profileId ? String(body.profileId) : '';- const createdAt = new Date(body.timestamp);- const isTimestampFromThePast = body.isTimestampFromThePast;- const url = getProperty('__path');- const { path, hash, query, origin } = parsePath(url);- const referrer = isSameDomain(getProperty('__referrer'), url)- ? null- : parseReferrer(getProperty('__referrer'));- const utmReferrer = getReferrerWithQuery(query);- const sdkName = headers['openpanel-sdk-name'];- const sdkVersion = headers['openpanel-sdk-version'];-- const baseEvent: IServiceCreateEventPayload = {- name: body.name,- profileId,- projectId,- deviceId,- sessionId,- properties: omit(GLOBAL_PROPERTIES, {- ...properties,- __hash: hash,- __query: query,- }),- groups: body.groups ?? [],- createdAt,- duration: 0,- sdkName,- sdkVersion,- city: geo.city,- country: geo.country,- region: geo.region,- longitude: geo.longitude,- latitude: geo.latitude,- path,- origin,- referrer: referrer?.url || '',- referrerName: utmReferrer?.name || referrer?.name || referrer?.url,- referrerType: utmReferrer?.type || referrer?.type || '',- os: uaInfo.os,- osVersion: uaInfo.osVersion,- browser: uaInfo.browser,- browserVersion: uaInfo.browserVersion,- device: uaInfo.device,- brand: uaInfo.brand,- model: uaInfo.model,- revenue:- body.name === 'revenue' && '__revenue' in properties- ? parseRevenue(properties.__revenue)- : undefined,- };-- // Server-side and "timestamp from the past" events ride alongside an- // existing client session if there is one — they don't open / close- // sessions of their own.- if (uaInfo.isServer || isTimestampFromThePast) {- const session =- profileId && !isTimestampFromThePast- ? await sessionBuffer.getExistingSession({ profileId, projectId })- : null;-- const payload = {- ...baseEvent,- deviceId: session?.device_id ?? '',- sessionId: session?.id ?? '',- referrer: session?.referrer ?? undefined,- referrerName: session?.referrer_name ?? undefined,- referrerType: session?.referrer_type ?? undefined,- path: session?.exit_path ?? baseEvent.path,- origin: session?.exit_origin ?? baseEvent.origin,- os: session?.os ?? baseEvent.os,- osVersion: session?.os_version ?? baseEvent.osVersion,- browserVersion: session?.browser_version ?? baseEvent.browserVersion,- browser: session?.browser ?? baseEvent.browser,- device: session?.device ?? baseEvent.device,- brand: session?.brand ?? baseEvent.brand,- model: session?.model ?? baseEvent.model,- city: session?.city ?? baseEvent.city,- country: session?.country ?? baseEvent.country,- region: session?.region ?? baseEvent.region,- longitude: session?.longitude ?? baseEvent.longitude,- latitude: session?.latitude ?? baseEvent.latitude,- };-- return createEventAndNotify(- { ...(payload as IServiceEvent), id: eventId },- logger,- projectId- );- }-- if (await isEventExcludedByProjectFilter(baseEvent, projectId)) {- logger.info(- { event: baseEvent.name, projectId },- 'Skipping session_start and event (excluded by project filter)'- );- return null;- }-- // The single source of truth for session lifecycle. Reads the current- // session, decides extend/new/boundary, writes back. The returned- // `current` is the canonical session — use its referrer fields for- // inheritance, just like the previous behavior.- const session = await sessionBuffer.ingest(baseEvent);-- if (session?.kind === 'boundary') {- // Close the old session in a separate job (one Redis-buffered insert- // for the session_end event + notification rule check). Idempotent via- // BullMQ jobId dedup.- await enqueueSessionEndV2({- payload: baseEvent,- closedSession: session.closed,- })- .then(() => sessionEndsEnqueued.inc({ source: 'boundary' }))- .catch((error) => {- logger.error(- { err: error, deviceId, sessionId: session.closed.id },- 'Error enqueueing session_end on boundary'- );- });- }-- if (session?.kind === 'new' || session?.kind === 'boundary') {- sessionsStarted.inc({ kind: session.kind });- // No `id` here: session_start is a second, derived row and must not share- // the producer-minted id of the event that triggered it.- await createEventAndNotify(- {- ...baseEvent,- name: 'session_start',- createdAt: new Date(getTime(baseEvent.createdAt) - 100),- },- logger,- projectId- ).catch((error) => {- logger.error(- { err: error, event: baseEvent },- 'Error creating session start event'- );- throw error;- });- }-- // Inherit referrer fields from the canonical session for the actual event.- // For 'extend' this preserves the original session's referrer across- // mid-session events; for 'new' / 'boundary' it's the event's own referrer- // (which `ingest` just stored on the fresh session).- const finalPayload: IServiceCreateEventPayload = session- ? merge(baseEvent, {- referrer: session.current.referrer,- referrerName: session.current.referrer_name,- referrerType: session.current.referrer_type,- } as Partial<IServiceCreateEventPayload>)- : baseEvent;-- return createEventAndNotify(- { ...finalPayload, id: eventId },- logger,- projectId- );+ return incomingEventCore(jobPayload, await getDeps(), meta);}diff --git a/apps/worker/src/jobs/events.incoming-events.test.ts b/apps/worker/src/jobs/events.incoming-events.test.tsdeleted file mode 100644index 38d62d5a9..000000000--- a/apps/worker/src/jobs/events.incoming-events.test.ts+++ /dev/null@@ -1,426 +0,0 @@-import {- createEvent,- formatClickhouseDate,- type IClickhouseSession,- sessionBuffer,-} from '@openpanel/db';-import {- type EventsQueuePayloadIncomingEvent,- sessionsQueue,-} from '@openpanel/queue';-import type { Job } from 'bullmq';-import { beforeEach, describe, expect, it, type Mock, vi } from 'vitest';-import { incomingEvent } from './events.incoming-event';--vi.mock('@openpanel/queue');-vi.mock('@openpanel/db', async () => {- const actual = await vi.importActual('@openpanel/db');- return {- ...actual,- createEvent: vi.fn(),- checkNotificationRulesForEvent: vi.fn().mockResolvedValue(true),- getProjectByIdCached: vi.fn().mockResolvedValue({ filters: [] }),- matchEvent: vi.fn().mockReturnValue(false),- sessionBuffer: {- // name/getBufferSize keep metrics.ts happy (it registers a per-buffer- // gauge at import); the tests only drive getExistingSession/ingest.- name: 'session',- getBufferSize: vi.fn().mockResolvedValue(0),- getExistingSession: vi.fn(),- ingest: vi.fn(),- },- };-});--const projectId = 'test-project';-const deviceId = 'device-123';-const newSessionId = 'a1b2c3d4-e5f6-4789-a012-345678901234';-// What the producer (/track) would have minted for one request.-const producedEventId = '11111111-2222-4333-8444-555555555555';-const geo = {- country: 'US',- city: 'New York',- region: 'NY',- longitude: 0,- latitude: 0,-};--const uaInfo: EventsQueuePayloadIncomingEvent['payload']['uaInfo'] = {- isServer: false,- device: 'desktop',- os: 'Windows',- osVersion: '10',- browser: 'Chrome',- browserVersion: '91.0.4472.124',- brand: '',- model: '',-};--const uaInfoServer: EventsQueuePayloadIncomingEvent['payload']['uaInfo'] = {- isServer: true,- device: 'server',- os: '',- osVersion: '',- browser: '',- browserVersion: '',- brand: '',- model: '',-};--function makeSession(- overrides: Partial<IClickhouseSession> = {}-): IClickhouseSession {- const now = new Date();- return {- id: 'session-existing',- project_id: projectId,- device_id: deviceId,- profile_id: '',- event_count: 1,- screen_view_count: 0,- entry_path: '/',- entry_origin: 'https://example.com',- exit_path: '/',- exit_origin: 'https://example.com',- created_at: formatClickhouseDate(now),- ended_at: formatClickhouseDate(now),- os: 'Windows',- os_version: '10',- browser: 'Chrome',- browser_version: '91.0.4472.124',- device: 'desktop',- brand: '',- model: '',- country: 'US',- region: 'NY',- city: 'New York',- longitude: 0,- latitude: 0,- duration: 0,- referrer: '',- referrer_name: '',- referrer_type: '',- is_bounce: true,- utm_term: '',- utm_source: '',- utm_campaign: '',- utm_content: '',- utm_medium: '',- revenue: 0,- sign: 1,- version: 1,- groups: [],- ...overrides,- } satisfies IClickhouseSession;-}--function buildJobData(- overrides: Partial<EventsQueuePayloadIncomingEvent['payload']> = {}-): EventsQueuePayloadIncomingEvent['payload'] {- return {- geo,- event: {- name: 'test_event',- timestamp: new Date().toISOString(),- isTimestampFromThePast: false,- properties: { __path: 'https://example.com/test' },- },- uaInfo,- headers: {- 'request-id': '123',- 'user-agent': 'Mozilla/5.0',- 'openpanel-sdk-name': 'web',- 'openpanel-sdk-version': '1.0.0',- },- projectId,- deviceId,- sessionId: newSessionId,- ...overrides,- };-}--describe('incomingEvent', () => {- beforeEach(() => {- vi.clearAllMocks();- (createEvent as Mock).mockImplementation((event) => event);- });-- it('emits session_start when ingest returns kind="new"', async () => {- vi.mocked(sessionBuffer.ingest).mockResolvedValueOnce({- kind: 'new',- current: makeSession({ id: newSessionId }),- });-- await incomingEvent(buildJobData());-- expect(sessionsQueue.add).not.toHaveBeenCalled();- const calls = (createEvent as Mock).mock.calls;- expect(calls).toHaveLength(2);- expect(calls[0]![0].name).toBe('session_start');- expect(calls[1]![0].name).toBe('test_event');- });-- it('skips session_start when ingest returns kind="extend"', async () => {- vi.mocked(sessionBuffer.ingest).mockResolvedValueOnce({- kind: 'extend',- current: makeSession(),- });-- await incomingEvent(buildJobData());-- expect(sessionsQueue.add).not.toHaveBeenCalled();- const sessionStartCalls = (createEvent as Mock).mock.calls.filter(- ([arg]) => arg?.name === 'session_start'- );- expect(sessionStartCalls).toHaveLength(0);- });-- it('closes old session and emits session_start when ingest returns kind="boundary"', async () => {- const closed = makeSession({ id: 'old-session-id' });- const current = makeSession({ id: newSessionId });- vi.mocked(sessionBuffer.ingest).mockResolvedValueOnce({- kind: 'boundary',- current,- closed,- });-- const spy = vi- .spyOn(sessionsQueue, 'add')- .mockResolvedValue({} as Job);-- await incomingEvent(buildJobData());-- expect(spy).toHaveBeenCalledTimes(1);- const [, payload, opts] = spy.mock.calls[0]!;- expect((payload as any).type).toBe('createSessionEnd');- expect((payload as any).payload.sessionId).toBe('old-session-id');- expect((payload as any).snapshot.id).toBe('old-session-id');- expect(opts?.jobId).toBe('sessionEnd:v2:old-session-id');-- const calls = (createEvent as Mock).mock.calls;- expect(calls.filter(([a]) => a?.name === 'session_start')).toHaveLength(1);- expect(calls.filter(([a]) => a?.name === 'test_event')).toHaveLength(1);- });-- it('inherits referrer from current session on the actual event', async () => {- vi.mocked(sessionBuffer.ingest).mockResolvedValueOnce({- kind: 'extend',- current: makeSession({- referrer: 'https://google.com',- referrer_name: 'Google',- referrer_type: 'search',- }),- });-- await incomingEvent(buildJobData());-- const testEventCall = (createEvent as Mock).mock.calls.find(- ([a]) => a?.name === 'test_event'- );- expect(testEventCall![0].referrer).toBe('https://google.com');- expect(testEventCall![0].referrerName).toBe('Google');- expect(testEventCall![0].referrerType).toBe('search');- });-- it('handles server events with existing profile session', async () => {- const timestamp = new Date();- const jobData = buildJobData({- event: {- name: 'server_event',- timestamp: timestamp.toISOString(),- properties: { custom_property: 'test_value' },- profileId: 'profile-123',- isTimestampFromThePast: false,- },- uaInfo: uaInfoServer,- deviceId: '',- sessionId: '',- headers: {- 'user-agent': 'OpenPanel Server/1.0',- 'openpanel-sdk-name': 'server',- 'openpanel-sdk-version': '1.0.0',- 'request-id': '123',- },- });-- vi.mocked(sessionBuffer.getExistingSession).mockResolvedValueOnce(- makeSession({- id: 'last-session-456',- device_id: 'last-device-123',- profile_id: 'profile-123',- os: 'iOS',- os_version: '15.0',- browser: 'Safari',- browser_version: '15.0',- device: 'mobile',- brand: 'Apple',- model: 'iPhone',- country: 'CA',- region: 'ON',- city: 'Toronto',- entry_path: '/last-path',- entry_origin: 'https://example.com',- exit_path: '/last-path',- exit_origin: 'https://example.com',- referrer: 'https://google.com',- referrer_name: 'Google',- referrer_type: 'search',- is_bounce: false,- event_count: 0,- screen_view_count: 0,- })- );-- await incomingEvent(jobData);-- expect(sessionBuffer.ingest).not.toHaveBeenCalled();- expect(sessionsQueue.add).not.toHaveBeenCalled();- expect((createEvent as Mock).mock.calls[0]![0]).toMatchObject({- name: 'server_event',- deviceId: 'last-device-123',- sessionId: 'last-session-456',- profileId: 'profile-123',- city: 'Toronto',- country: 'CA',- referrer: 'https://google.com',- });- });-- it('handles server events without any active session', async () => {- vi.mocked(sessionBuffer.getExistingSession).mockResolvedValueOnce(null);-- await incomingEvent(- buildJobData({- event: {- name: 'server_event',- timestamp: new Date().toISOString(),- properties: { custom_property: 'test_value' },- profileId: 'profile-123',- isTimestampFromThePast: false,- },- uaInfo: uaInfoServer,- deviceId: '',- sessionId: '',- headers: {- 'user-agent': 'OpenPanel Server/1.0',- 'openpanel-sdk-name': 'server',- 'openpanel-sdk-version': '1.0.0',- 'request-id': '123',- },- })- );-- expect(sessionBuffer.ingest).not.toHaveBeenCalled();- expect(sessionsQueue.add).not.toHaveBeenCalled();- expect((createEvent as Mock).mock.calls[0]![0]).toMatchObject({- name: 'server_event',- deviceId: '',- sessionId: '',- profileId: 'profile-123',- });- });-- it('re-creates the same row id when Kafka redelivers the message', async () => {- const session = makeSession({ id: newSessionId });- vi.mocked(sessionBuffer.ingest)- .mockResolvedValueOnce({ kind: 'new', current: session })- .mockResolvedValueOnce({ kind: 'extend', current: session });-- // Same message, delivered twice: identical payload, identical offset.- const jobData = buildJobData({ id: producedEventId });- const delivery = { partition: 3, offset: '42' };- await incomingEvent(jobData, delivery);- await incomingEvent(jobData, delivery);-- const eventCalls = (createEvent as Mock).mock.calls.filter(- ([a]) => a?.name === 'test_event'- );- expect(eventCalls).toHaveLength(2);- expect(eventCalls[0]![0].id).toBe(producedEventId);- expect(eventCalls[1]![0].id).toBe(producedEventId);-- // The derived session_start row must NOT reuse the event's id.- const sessionStartCalls = (createEvent as Mock).mock.calls.filter(- ([a]) => a?.name === 'session_start'- );- expect(sessionStartCalls).toHaveLength(1);- expect(sessionStartCalls[0]![0].id).toBeUndefined();- });-- it('carries the producer-minted id on server-side events', async () => {- vi.mocked(sessionBuffer.getExistingSession).mockResolvedValueOnce(null);-- await incomingEvent(- buildJobData({- id: producedEventId,- event: {- name: 'server_event',- timestamp: new Date().toISOString(),- isTimestampFromThePast: false,- profileId: 'profile-123',- },- uaInfo: uaInfoServer,- deviceId: '',- sessionId: '',- })- );-- expect((createEvent as Mock).mock.calls[0]![0].id).toBe(producedEventId);- });-- it('leaves the id undefined for a payload produced without one', async () => {- vi.mocked(sessionBuffer.ingest).mockResolvedValueOnce({- kind: 'extend',- current: makeSession(),- });-- await incomingEvent(buildJobData());-- expect((createEvent as Mock).mock.calls[0]![0].id).toBeUndefined();- });-- it('emits session_start only once across 3 rapid events (new → extend → extend)', async () => {- const session = makeSession({ id: newSessionId });- vi.mocked(sessionBuffer.ingest)- .mockResolvedValueOnce({ kind: 'new', current: session })- .mockResolvedValueOnce({ kind: 'extend', current: session })- .mockResolvedValueOnce({ kind: 'extend', current: session });-- await incomingEvent(- buildJobData({- event: {- name: 'e1',- timestamp: new Date().toISOString(),- isTimestampFromThePast: false,- properties: { __path: 'https://example.com/test' },- },- })- );- await incomingEvent(- buildJobData({- event: {- name: 'e2',- timestamp: new Date().toISOString(),- isTimestampFromThePast: false,- properties: { __path: 'https://example.com/test' },- },- })- );- await incomingEvent(- buildJobData({- event: {- name: 'e3',- timestamp: new Date().toISOString(),- isTimestampFromThePast: false,- properties: { __path: 'https://example.com/test' },- },- })- );-- const sessionStartCalls = (createEvent as Mock).mock.calls.filter(- ([a]) => a?.name === 'session_start'- );- expect(sessionStartCalls).toHaveLength(1);- expect(sessionsQueue.add).not.toHaveBeenCalled();- });-});diff --git a/apps/worker/src/jobs/events.kafka-consumer.ts b/apps/worker/src/jobs/events.kafka-consumer.tsindex 0454f7e01..bde98df53 100644--- a/apps/worker/src/jobs/events.kafka-consumer.ts+++ b/apps/worker/src/jobs/events.kafka-consumer.ts@@ -1,14 +1,18 @@+// Dissolved into @openpanel/core's ingest module (M8-003): the batch handler+// (per-key serial groups, the ascending contiguous-prefix offset walk,+// at-least-once, bounded retry, DLQ) is+// packages/core/src/modules/ingest/src/consumer.ts. This file stays (DELEGATE+// PATTERN) and supplies the halves core cannot reach: @openpanel/queue's+// kafkajs client and its topic / consumer-group / retry constants, this+// worker's registry counters, and the events heartbeat.+import { startKafkaEventsConsumer as startKafkaEventsConsumerCore } from '@openpanel/core';import {createKafkaEventsConsumer,- type DeadLetterMessage,- type EachBatchPayload,- type EventsQueuePayloadIncomingEvent,KAFKA_EVENTS_TOPIC,KAFKA_HANDLER_MAX_ATTEMPTS,KAFKA_HANDLER_RETRY_INITIAL_MS,KAFKA_HANDLER_RETRY_MAX_MS,KAFKA_PARTITIONS_CONCURRENT,- type KafkaMessage,kafkaLogger,produceDeadLetterEvent,} from '@openpanel/queue';@@ -22,385 +26,32 @@ import { logger } from '../utils/logger';import { markEventsActivity } from '../utils/worker-heartbeat';import { incomingEvent } from './events.incoming-event';-export interface KafkaConsumerHandle {- stop: () => Promise<void>;-}--// Heartbeat every N messages within a per-key group. The default kafkajs-// sessionTimeout is 30s — calling heartbeat every 16 messages keeps us-// comfortably under that even for slow handlers.-const HEARTBEAT_EVERY = 16;--const RETRY_BACKOFF_FACTOR = 2;--/** Why a message was dead-lettered. A label value — keep the set small. */-type DeadLetterReason = 'parse_error' | 'handler_error';--interface ConsumerLogger {- info: (obj: object, msg?: string) => void;- warn: (obj: object, msg?: string) => void;- error: (obj: object, msg?: string) => void;-}--interface ConsumerMetrics {- reprocessed: (partition: string) => void;- handlerFailed: (partition: string) => void;- deadLettered: (partition: string, reason: DeadLetterReason) => void;- deadLetterFailed: (partition: string) => void;-}--export interface EventsBatchHandlerDeps {- handleEvent: (- payload: EventsQueuePayloadIncomingEvent['payload'],- meta: { partition: number; offset: string }- ) => Promise<unknown>;- sendToDeadLetter: (message: DeadLetterMessage) => Promise<void>;- logger: ConsumerLogger;- metrics: ConsumerMetrics;- onActivity: () => void;- topic: string;- maxAttempts: number;- initialRetryMs: number;- maxRetryMs: number;- sleep: (ms: number) => Promise<void>;-}--const defaultSleep = (ms: number): Promise<void> =>- new Promise((resolve) => setTimeout(resolve, ms));--const errorMessage = (err: unknown): string =>- err instanceof Error ? err.message : String(err);--const defaultDeps = (): EventsBatchHandlerDeps => ({- handleEvent: incomingEvent,- sendToDeadLetter: produceDeadLetterEvent,- logger,- metrics: {- reprocessed: (partition) => kafkaReprocessedTotal.inc({ partition }),- handlerFailed: (partition) => kafkaHandlerFailuresTotal.inc({ partition }),- deadLettered: (partition, reason) =>- kafkaDeadLetteredTotal.inc({ partition, reason }),- deadLetterFailed: (partition) =>- kafkaDeadLetterFailedTotal.inc({ partition }),- },- onActivity: markEventsActivity,- topic: KAFKA_EVENTS_TOPIC,- maxAttempts: KAFKA_HANDLER_MAX_ATTEMPTS,- initialRetryMs: KAFKA_HANDLER_RETRY_INITIAL_MS,- maxRetryMs: KAFKA_HANDLER_RETRY_MAX_MS,- sleep: defaultSleep,-});--export interface EventsBatchHandler {- eachBatch: (payload: EachBatchPayload) => Promise<void>;- /** Called on GROUP_JOIN: a new assignment invalidates the watermarks. */- resetWatermarks: () => void;-}--/**- * The `eachBatch` handler, with its dependencies injected so the offset walk,- * the per-key serial groups and the failure paths are testable without a- * broker.- *- * Delivery contract (ADR-004): at-least-once. An offset is resolved only once- * its message has been handled, dead-lettered, or deliberately skipped —- * never merely because it failed.- */-export function createEventsBatchHandler(- overrides: Partial<EventsBatchHandlerDeps> = {}-): EventsBatchHandler {- const deps: EventsBatchHandlerDeps = { ...defaultDeps(), ...overrides };-- // Highest Kafka offset we have *resolved* (committed) per `topic-partition`,- // tracked across batches. Used purely for the reprocess detector below: if we- // ever see an offset at or below this watermark again, the message is being- // redelivered (at-least-once duplicate) outside of a rebalance — which is the- // signature of an offset-handling bug. Cleared on GROUP_JOIN so legitimate- // post-rebalance redelivery from the last committed offset doesn't trip it.- const resolvedHWM = new Map<string, number>();-- const deadLetter = async (- message: KafkaMessage,- partition: number,- reason: DeadLetterReason,- err: unknown- ): Promise<boolean> => {- const label = String(partition);- try {- await deps.sendToDeadLetter({- key: message.key,- value: message.value,- headers: message.headers,- topic: deps.topic,- partition,- offset: message.offset,- reason,- error: errorMessage(err),- });- } catch (dlqErr) {- // The event is not lost: leaving the offset unresolved stops the- // contiguous-prefix walk here, so the broker redelivers this message.- deps.metrics.deadLetterFailed(label);- deps.logger.error(- { err: dlqErr, partition, offset: message.offset, reason },- 'kafka dead-letter produce failed — offset left unresolved for redelivery'- );- return false;- }- deps.metrics.deadLettered(label, reason);- deps.logger.error(- { err, partition, offset: message.offset, reason },- 'kafka message dead-lettered'- );- return true;- };-- const runHandlerWithRetry = async (- payload: EventsQueuePayloadIncomingEvent['payload'],- partition: number,- offset: string- ): Promise<{ ok: true } | { ok: false; err: unknown }> => {- const label = String(partition);- let delayMs = deps.initialRetryMs;- let lastErr: unknown;-- for (let attempt = 1; attempt <= deps.maxAttempts; attempt++) {- try {- await deps.handleEvent(payload, { partition, offset });- return { ok: true };- } catch (err) {- lastErr = err;- deps.metrics.handlerFailed(label);- deps.logger.warn(- {- err,- partition,- offset,- projectId: payload.projectId,- attempt,- maxAttempts: deps.maxAttempts,- },- 'kafka incomingEvent handler failed'- );- if (attempt < deps.maxAttempts) {- await deps.sleep(delayMs);- delayMs = Math.min(delayMs * RETRY_BACKOFF_FACTOR, deps.maxRetryMs);- }- }- }-- return { ok: false, err: lastErr };- };-- /** @returns true when the offset may be resolved. */- const processMessage = async (- message: KafkaMessage,- partition: number- ): Promise<boolean> => {- let payload: EventsQueuePayloadIncomingEvent['payload'];- try {- if (!message.value) {- throw new Error('kafka message has no value');- }- payload = JSON.parse(- message.value.toString()- ) as EventsQueuePayloadIncomingEvent['payload'];- } catch (err) {- deps.logger.error(- { err, partition, offset: message.offset },- 'kafka message parse failed'- );- // Deterministic: retrying identical bytes cannot succeed.- return deadLetter(message, partition, 'parse_error', err);- }-- const result = await runHandlerWithRetry(- payload,- partition,- message.offset- );- if (result.ok) {- return true;- }- return deadLetter(message, partition, 'handler_error', result.err);- };-- const eachBatch = async ({- batch,- resolveOffset,- heartbeat,- isRunning,- isStale,- }: EachBatchPayload): Promise<void> => {- if (batch.messages.length === 0) {- return;- }-- const pk = `${batch.topic}-${batch.partition}`;- // Watermark from *previous* batches. Anything at or below this that we see- // now is a redelivery. Captured before this batch so intra-batch- // out-of-order processing (normal, see below) is never counted.- const priorHWM = resolvedHWM.get(pk) ?? -1;-- // Group by partition key (= deviceId or `${projectId}:${profileId}`).- // Same-key messages stay serial so sessionBuffer/session-end-job state- // can't race; different keys run in parallel via Promise.all.- // Keyless messages get their own singleton group.- const groups = new Map<string, KafkaMessage[]>();- for (const m of batch.messages) {- const key = m.key ? m.key.toString() : `__no_key__:${m.offset}`;- const arr = groups.get(key);- if (arr) {- arr.push(m);- } else {- groups.set(key, [m]);- }- }+export type { KafkaConsumerHandle } from '@openpanel/core';- // Offsets that finished processing this batch. We resolve them AFTER all- // groups complete, in strict ascending order (see the loop below).- //- // Why: KafkaJS `resolveOffset` is last-write-wins and the next fetch- // starts from the last resolved offset (offsetManager.nextOffset). If we- // resolved inside the concurrent per-key loop, a lower offset resolving- // after a higher one would move the fetch position BACKWARDS and- // re-deliver everything in between — the root cause of the duplicate- // events. Resolving the contiguous ascending prefix at the end avoids- // both duplicates (never regress) and loss (stop at the first gap).- const processed = new Set<string>();- let processedCount = 0;-- await Promise.all(- [...groups.values()].map(async (msgs) => {- for (const m of msgs) {- if (!isRunning() || isStale()) {- return;- }-- // Reprocess detector: only fires for offsets already resolved in a- // PRIOR batch (redelivery). Intra-batch out-of-order processing- // across key-groups is expected and is not flagged.- if (Number(m.offset) <= priorHWM) {- deps.metrics.reprocessed(String(batch.partition));- deps.logger.warn(- {- partition: batch.partition,- offset: m.offset,- resolvedHighWaterMark: priorHWM,- },- 'kafka offset REPROCESSED — at-least-once duplicate (outside rebalance)'- );- }-- const resolvable = await processMessage(m, batch.partition);- if (!resolvable) {- // Stop this key-group: continuing past an unresolvable offset only- // adds duplicates, since the whole tail is redelivered anyway.- return;- }-- processed.add(m.offset);- processedCount += 1;- if (processedCount % HEARTBEAT_EVERY === 0) {- await heartbeat();- }- }- })- );-- // Resolve in strict ascending offset order, stopping at the first offset- // that did not finish (e.g. an isStale/isRunning early-return mid-batch).- // batch.messages is already ordered by offset.- let newHWM = priorHWM;- for (const m of batch.messages) {- if (!processed.has(m.offset)) {- break;- }- resolveOffset(m.offset);- newHWM = Math.max(newHWM, Number(m.offset));- }- resolvedHWM.set(pk, newHWM);-- await heartbeat();- deps.onActivity();- };-- return { eachBatch, resetWatermarks: () => resolvedHWM.clear() };-}--export async function startKafkaEventsConsumer(): Promise<KafkaConsumerHandle> {- const consumer = createKafkaEventsConsumer();- const handler = createEventsBatchHandler();- await consumer.connect();- await consumer.subscribe({+export function startKafkaEventsConsumer() {+ return startKafkaEventsConsumerCore({+ createConsumer: createKafkaEventsConsumer,+ logger,+ kafkaLogger,topic: KAFKA_EVENTS_TOPIC,- fromBeginning: false,- });-- consumer.on(consumer.events.HEARTBEAT, markEventsActivity);-- // ---- Lifecycle / rebalance ("re-election") visibility ----- // We had no logging for partition reassignment before; without it a rebalance- // storm (a common source of at-least-once duplicates) is invisible.- consumer.on(consumer.events.GROUP_JOIN, ({ payload }) => {- // A new assignment means partitions may have moved between members; reset- // the reprocess watermarks so legitimate resume-from-committed-offset after- // a rebalance is not flagged as a duplicate.- handler.resetWatermarks();- logger.info(- {- memberId: payload.memberId,- groupId: payload.groupId,- isLeader: payload.isLeader,- memberAssignment: payload.memberAssignment,- duration: payload.duration,- },- 'kafka consumer joined group (rebalance complete)'- );- });- consumer.on(consumer.events.REBALANCING, ({ payload }) => {- logger.warn(- { memberId: payload.memberId, groupId: payload.groupId },- 'kafka consumer rebalancing'- );- });- consumer.on(consumer.events.CRASH, ({ payload }) => {- logger.error(- {- err: payload.error,- groupId: payload.groupId,- restart: payload.restart,- },- 'kafka consumer crashed'- );- });- consumer.on(consumer.events.DISCONNECT, () => {- logger.warn('kafka consumer disconnected');- });- consumer.on(consumer.events.REQUEST_TIMEOUT, ({ payload }) => {- logger.warn(- { broker: payload.broker, clientId: payload.clientId },- 'kafka consumer request timeout'- );- });-- await consumer.run({partitionsConsumedConcurrently: KAFKA_PARTITIONS_CONCURRENT,- eachBatchAutoResolve: false,- eachBatch: handler.eachBatch,- });-- kafkaLogger.info(- {- topic: KAFKA_EVENTS_TOPIC,- partitionsConsumedConcurrently: KAFKA_PARTITIONS_CONCURRENT,- },- 'kafka events consumer running'- );-- return {- stop: async () => {- await consumer.disconnect();+ batch: {+ handleEvent: incomingEvent,+ sendToDeadLetter: produceDeadLetterEvent,+ logger,+ metrics: {+ reprocessed: (partition) => kafkaReprocessedTotal.inc({ partition }),+ handlerFailed: (partition) =>+ kafkaHandlerFailuresTotal.inc({ partition }),+ deadLettered: (partition, reason) =>+ kafkaDeadLetteredTotal.inc({ partition, reason }),+ deadLetterFailed: (partition) =>+ kafkaDeadLetterFailedTotal.inc({ partition }),+ },+ onActivity: markEventsActivity,+ maxAttempts: KAFKA_HANDLER_MAX_ATTEMPTS,+ initialRetryMs: KAFKA_HANDLER_RETRY_INITIAL_MS,+ maxRetryMs: KAFKA_HANDLER_RETRY_MAX_MS,},- };+ });}diff --git a/packages/core/package.json b/packages/core/package.jsonindex fa3a1d6a4..88934ae44 100644--- a/packages/core/package.json+++ b/packages/core/package.json@@ -52,6 +52,7 @@"fast-json-stable-hash": "^1.0.3","ipaddr.js": "^2.5.0","js-yaml": "^4.1.0",+ "kafkajs": "^2.2.4","lru-cache": "^11.2.4","mathjs": "^12.3.2","nanoid": "^5.1.6",diff --git a/packages/core/src/context.ts b/packages/core/src/context.tsindex 1f1f7f4c5..16824e9a9 100644--- a/packages/core/src/context.ts+++ b/packages/core/src/context.ts@@ -52,8 +52,9 @@ export interface AppDeps {/*** The events transport. `produceIncomingEvent` still lives in* @openpanel/queue, which imports core for its logger, so core cannot- * import it back (M8-002); main.ts supplies it and M8-003 replaces it with- * core's own Kafka producer.+ * import it back (M8-002); main.ts supplies it. It stays injected until+ * @openpanel/queue dies at P9 — M8-003 moved the consumer, not the+ * producer, and the consumer takes its kafkajs client the same way.*/produceIncomingEvent: IncomingEventProducer;logger: Logger;diff --git a/packages/core/src/index.ts b/packages/core/src/index.tsindex 79253497d..82177897a 100644--- a/packages/core/src/index.ts+++ b/packages/core/src/index.ts@@ -1151,6 +1151,34 @@ export {summarizeBotSignals,validateIngestRequest,} from './modules/ingest/ingest.service';+// The Kafka consumer and its per-message handler (M8-003). apps/worker's+// events.kafka-consumer.ts / events.incoming-event.ts are thin delegates that+// hand in the kafkajs client, the topic/group names and their own registry's+// counters — none of which this package spells, so they stay byte-identical.+export type {+ ConsumerLogger,+ ConsumerMetrics,+ DeadLetterMessage,+ DeadLetterReason,+ EventsBatchHandler,+ EventsBatchHandlerDeps,+ EventsConsumerDeps,+ KafkaConsumerHandle,+} from './modules/ingest/src/consumer';+export {+ createEventsBatchHandler,+ startKafkaEventsConsumer,+} from './modules/ingest/src/consumer';+export type {+ IncomingEventDelivery,+ IncomingEventDeps,+ IncomingEventMetrics,+ IncomingEventSessions,+} from './modules/ingest/src/incoming-event-handler';+export {+ incomingEvent,+ loadIncomingEventDeps,+} from './modules/ingest/src/incoming-event-handler';// Dissolved from @openpanel/db's services/insights* + referrer-spikes.service// (M5-001) — apps/worker's insight job files and packages/trpc's insight// router call these directly, the same way V1 reaches every other dissolveddiff --git a/apps/worker/src/jobs/events.kafka-consumer.test.ts b/packages/core/src/modules/ingest/src/consumer.test.tssimilarity index 77%rename from apps/worker/src/jobs/events.kafka-consumer.test.tsrename to packages/core/src/modules/ingest/src/consumer.test.tsindex 4c0d6c9d6..f4645d4cf 100644--- a/apps/worker/src/jobs/events.kafka-consumer.test.ts+++ b/packages/core/src/modules/ingest/src/consumer.test.ts@@ -3,32 +3,19 @@* An offset is resolved once its message has been handled, dead-lettered, or* deliberately skipped — never merely because it failed.*- * The batch handler is exercised directly with injected dependencies; the- * broker, the metrics registry and `incomingEvent` are all out of the picture.+ * Ported from apps/worker/src/jobs/events.kafka-consumer.test.ts (M8-003)+ * with the assertions unchanged. The batch handler is exercised directly with+ * injected dependencies; the broker, the metrics registry and `incomingEvent`+ * are all out of the picture — which is why no `mock.module` appears here.*/-import type {- DeadLetterMessage,- EachBatchPayload,- KafkaMessage,-} from '@openpanel/queue';-import { beforeEach, describe, expect, it, vi } from 'vitest';--vi.mock('../metrics', () => ({- kafkaReprocessedTotal: { inc: vi.fn() },- kafkaHandlerFailuresTotal: { inc: vi.fn() },- kafkaDeadLetteredTotal: { inc: vi.fn() },- kafkaDeadLetterFailedTotal: { inc: vi.fn() },-}));-vi.mock('./events.incoming-event', () => ({ incomingEvent: vi.fn() }));-vi.mock('../utils/logger', () => ({- logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() },-}));-+import { describe, expect, mock, test } from 'bun:test';+import type { EachBatchPayload, KafkaMessage } from 'kafkajs';import {createEventsBatchHandler,+ type DeadLetterMessage,type EventsBatchHandlerDeps,-} from './events.kafka-consumer';+} from './consumer';const PARTITION = 7;const TOPIC = 'events';@@ -57,8 +44,8 @@ function makeBatch(messages: KafkaMessage[],options: { isRunning?: () => boolean; isStale?: () => boolean } = {}) {- const resolveOffset = vi.fn();- const heartbeat = vi.fn().mockResolvedValue(undefined);+ const resolveOffset = mock((_offset: string) => undefined);+ const heartbeat = mock(async () => undefined);const payload = {batch: { topic: TOPIC, partition: PARTITION, messages },resolveOffset,@@ -69,44 +56,56 @@ function makeBatch(return { payload, resolveOffset, heartbeat };}-const resolvedOffsets = (resolveOffset: ReturnType<typeof vi.fn>): string[] =>+interface MockCalls {+ mock: { calls: unknown[][] };+}++const resolvedOffsets = (resolveOffset: MockCalls): string[] =>resolveOffset.mock.calls.map(([offset]) => offset as string);function makeDeps(overrides: Partial<EventsBatchHandlerDeps> = {}) {- const metrics = {- reprocessed: vi.fn(),- handlerFailed: vi.fn(),- deadLettered: vi.fn(),- deadLetterFailed: vi.fn(),+ const metrics: EventsBatchHandlerDeps['metrics'] = {+ reprocessed: mock(() => undefined),+ handlerFailed: mock(() => undefined),+ deadLettered: mock(() => undefined),+ deadLetterFailed: mock(() => undefined),};- const deps: Partial<EventsBatchHandlerDeps> = {- handleEvent: vi.fn().mockResolvedValue(undefined),- sendToDeadLetter: vi.fn().mockResolvedValue(undefined),- logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() },- onActivity: vi.fn(),+ const handleEvent = mock<EventsBatchHandlerDeps['handleEvent']>(+ async () => undefined+ );+ const sendToDeadLetter = mock<EventsBatchHandlerDeps['sendToDeadLetter']>(+ async () => undefined+ );+ const sleep = mock(async (_ms: number) => undefined);+ const deps: EventsBatchHandlerDeps = {+ handleEvent,+ sendToDeadLetter,+ logger: {+ info: mock(() => undefined),+ warn: mock(() => undefined),+ error: mock(() => undefined),+ },+ onActivity: mock(() => undefined),topic: TOPIC,maxAttempts: 3,initialRetryMs: 10,maxRetryMs: 25,- sleep: vi.fn().mockResolvedValue(undefined),+ sleep,...overrides,metrics: { ...metrics, ...overrides.metrics },};return {deps,metrics: deps.metrics as typeof metrics,- handleEvent: deps.handleEvent as ReturnType<typeof vi.fn>,- sendToDeadLetter: deps.sendToDeadLetter as ReturnType<typeof vi.fn>,- sleep: deps.sleep as ReturnType<typeof vi.fn>,+ handleEvent,+ sendToDeadLetter: (overrides.sendToDeadLetter ??+ sendToDeadLetter) as typeof sendToDeadLetter,+ sleep,};}-beforeEach(() => {- vi.clearAllMocks();-});-describe('offset walk', () => {- it('resolves the whole contiguous prefix in ascending order', async () => {+ test('resolves the whole contiguous prefix in ascending order', async () => {const { deps } = makeDeps();const handler = createEventsBatchHandler(deps);// Two interleaved key-groups: they run concurrently, so completion order@@ -123,7 +122,7 @@ describe('offset walk', () => {expect(resolvedOffsets(resolveOffset)).toEqual(['0', '1', '2', '3']);});- it('stops at the first gap, even when later offsets did finish', async () => {+ test('stops at the first gap, even when later offsets did finish', async () => {// Keys 'a' (0, 2, 4) and 'b' (1, 3, 5) run concurrently. 'a' is held on// its first message until 'b' has finished all of its own, then the// consumer stops — so 3 and 5 are processed while 2 never runs. Resolving@@ -158,15 +157,13 @@ describe('offset walk', () => {await handler.eachBatch(payload);- const handled = handleEvent.mock.calls.map(- ([p]) => (p as { projectId: string }).projectId- );+ const handled = handleEvent.mock.calls.map(([p]) => p.projectId);expect(handled).toContain('project-5');expect(handled).not.toContain('project-2');expect(resolvedOffsets(resolveOffset)).toEqual(['0', '1']);});- it('resolves nothing when the batch is stale before the first message', async () => {+ test('resolves nothing when the batch is stale before the first message', async () => {const { deps, handleEvent } = makeDeps();const handler = createEventsBatchHandler(deps);const { payload, resolveOffset } = makeBatch(@@ -182,7 +179,7 @@ describe('offset walk', () => {});describe('kill consumer mid-batch', () => {- it('loses no event: every unresolved offset is left for redelivery', async () => {+ test('loses no event: every unresolved offset is left for redelivery', async () => {const KILL_AFTER = 3;const TOTAL = 8;let handled = 0;@@ -215,11 +212,11 @@ describe('kill consumer mid-batch', () => {});describe('handler failure', () => {- it('retries with bounded backoff before giving up', async () => {+ test('retries with bounded backoff before giving up', async () => {const { deps, handleEvent, sleep, sendToDeadLetter } = makeDeps({maxAttempts: 4,});- handleEvent.mockRejectedValue(new Error('boom'));+ handleEvent.mockImplementation(() => Promise.reject(new Error('boom')));const handler = createEventsBatchHandler(deps);const { payload } = makeBatch([message(0, 'a')]);@@ -231,11 +228,15 @@ describe('handler failure', () => {expect(sendToDeadLetter).toHaveBeenCalledTimes(1);});- it('stops retrying as soon as the handler succeeds', async () => {+ test('stops retrying as soon as the handler succeeds', async () => {const { deps, handleEvent, sendToDeadLetter, metrics } = makeDeps();- handleEvent- .mockRejectedValueOnce(new Error('transient'))- .mockResolvedValue(undefined);+ let attempts = 0;+ handleEvent.mockImplementation(() => {+ attempts += 1;+ return attempts === 1+ ? Promise.reject(new Error('transient'))+ : Promise.resolve(undefined);+ });const handler = createEventsBatchHandler(deps);const { payload, resolveOffset } = makeBatch([message(0, 'a')]);@@ -247,7 +248,7 @@ describe('handler failure', () => {expect(resolvedOffsets(resolveOffset)).toEqual(['0']);});- it('dead-letters the raw message, counts it, and keeps the partition moving', async () => {+ test('dead-letters the raw message, counts it, and keeps the partition moving', async () => {const poison = 'poison-payload';const { deps, handleEvent, sendToDeadLetter, metrics } = makeDeps();handleEvent.mockImplementation(async (p: { projectId: string }) => {@@ -282,9 +283,9 @@ describe('handler failure', () => {expect(resolvedOffsets(resolveOffset)).toEqual(['0', '1', '2']);});- it('leaves the offset unresolved when the dead-letter produce fails', async () => {+ test('leaves the offset unresolved when the dead-letter produce fails', async () => {const { deps, handleEvent, metrics } = makeDeps({- sendToDeadLetter: vi.fn().mockRejectedValue(new Error('broker down')),+ sendToDeadLetter: mock(() => Promise.reject(new Error('broker down'))),});handleEvent.mockImplementation(async (p: { projectId: string }) => {if (p.projectId === 'project-1') {@@ -307,7 +308,7 @@ describe('handler failure', () => {});describe('unparseable messages', () => {- it('dead-letters instead of dropping, without retrying', async () => {+ test('dead-letters instead of dropping, without retrying', async () => {const { deps, handleEvent, sendToDeadLetter, metrics, sleep } = makeDeps();const handler = createEventsBatchHandler(deps);const { payload, resolveOffset } = makeBatch([@@ -330,7 +331,7 @@ describe('unparseable messages', () => {expect(resolvedOffsets(resolveOffset)).toEqual(['0']);});- it('dead-letters a message with no value at all', async () => {+ test('dead-letters a message with no value at all', async () => {const { deps, sendToDeadLetter, metrics } = makeDeps();const handler = createEventsBatchHandler(deps);const { payload, resolveOffset } = makeBatch([message(0, 'a', '')]);@@ -347,7 +348,7 @@ describe('unparseable messages', () => {});describe('reprocess counter', () => {- it('does not fire for intra-batch out-of-order processing', async () => {+ test('does not fire for intra-batch out-of-order processing', async () => {const { deps, metrics } = makeDeps();const handler = createEventsBatchHandler(deps);const { payload } = makeBatch([@@ -361,7 +362,7 @@ describe('reprocess counter', () => {expect(metrics.reprocessed).not.toHaveBeenCalled();});- it('fires once per redelivered offset in a later batch', async () => {+ test('fires once per redelivered offset in a later batch', async () => {const { deps, metrics } = makeDeps();const handler = createEventsBatchHandler(deps);const messages = [message(0, 'a'), message(1, 'a')];@@ -374,7 +375,7 @@ describe('reprocess counter', () => {expect(metrics.reprocessed).toHaveBeenCalledWith(String(PARTITION));});- it('does not fire after a rebalance clears the watermarks', async () => {+ test('does not fire after a rebalance clears the watermarks', async () => {const { deps, metrics } = makeDeps();const handler = createEventsBatchHandler(deps);const messages = [message(0, 'a'), message(1, 'a')];@@ -386,12 +387,12 @@ describe('reprocess counter', () => {expect(metrics.reprocessed).not.toHaveBeenCalled();});- it('does not fire for a dead-lettered offset re-seen only within the batch', async () => {+ test('does not fire for a dead-lettered offset re-seen only within the batch', async () => {// A dead-lettered offset is resolved, so it must move the watermark just// like a handled one — otherwise the next batch would flag it as a// reprocess.const { deps, handleEvent, metrics } = makeDeps();- handleEvent.mockRejectedValue(new Error('boom'));+ handleEvent.mockImplementation(() => Promise.reject(new Error('boom')));const handler = createEventsBatchHandler(deps);await handler.eachBatch(makeBatch([message(0, 'a')]).payload);diff --git a/packages/core/src/modules/ingest/src/consumer.ts b/packages/core/src/modules/ingest/src/consumer.tsnew file mode 100644index 000000000..7408baa29--- /dev/null+++ b/packages/core/src/modules/ingest/src/consumer.ts@@ -0,0 +1,416 @@+// Ported from apps/worker/src/jobs/events.kafka-consumer.ts (M8-003). The+// delivery contract is ADR-004's and is unchanged line for line: per-key+// serial groups, the ascending contiguous-prefix offset walk, at-least-once,+// bounded in-consumer retry, and a dead-letter topic on exhaustion.+//+// The kafkajs client, the topic/consumer-group names, the DLQ producer and+// the retry bounds are all INJECTED. They still live in @openpanel/queue,+// which imports @openpanel/core for its logger, so core cannot import it back+// without a package cycle (see ingest.service.ts's header). Injection is also+// what keeps those names byte-identical: this file never spells one.++import type {+ Consumer,+ EachBatchPayload,+ IHeaders,+ KafkaMessage,+} from 'kafkajs';+import type { IncomingEventPayload } from './incoming-event';++export interface KafkaConsumerHandle {+ stop: () => Promise<void>;+}++// Heartbeat every N messages within a per-key group. The default kafkajs+// sessionTimeout is 30s — calling heartbeat every 16 messages keeps us+// comfortably under that even for slow handlers.+const HEARTBEAT_EVERY = 16;++const RETRY_BACKOFF_FACTOR = 2;++/** Why a message was dead-lettered. A label value — keep the set small. */+export type DeadLetterReason = 'parse_error' | 'handler_error';++/**+ * What the consumer hands to the dead-letter producer. Structurally identical+ * to @openpanel/queue's `DeadLetterMessage`: the value stays the producer's+ * original bytes, so a DLQ message can be replayed onto the events topic+ * unchanged.+ */+export interface DeadLetterMessage {+ key: Buffer | null;+ value: Buffer | null;+ headers?: IHeaders;+ topic: string;+ partition: number;+ offset: string;+ reason: string;+ error: string;+}++export interface ConsumerLogger {+ info: (obj: object, msg?: string) => void;+ warn: (obj: object, msg?: string) => void;+ error: (obj: object, msg?: string) => void;+}++export interface ConsumerMetrics {+ reprocessed: (partition: string) => void;+ handlerFailed: (partition: string) => void;+ deadLettered: (partition: string, reason: DeadLetterReason) => void;+ deadLetterFailed: (partition: string) => void;+}++export interface EventsBatchHandlerDeps {+ handleEvent: (+ payload: IncomingEventPayload,+ meta: { partition: number; offset: string }+ ) => Promise<unknown>;+ sendToDeadLetter: (message: DeadLetterMessage) => Promise<void>;+ logger: ConsumerLogger;+ metrics: ConsumerMetrics;+ onActivity: () => void;+ topic: string;+ maxAttempts: number;+ initialRetryMs: number;+ maxRetryMs: number;+ /** Injected by the tests; production uses the timer below. */+ sleep?: (ms: number) => Promise<void>;+}++const defaultSleep = (ms: number): Promise<void> =>+ new Promise((resolve) => setTimeout(resolve, ms));++const errorMessage = (err: unknown): string =>+ err instanceof Error ? err.message : String(err);++export interface EventsBatchHandler {+ eachBatch: (payload: EachBatchPayload) => Promise<void>;+ /** Called on GROUP_JOIN: a new assignment invalidates the watermarks. */+ resetWatermarks: () => void;+}++/**+ * The `eachBatch` handler, with its dependencies injected so the offset walk,+ * the per-key serial groups and the failure paths are testable without a+ * broker.+ *+ * Delivery contract (ADR-004): at-least-once. An offset is resolved only once+ * its message has been handled, dead-lettered, or deliberately skipped —+ * never merely because it failed.+ */+export function createEventsBatchHandler(+ deps: EventsBatchHandlerDeps+): EventsBatchHandler {+ const sleep = deps.sleep ?? defaultSleep;++ // Highest Kafka offset we have *resolved* (committed) per `topic-partition`,+ // tracked across batches. Used purely for the reprocess detector below: if we+ // ever see an offset at or below this watermark again, the message is being+ // redelivered (at-least-once duplicate) outside of a rebalance — which is the+ // signature of an offset-handling bug. Cleared on GROUP_JOIN so legitimate+ // post-rebalance redelivery from the last committed offset doesn't trip it.+ const resolvedHWM = new Map<string, number>();++ const deadLetter = async (+ message: KafkaMessage,+ partition: number,+ reason: DeadLetterReason,+ err: unknown+ ): Promise<boolean> => {+ const label = String(partition);+ try {+ await deps.sendToDeadLetter({+ key: message.key,+ value: message.value,+ headers: message.headers,+ topic: deps.topic,+ partition,+ offset: message.offset,+ reason,+ error: errorMessage(err),+ });+ } catch (dlqErr) {+ // The event is not lost: leaving the offset unresolved stops the+ // contiguous-prefix walk here, so the broker redelivers this message.+ deps.metrics.deadLetterFailed(label);+ deps.logger.error(+ { err: dlqErr, partition, offset: message.offset, reason },+ 'kafka dead-letter produce failed — offset left unresolved for redelivery'+ );+ return false;+ }+ deps.metrics.deadLettered(label, reason);+ deps.logger.error(+ { err, partition, offset: message.offset, reason },+ 'kafka message dead-lettered'+ );+ return true;+ };++ const runHandlerWithRetry = async (+ payload: IncomingEventPayload,+ partition: number,+ offset: string+ ): Promise<{ ok: true } | { ok: false; err: unknown }> => {+ const label = String(partition);+ let delayMs = deps.initialRetryMs;+ let lastErr: unknown;++ for (let attempt = 1; attempt <= deps.maxAttempts; attempt++) {+ try {+ await deps.handleEvent(payload, { partition, offset });+ return { ok: true };+ } catch (err) {+ lastErr = err;+ deps.metrics.handlerFailed(label);+ deps.logger.warn(+ {+ err,+ partition,+ offset,+ projectId: payload.projectId,+ attempt,+ maxAttempts: deps.maxAttempts,+ },+ 'kafka incomingEvent handler failed'+ );+ if (attempt < deps.maxAttempts) {+ await sleep(delayMs);+ delayMs = Math.min(delayMs * RETRY_BACKOFF_FACTOR, deps.maxRetryMs);+ }+ }+ }++ return { ok: false, err: lastErr };+ };++ /** @returns true when the offset may be resolved. */+ const processMessage = async (+ message: KafkaMessage,+ partition: number+ ): Promise<boolean> => {+ let payload: IncomingEventPayload;+ try {+ if (!message.value) {+ throw new Error('kafka message has no value');+ }+ payload = JSON.parse(message.value.toString()) as IncomingEventPayload;+ } catch (err) {+ deps.logger.error(+ { err, partition, offset: message.offset },+ 'kafka message parse failed'+ );+ // Deterministic: retrying identical bytes cannot succeed.+ return deadLetter(message, partition, 'parse_error', err);+ }++ const result = await runHandlerWithRetry(+ payload,+ partition,+ message.offset+ );+ if (result.ok) {+ return true;+ }+ return deadLetter(message, partition, 'handler_error', result.err);+ };++ const eachBatch = async ({+ batch,+ resolveOffset,+ heartbeat,+ isRunning,+ isStale,+ }: EachBatchPayload): Promise<void> => {+ if (batch.messages.length === 0) {+ return;+ }++ const pk = `${batch.topic}-${batch.partition}`;+ // Watermark from *previous* batches. Anything at or below this that we see+ // now is a redelivery. Captured before this batch so intra-batch+ // out-of-order processing (normal, see below) is never counted.+ const priorHWM = resolvedHWM.get(pk) ?? -1;++ // Group by partition key (= deviceId or `${projectId}:${profileId}`).+ // Same-key messages stay serial so sessionBuffer/session-end-job state+ // can't race; different keys run in parallel via Promise.all.+ // Keyless messages get their own singleton group.+ const groups = new Map<string, KafkaMessage[]>();+ for (const m of batch.messages) {+ const key = m.key ? m.key.toString() : `__no_key__:${m.offset}`;+ const arr = groups.get(key);+ if (arr) {+ arr.push(m);+ } else {+ groups.set(key, [m]);+ }+ }++ // Offsets that finished processing this batch. We resolve them AFTER all+ // groups complete, in strict ascending order (see the loop below).+ //+ // Why: KafkaJS `resolveOffset` is last-write-wins and the next fetch+ // starts from the last resolved offset (offsetManager.nextOffset). If we+ // resolved inside the concurrent per-key loop, a lower offset resolving+ // after a higher one would move the fetch position BACKWARDS and+ // re-deliver everything in between — the root cause of the duplicate+ // events. Resolving the contiguous ascending prefix at the end avoids+ // both duplicates (never regress) and loss (stop at the first gap).+ const processed = new Set<string>();+ let processedCount = 0;++ await Promise.all(+ [...groups.values()].map(async (msgs) => {+ for (const m of msgs) {+ if (!isRunning() || isStale()) {+ return;+ }++ // Reprocess detector: only fires for offsets already resolved in a+ // PRIOR batch (redelivery). Intra-batch out-of-order processing+ // across key-groups is expected and is not flagged.+ if (Number(m.offset) <= priorHWM) {+ deps.metrics.reprocessed(String(batch.partition));+ deps.logger.warn(+ {+ partition: batch.partition,+ offset: m.offset,+ resolvedHighWaterMark: priorHWM,+ },+ 'kafka offset REPROCESSED — at-least-once duplicate (outside rebalance)'+ );+ }++ const resolvable = await processMessage(m, batch.partition);+ if (!resolvable) {+ // Stop this key-group: continuing past an unresolvable offset only+ // adds duplicates, since the whole tail is redelivered anyway.+ return;+ }++ processed.add(m.offset);+ processedCount += 1;+ if (processedCount % HEARTBEAT_EVERY === 0) {+ await heartbeat();+ }+ }+ })+ );++ // Resolve in strict ascending offset order, stopping at the first offset+ // that did not finish (e.g. an isStale/isRunning early-return mid-batch).+ // batch.messages is already ordered by offset.+ let newHWM = priorHWM;+ for (const m of batch.messages) {+ if (!processed.has(m.offset)) {+ break;+ }+ resolveOffset(m.offset);+ newHWM = Math.max(newHWM, Number(m.offset));+ }+ resolvedHWM.set(pk, newHWM);++ await heartbeat();+ deps.onActivity();+ };++ return { eachBatch, resetWatermarks: () => resolvedHWM.clear() };+}++/** The broker-facing half, supplied by whoever owns the kafkajs client. */+export interface EventsConsumerDeps {+ /** Already configured with the group id, timeouts and fetch bounds. */+ createConsumer: () => Consumer;+ /** Lifecycle / rebalance visibility. */+ logger: ConsumerLogger;+ /** The "consumer running" line, on the kafka-scoped logger. */+ kafkaLogger: ConsumerLogger;+ topic: string;+ partitionsConsumedConcurrently: number;+ batch: Omit<EventsBatchHandlerDeps, 'topic'>;+}++export async function startKafkaEventsConsumer(+ deps: EventsConsumerDeps+): Promise<KafkaConsumerHandle> {+ const { logger } = deps;+ const consumer = deps.createConsumer();+ const handler = createEventsBatchHandler({+ ...deps.batch,+ topic: deps.topic,+ });+ await consumer.connect();+ await consumer.subscribe({+ topic: deps.topic,+ fromBeginning: false,+ });++ consumer.on(consumer.events.HEARTBEAT, deps.batch.onActivity);++ // ---- Lifecycle / rebalance ("re-election") visibility ----+ // We had no logging for partition reassignment before; without it a rebalance+ // storm (a common source of at-least-once duplicates) is invisible.+ consumer.on(consumer.events.GROUP_JOIN, ({ payload }) => {+ // A new assignment means partitions may have moved between members; reset+ // the reprocess watermarks so legitimate resume-from-committed-offset after+ // a rebalance is not flagged as a duplicate.+ handler.resetWatermarks();+ logger.info(+ {+ memberId: payload.memberId,+ groupId: payload.groupId,+ isLeader: payload.isLeader,+ memberAssignment: payload.memberAssignment,+ duration: payload.duration,+ },+ 'kafka consumer joined group (rebalance complete)'+ );+ });+ consumer.on(consumer.events.REBALANCING, ({ payload }) => {+ logger.warn(+ { memberId: payload.memberId, groupId: payload.groupId },+ 'kafka consumer rebalancing'+ );+ });+ consumer.on(consumer.events.CRASH, ({ payload }) => {+ logger.error(+ {+ err: payload.error,+ groupId: payload.groupId,+ restart: payload.restart,+ },+ 'kafka consumer crashed'+ );+ });+ consumer.on(consumer.events.DISCONNECT, () => {+ logger.warn({}, 'kafka consumer disconnected');+ });+ consumer.on(consumer.events.REQUEST_TIMEOUT, ({ payload }) => {+ logger.warn(+ { broker: payload.broker, clientId: payload.clientId },+ 'kafka consumer request timeout'+ );+ });++ await consumer.run({+ partitionsConsumedConcurrently: deps.partitionsConsumedConcurrently,+ eachBatchAutoResolve: false,+ eachBatch: handler.eachBatch,+ });++ deps.kafkaLogger.info(+ {+ topic: deps.topic,+ partitionsConsumedConcurrently: deps.partitionsConsumedConcurrently,+ },+ 'kafka events consumer running'+ );++ return {+ stop: async () => {+ await consumer.disconnect();+ },+ };+}diff --git a/packages/core/src/modules/ingest/src/incoming-event-handler.test.ts b/packages/core/src/modules/ingest/src/incoming-event-handler.test.tsnew file mode 100644index 000000000..3a99cf1c9--- /dev/null+++ b/packages/core/src/modules/ingest/src/incoming-event-handler.test.ts@@ -0,0 +1,456 @@+// Ported from apps/worker/src/jobs/events.incoming-events.test.ts (M8-003)+// with every assertion unchanged. What changed is the seam: V1 reached the+// session buffer, `createEvent` and the sessions queue through+// `vi.mock('@openpanel/db')` / `vi.mock('@openpanel/queue')`; here they are+// recording doubles handed in as `IncomingEventDeps`, so the assertions are+// on calls the code under test actually made and no `mock.module` is needed.+//+// The session_end enqueue is asserted through the real `sessionEndJobPayload`+// / `sessionEndEnqueueOptions` — the same functions V1's producer+// (`enqueueSessionEndV2`) calls — so the jobId and the `{payload, snapshot}`+// mapping are still pinned here, not re-implemented by the test.++import { beforeEach, describe, expect, mock, test } from 'bun:test';+import { formatClickhouseDate } from '../../event/src/dates';+import type { IClickhouseSession } from '../../session/session.service';+import {+ sessionEndEnqueueOptions,+ sessionEndJobPayload,+} from '../../session/src/session-end';+import type { IncomingEventPayload } from './incoming-event';+import {+ type IncomingEventDeps,+ incomingEvent,+} from './incoming-event-handler';++const projectId = 'test-project';+const deviceId = 'device-123';+const newSessionId = 'a1b2c3d4-e5f6-4789-a012-345678901234';+// What the producer (/track) would have minted for one request.+const producedEventId = '11111111-2222-4333-8444-555555555555';+const geo = {+ country: 'US',+ city: 'New York',+ region: 'NY',+ longitude: 0,+ latitude: 0,+};++const uaInfo: IncomingEventPayload['uaInfo'] = {+ isServer: false,+ device: 'desktop',+ os: 'Windows',+ osVersion: '10',+ browser: 'Chrome',+ browserVersion: '91.0.4472.124',+ brand: '',+ model: '',+};++const uaInfoServer: IncomingEventPayload['uaInfo'] = {+ isServer: true,+ device: 'server',+ os: '',+ osVersion: '',+ browser: '',+ browserVersion: '',+ brand: '',+ model: '',+};++function makeSession(+ overrides: Partial<IClickhouseSession> = {}+): IClickhouseSession {+ const now = new Date();+ return {+ id: 'session-existing',+ project_id: projectId,+ device_id: deviceId,+ profile_id: '',+ event_count: 1,+ screen_view_count: 0,+ entry_path: '/',+ entry_origin: 'https://example.com',+ exit_path: '/',+ exit_origin: 'https://example.com',+ created_at: formatClickhouseDate(now),+ ended_at: formatClickhouseDate(now),+ os: 'Windows',+ os_version: '10',+ browser: 'Chrome',+ browser_version: '91.0.4472.124',+ device: 'desktop',+ brand: '',+ model: '',+ country: 'US',+ region: 'NY',+ city: 'New York',+ longitude: 0,+ latitude: 0,+ duration: 0,+ referrer: '',+ referrer_name: '',+ referrer_type: '',+ is_bounce: true,+ utm_term: '',+ utm_source: '',+ utm_campaign: '',+ utm_content: '',+ utm_medium: '',+ revenue: 0,+ sign: 1,+ version: 1,+ groups: [],+ ...overrides,+ } satisfies IClickhouseSession;+}++function buildJobData(+ overrides: Partial<IncomingEventPayload> = {}+): IncomingEventPayload {+ return {+ geo,+ event: {+ name: 'test_event',+ timestamp: new Date().toISOString(),+ isTimestampFromThePast: false,+ properties: { __path: 'https://example.com/test' },+ },+ uaInfo,+ headers: {+ 'request-id': '123',+ 'user-agent': 'Mozilla/5.0',+ 'openpanel-sdk-name': 'web',+ 'openpanel-sdk-version': '1.0.0',+ },+ projectId,+ deviceId,+ sessionId: newSessionId,+ ...overrides,+ };+}++const noopLogger = (() => {+ const noop = () => undefined;+ const logger = {+ fatal: noop,+ error: noop,+ warn: noop,+ info: noop,+ debug: noop,+ trace: noop,+ child: () => logger,+ };+ return logger;+})();++function makeDeps() {+ const createEvent = mock<IncomingEventDeps['createEvent']>(+ async (event) => event+ );+ const getExistingSession = mock<+ IncomingEventDeps['sessions']['getExistingSession']+ >(async () => null);+ const ingest = mock<IncomingEventDeps['sessions']['ingest']>(+ async () => null+ );+ const enqueueSessionEnd = mock<IncomingEventDeps['enqueueSessionEnd']>(+ async () => undefined+ );+ const deps: IncomingEventDeps = {+ logger: noopLogger,+ sessions: { getExistingSession, ingest },+ createEvent,+ checkNotificationRulesForEvent: mock(async () => true),+ projects: {+ getCached: mock(async () => ({ firstEventAt: new Date(), filters: [] })),+ markFirstEvent: mock(async () => undefined),+ },+ enqueueSessionEnd,+ metrics: {+ sessionStarted: mock(() => undefined),+ sessionEndEnqueued: mock(() => undefined),+ },+ };+ return { deps, createEvent, getExistingSession, ingest, enqueueSessionEnd };+}++/** Every `createEvent` call's first argument, in order. */+const createdEvents = (createEvent: {+ mock: { calls: unknown[][] };+}): Record<string, unknown>[] =>+ createEvent.mock.calls.map(([arg]) => arg as Record<string, unknown>);++let ctx: ReturnType<typeof makeDeps>;++beforeEach(() => {+ ctx = makeDeps();+});++describe('incomingEvent', () => {+ test('emits session_start when ingest returns kind="new"', async () => {+ const { deps, createEvent, ingest, enqueueSessionEnd } = ctx;+ ingest.mockResolvedValueOnce({+ kind: 'new',+ current: makeSession({ id: newSessionId }),+ });++ await incomingEvent(buildJobData(), deps);++ expect(enqueueSessionEnd).not.toHaveBeenCalled();+ const calls = createdEvents(createEvent);+ expect(calls).toHaveLength(2);+ expect(calls[0]!.name).toBe('session_start');+ expect(calls[1]!.name).toBe('test_event');+ });++ test('skips session_start when ingest returns kind="extend"', async () => {+ const { deps, createEvent, ingest, enqueueSessionEnd } = ctx;+ ingest.mockResolvedValueOnce({ kind: 'extend', current: makeSession() });++ await incomingEvent(buildJobData(), deps);++ expect(enqueueSessionEnd).not.toHaveBeenCalled();+ const sessionStartCalls = createdEvents(createEvent).filter(+ (arg) => arg.name === 'session_start'+ );+ expect(sessionStartCalls).toHaveLength(0);+ });++ test('closes old session and emits session_start when ingest returns kind="boundary"', async () => {+ const { deps, createEvent, ingest, enqueueSessionEnd } = ctx;+ const closed = makeSession({ id: 'old-session-id' });+ const current = makeSession({ id: newSessionId });+ ingest.mockResolvedValueOnce({ kind: 'boundary', current, closed });++ await incomingEvent(buildJobData(), deps);++ expect(enqueueSessionEnd).toHaveBeenCalledTimes(1);+ const input = enqueueSessionEnd.mock.calls[0]![0];+ // The wire payload V1's producer builds from this input, through the same+ // core functions it calls.+ const { event, snapshot } = sessionEndJobPayload(input);+ expect(event.sessionId).toBe('old-session-id');+ expect(snapshot.id).toBe('old-session-id');+ expect(sessionEndEnqueueOptions(input.closedSession.id).jobId).toBe(+ 'sessionEnd:v2:old-session-id'+ );++ const calls = createdEvents(createEvent);+ expect(calls.filter((a) => a.name === 'session_start')).toHaveLength(1);+ expect(calls.filter((a) => a.name === 'test_event')).toHaveLength(1);+ });++ test('inherits referrer from current session on the actual event', async () => {+ const { deps, createEvent, ingest } = ctx;+ ingest.mockResolvedValueOnce({+ kind: 'extend',+ current: makeSession({+ referrer: 'https://google.com',+ referrer_name: 'Google',+ referrer_type: 'search',+ }),+ });++ await incomingEvent(buildJobData(), deps);++ const testEventCall = createdEvents(createEvent).find(+ (a) => a.name === 'test_event'+ );+ expect(testEventCall!.referrer).toBe('https://google.com');+ expect(testEventCall!.referrerName).toBe('Google');+ expect(testEventCall!.referrerType).toBe('search');+ });++ test('handles server events with existing profile session', async () => {+ const { deps, createEvent, ingest, getExistingSession, enqueueSessionEnd } =+ ctx;+ const timestamp = new Date();+ const jobData = buildJobData({+ event: {+ name: 'server_event',+ timestamp: timestamp.toISOString(),+ properties: { custom_property: 'test_value' },+ profileId: 'profile-123',+ isTimestampFromThePast: false,+ },+ uaInfo: uaInfoServer,+ deviceId: '',+ sessionId: '',+ headers: {+ 'user-agent': 'OpenPanel Server/1.0',+ 'openpanel-sdk-name': 'server',+ 'openpanel-sdk-version': '1.0.0',+ 'request-id': '123',+ },+ });++ getExistingSession.mockResolvedValueOnce(+ makeSession({+ id: 'last-session-456',+ device_id: 'last-device-123',+ profile_id: 'profile-123',+ os: 'iOS',+ os_version: '15.0',+ browser: 'Safari',+ browser_version: '15.0',+ device: 'mobile',+ brand: 'Apple',+ model: 'iPhone',+ country: 'CA',+ region: 'ON',+ city: 'Toronto',+ entry_path: '/last-path',+ entry_origin: 'https://example.com',+ exit_path: '/last-path',+ exit_origin: 'https://example.com',+ referrer: 'https://google.com',+ referrer_name: 'Google',+ referrer_type: 'search',+ is_bounce: false,+ event_count: 0,+ screen_view_count: 0,+ })+ );++ await incomingEvent(jobData, deps);++ expect(ingest).not.toHaveBeenCalled();+ expect(enqueueSessionEnd).not.toHaveBeenCalled();+ expect(createdEvents(createEvent)[0]!).toMatchObject({+ name: 'server_event',+ deviceId: 'last-device-123',+ sessionId: 'last-session-456',+ profileId: 'profile-123',+ city: 'Toronto',+ country: 'CA',+ referrer: 'https://google.com',+ });+ });++ test('handles server events without any active session', async () => {+ const { deps, createEvent, ingest, getExistingSession, enqueueSessionEnd } =+ ctx;+ getExistingSession.mockResolvedValueOnce(null);++ await incomingEvent(+ buildJobData({+ event: {+ name: 'server_event',+ timestamp: new Date().toISOString(),+ properties: { custom_property: 'test_value' },+ profileId: 'profile-123',+ isTimestampFromThePast: false,+ },+ uaInfo: uaInfoServer,+ deviceId: '',+ sessionId: '',+ headers: {+ 'user-agent': 'OpenPanel Server/1.0',+ 'openpanel-sdk-name': 'server',+ 'openpanel-sdk-version': '1.0.0',+ 'request-id': '123',+ },+ }),+ deps+ );++ expect(ingest).not.toHaveBeenCalled();+ expect(enqueueSessionEnd).not.toHaveBeenCalled();+ expect(createdEvents(createEvent)[0]!).toMatchObject({+ name: 'server_event',+ deviceId: '',+ sessionId: '',+ profileId: 'profile-123',+ });+ });++ test('re-creates the same row id when Kafka redelivers the message', async () => {+ const { deps, createEvent, ingest } = ctx;+ const session = makeSession({ id: newSessionId });+ ingest+ .mockResolvedValueOnce({ kind: 'new', current: session })+ .mockResolvedValueOnce({ kind: 'extend', current: session });++ // Same message, delivered twice: identical payload, identical offset.+ const jobData = buildJobData({ id: producedEventId });+ const delivery = { partition: 3, offset: '42' };+ await incomingEvent(jobData, deps, delivery);+ await incomingEvent(jobData, deps, delivery);++ const eventCalls = createdEvents(createEvent).filter(+ (a) => a.name === 'test_event'+ );+ expect(eventCalls).toHaveLength(2);+ expect(eventCalls[0]!.id).toBe(producedEventId);+ expect(eventCalls[1]!.id).toBe(producedEventId);++ // The derived session_start row must NOT reuse the event's id.+ const sessionStartCalls = createdEvents(createEvent).filter(+ (a) => a.name === 'session_start'+ );+ expect(sessionStartCalls).toHaveLength(1);+ expect(sessionStartCalls[0]!.id).toBeUndefined();+ });++ test('carries the producer-minted id on server-side events', async () => {+ const { deps, createEvent, getExistingSession } = ctx;+ getExistingSession.mockResolvedValueOnce(null);++ await incomingEvent(+ buildJobData({+ id: producedEventId,+ event: {+ name: 'server_event',+ timestamp: new Date().toISOString(),+ isTimestampFromThePast: false,+ profileId: 'profile-123',+ },+ uaInfo: uaInfoServer,+ deviceId: '',+ sessionId: '',+ }),+ deps+ );++ expect(createdEvents(createEvent)[0]!.id).toBe(producedEventId);+ });++ test('leaves the id undefined for a payload produced without one', async () => {+ const { deps, createEvent, ingest } = ctx;+ ingest.mockResolvedValueOnce({ kind: 'extend', current: makeSession() });++ await incomingEvent(buildJobData(), deps);++ expect(createdEvents(createEvent)[0]!.id).toBeUndefined();+ });++ test('emits session_start only once across 3 rapid events (new → extend → extend)', async () => {+ const { deps, createEvent, ingest, enqueueSessionEnd } = ctx;+ const session = makeSession({ id: newSessionId });+ ingest+ .mockResolvedValueOnce({ kind: 'new', current: session })+ .mockResolvedValueOnce({ kind: 'extend', current: session })+ .mockResolvedValueOnce({ kind: 'extend', current: session });++ for (const name of ['e1', 'e2', 'e3']) {+ await incomingEvent(+ buildJobData({+ event: {+ name,+ timestamp: new Date().toISOString(),+ isTimestampFromThePast: false,+ properties: { __path: 'https://example.com/test' },+ },+ }),+ deps+ );+ }++ const sessionStartCalls = createdEvents(createEvent).filter(+ (a) => a.name === 'session_start'+ );+ expect(sessionStartCalls).toHaveLength(1);+ expect(enqueueSessionEnd).not.toHaveBeenCalled();+ });+});diff --git a/packages/core/src/modules/ingest/src/incoming-event-handler.ts b/packages/core/src/modules/ingest/src/incoming-event-handler.tsnew file mode 100644index 000000000..d841c0992--- /dev/null+++ b/packages/core/src/modules/ingest/src/incoming-event-handler.ts@@ -0,0 +1,427 @@+// Ported from apps/worker/src/jobs/events.incoming-event.ts (M8-003): the+// Kafka consumer's per-message handler — the step that turns one topic+// payload into session lifecycle decisions and ClickHouse rows. V1's job file+// stays (DELEGATE PATTERN) and calls this until apps/worker dies (P9).+//+// Everything that touches Redis, Postgres, ClickHouse or a queue is INJECTED+// (`IncomingEventDeps`), the way session-end.ts does it: the tests then drive+// the real code with recording doubles instead of `mock.module`, and V1 keeps+// handing in its own buffers, counters and `sessionsQueue` producer so its+// behaviour and its `/metrics` body are unchanged.+//+// createEvent is a static import for the reason session.service.ts's header+// gives: a dynamic edge here closes a dynamic-import cycle that panics+// rolldown when apps/worker bundles the workspace.++import { getTime, isSameDomain, parsePath } from '@openpanel/common';+import type { IProjectFilters } from '@openpanel/validation';+import { anyPass, isEmpty, isNil, mergeDeepRight, omit, reject } from 'ramda';+import type { SessionIngestResult } from '../../../buffers/session-buffer';+import type { Logger } from '../../../logger';+import {+ getReferrerWithQuery,+ parseReferrer,+} from '../../../shared/parse-referrer';+import {+ createEvent,+ type IServiceCreateEventPayload,+ type IServiceCreateEventPayloadWithId,+ type IServiceEvent,+} from '../../event/event.service';+import { matchEvent } from '../../notification/notification.service';+// Static, not lazy: a dynamic edge into project.service becomes its own+// rolldown chunk, which stops `hashPassword` (and with it the @node-rs/argon2+// native addon, external in apps/worker's bundle) from being shaken out of+// the worker build.+import { getProjectByIdCached } from '../../project/project.service';+import type { IClickhouseSession } from '../../session/session.service';+import { sessionEndsEnqueued } from '../../session/src/session.metrics';+import type { EnqueueSessionEndInput } from '../../session/src/session-end';+import type { IncomingEventPayload } from './incoming-event';+import { sessionsStarted } from './ingest.metrics';++const GLOBAL_PROPERTIES = ['__path', '__referrer', '__timestamp', '__revenue'];++/** How far before the event `session_start` is stamped, so it sorts first. */+const SESSION_START_OFFSET_MS = 100;++// Strip empty/nullish from B, then deep-merge over A.+const merge = <A, B>(a: Partial<A>, b: Partial<B>): A & B =>+ mergeDeepRight(a, reject(anyPass([isEmpty, isNil]))(b)) as A & B;++/** Kafka delivery coordinates, when the event came through the consumer. */+export interface IncomingEventDelivery {+ partition: number;+ offset: string;+}++/** The project fields the filter and first-event checks read. */+interface IncomingEventProject {+ firstEventAt?: Date | null;+ filters?: IProjectFilters[] | null;+}++/** The session-buffer subset this handler drives. */+export interface IncomingEventSessions {+ getExistingSession(options: {+ projectId: string;+ profileId: string;+ }): Promise<IClickhouseSession | null>;+ ingest(+ payload: IServiceCreateEventPayload+ ): Promise<SessionIngestResult | null>;+}++/**+ * Injected so V1's worker keeps incrementing its own registry's counters+ * while core's registry owns the V2 ones (ingest.metrics.ts).+ */+export interface IncomingEventMetrics {+ sessionStarted(kind: 'new' | 'boundary'): void;+ sessionEndEnqueued(source: 'boundary'): void;+}++export interface IncomingEventDeps {+ /** Base logger; the handler child()s it per message. */+ logger: Logger;+ sessions: IncomingEventSessions;+ createEvent(+ payload: IServiceCreateEventPayloadWithId+ ): Promise<{ document: unknown } | unknown>;+ checkNotificationRulesForEvent(+ payload: IServiceCreateEventPayload+ ): Promise<unknown>;+ projects: {+ getCached(projectId: string): Promise<IncomingEventProject | null>;+ /** Conditional update + cache clear; idempotent across workers. */+ markFirstEvent(projectId: string): Promise<unknown>;+ };+ enqueueSessionEnd(input: EnqueueSessionEndInput): Promise<unknown>;+ metrics: IncomingEventMetrics;+}++/**+ * Binds `IncomingEventDeps` to the real implementations. db/ch access stays+ * lazy — see event.service.ts's header.+ */+export async function loadIncomingEventDeps(+ logger: Logger,+ enqueueSessionEnd: (input: EnqueueSessionEndInput) => Promise<unknown>,+ overrides: Partial<IncomingEventDeps> = {}+): Promise<IncomingEventDeps> {+ const [{ sessionBuffer }, { db }, notifications] = await Promise.all([+ import('@openpanel/db/src/buffers'),+ import('@openpanel/db/src/prisma-client'),+ import('@openpanel/db/src/services/notification.service'),+ ]);+ return {+ logger,+ sessions: sessionBuffer,+ createEvent,+ checkNotificationRulesForEvent:+ notifications.checkNotificationRulesForEvent,+ projects: {+ getCached: getProjectByIdCached,+ markFirstEvent: async (projectId) => {+ await db.project.updateMany({+ where: { id: projectId, firstEventAt: null },+ data: { firstEventAt: new Date() },+ });+ await getProjectByIdCached.clear(projectId);+ },+ },+ enqueueSessionEnd,+ metrics: {+ sessionStarted: (kind) => sessionsStarted.inc({ kind }),+ sessionEndEnqueued: (source) => sessionEndsEnqueued.inc({ source }),+ },+ ...overrides,+ };+}++async function isEventExcludedByProjectFilter(+ payload: IServiceCreateEventPayload,+ projectId: string,+ deps: IncomingEventDeps+): Promise<boolean> {+ const project = await deps.projects.getCached(projectId);+ const eventExcludeFilters = (project?.filters ?? []).filter(+ (f) => f.type === 'event'+ );+ if (eventExcludeFilters.length === 0) {+ return false;+ }+ return eventExcludeFilters.some((filter) => matchEvent(payload, filter));+}++/**+ * Records the project's first-ever event timestamp exactly once. The cached+ * project read makes this a no-op on every event after the first; the+ * conditional update keeps concurrent workers idempotent.+ */+async function markFirstEvent(+ projectId: string,+ logger: Logger,+ deps: IncomingEventDeps+) {+ const project = await deps.projects.getCached(projectId);+ if (!project || project.firstEventAt) {+ return;+ }+ await deps.projects.markFirstEvent(projectId);+ logger.info({ projectId }, 'Project received its first event');+}++async function createEventAndNotify(+ payload: IServiceCreateEventPayloadWithId,+ logger: Logger,+ projectId: string,+ deps: IncomingEventDeps+) {+ const isExcluded = await isEventExcludedByProjectFilter(+ payload,+ projectId,+ deps+ );+ if (isExcluded) {+ logger.info(+ { event: payload.name, projectId },+ 'Event excluded by project filter'+ );+ return null;+ }++ logger.info({ event: payload }, 'Creating event');+ const [event] = await Promise.all([+ deps.createEvent(payload),+ deps.checkNotificationRulesForEvent(payload).catch(() => null),+ ]);+ // Only after the event is accepted — recording the first event before a+ // failed createEvent would leave the activation checklist claiming data+ // arrived that was never persisted.+ await markFirstEvent(projectId, logger, deps).catch(() => null);+ return event;+}++const parseRevenue = (revenue: unknown): number | undefined => {+ if (!revenue) {+ return undefined;+ }+ if (typeof revenue === 'number') {+ return revenue;+ }+ if (typeof revenue === 'string') {+ const parsed = Number.parseFloat(revenue);+ return Number.isNaN(parsed) ? undefined : parsed;+ }+ return undefined;+};++export async function incomingEvent(+ jobPayload: IncomingEventPayload,+ deps: IncomingEventDeps,+ // Logged so a duplicate row in ClickHouse can be traced back to the exact+ // partition/offset that produced it.+ meta?: IncomingEventDelivery+) {+ const {+ geo,+ event: body,+ headers,+ projectId,+ deviceId,+ sessionId,+ uaInfo,+ // Producer-minted; a redelivered message re-creates the same row id.+ // Absent on messages produced before the field existed — createEvent then+ // falls back to a fresh uuid.+ id: eventId,+ } = jobPayload;+ const properties: Record<string, unknown> = body.properties ?? {};+ const reqId = headers['request-id'] ?? 'unknown';+ const logger = deps.logger.child({+ reqId,+ ...(meta+ ? { kafkaPartition: meta.partition, kafkaOffset: meta.offset }+ : {}),+ });+ const getProperty = (name: string): string | undefined => {+ // replace thing is just for older sdks when we didn't have `__`+ // remove when kiddokitchen app (24.09.02) is not used anymore+ return (+ ((properties[name] || properties[name.replace('__', '')]) as+ | string+ | null+ | undefined) ?? undefined+ );+ };++ const profileId = body.profileId ? String(body.profileId) : '';+ const createdAt = new Date(body.timestamp);+ const isTimestampFromThePast = body.isTimestampFromThePast;+ const url = getProperty('__path');+ const { path, hash, query, origin } = parsePath(url);+ const referrer = isSameDomain(getProperty('__referrer'), url)+ ? null+ : parseReferrer(getProperty('__referrer'));+ const utmReferrer = getReferrerWithQuery(query);+ // Typed as a plain record so `omit`'s key set is `string`, not the two+ // literal keys the spread would otherwise infer.+ const eventProperties: Record<string, unknown> = {+ ...properties,+ __hash: hash,+ __query: query,+ };+ const sdkName = headers['openpanel-sdk-name'];+ const sdkVersion = headers['openpanel-sdk-version'];++ const baseEvent: IServiceCreateEventPayload = {+ name: body.name,+ profileId,+ projectId,+ deviceId,+ sessionId,+ properties: omit(GLOBAL_PROPERTIES, eventProperties),+ groups: body.groups ?? [],+ createdAt,+ duration: 0,+ sdkName,+ sdkVersion,+ city: geo.city,+ country: geo.country,+ region: geo.region,+ longitude: geo.longitude,+ latitude: geo.latitude,+ path,+ origin,+ referrer: referrer?.url || '',+ referrerName: utmReferrer?.name || referrer?.name || referrer?.url,+ referrerType: utmReferrer?.type || referrer?.type || '',+ os: uaInfo.os,+ osVersion: uaInfo.osVersion,+ browser: uaInfo.browser,+ browserVersion: uaInfo.browserVersion,+ device: uaInfo.device,+ brand: uaInfo.brand,+ model: uaInfo.model,+ revenue:+ body.name === 'revenue' && '__revenue' in properties+ ? parseRevenue(properties.__revenue)+ : undefined,+ };++ // Server-side and "timestamp from the past" events ride alongside an+ // existing client session if there is one — they don't open / close+ // sessions of their own.+ if (uaInfo.isServer || isTimestampFromThePast) {+ const session =+ profileId && !isTimestampFromThePast+ ? await deps.sessions.getExistingSession({ profileId, projectId })+ : null;++ const payload = {+ ...baseEvent,+ deviceId: session?.device_id ?? '',+ sessionId: session?.id ?? '',+ referrer: session?.referrer ?? undefined,+ referrerName: session?.referrer_name ?? undefined,+ referrerType: session?.referrer_type ?? undefined,+ path: session?.exit_path ?? baseEvent.path,+ origin: session?.exit_origin ?? baseEvent.origin,+ os: session?.os ?? baseEvent.os,+ osVersion: session?.os_version ?? baseEvent.osVersion,+ browserVersion: session?.browser_version ?? baseEvent.browserVersion,+ browser: session?.browser ?? baseEvent.browser,+ device: session?.device ?? baseEvent.device,+ brand: session?.brand ?? baseEvent.brand,+ model: session?.model ?? baseEvent.model,+ city: session?.city ?? baseEvent.city,+ country: session?.country ?? baseEvent.country,+ region: session?.region ?? baseEvent.region,+ longitude: session?.longitude ?? baseEvent.longitude,+ latitude: session?.latitude ?? baseEvent.latitude,+ };++ return createEventAndNotify(+ { ...(payload as IServiceEvent), id: eventId },+ logger,+ projectId,+ deps+ );+ }++ if (await isEventExcludedByProjectFilter(baseEvent, projectId, deps)) {+ logger.info(+ { event: baseEvent.name, projectId },+ 'Skipping session_start and event (excluded by project filter)'+ );+ return null;+ }++ // The single source of truth for session lifecycle. Reads the current+ // session, decides extend/new/boundary, writes back. The returned+ // `current` is the canonical session — use its referrer fields for+ // inheritance, just like the previous behavior.+ const session = await deps.sessions.ingest(baseEvent);++ if (session?.kind === 'boundary') {+ // Close the old session in a separate job (one Redis-buffered insert+ // for the session_end event + notification rule check). Idempotent via+ // BullMQ jobId dedup.+ await deps+ .enqueueSessionEnd({+ payload: baseEvent,+ closedSession: session.closed,+ })+ .then(() => deps.metrics.sessionEndEnqueued('boundary'))+ .catch((error) => {+ logger.error(+ { err: error, deviceId, sessionId: session.closed.id },+ 'Error enqueueing session_end on boundary'+ );+ });+ }++ if (session?.kind === 'new' || session?.kind === 'boundary') {+ deps.metrics.sessionStarted(session.kind);+ // No `id` here: session_start is a second, derived row and must not share+ // the producer-minted id of the event that triggered it.+ await createEventAndNotify(+ {+ ...baseEvent,+ name: 'session_start',+ createdAt: new Date(+ getTime(baseEvent.createdAt) - SESSION_START_OFFSET_MS+ ),+ },+ logger,+ projectId,+ deps+ ).catch((error) => {+ logger.error(+ { err: error, event: baseEvent },+ 'Error creating session start event'+ );+ throw error;+ });+ }++ // Inherit referrer fields from the canonical session for the actual event.+ // For 'extend' this preserves the original session's referrer across+ // mid-session events; for 'new' / 'boundary' it's the event's own referrer+ // (which `ingest` just stored on the fresh session).+ const finalPayload: IServiceCreateEventPayload = session+ ? merge(baseEvent, {+ referrer: session.current.referrer,+ referrerName: session.current.referrer_name,+ referrerType: session.current.referrer_type,+ } as Partial<IServiceCreateEventPayload>)+ : baseEvent;++ return createEventAndNotify(+ { ...finalPayload, id: eventId },+ logger,+ projectId,+ deps+ );+}diff --git a/packages/core/src/modules/ingest/src/ingest.metrics.ts b/packages/core/src/modules/ingest/src/ingest.metrics.tsnew file mode 100644index 000000000..2c4762133--- /dev/null+++ b/packages/core/src/modules/ingest/src/ingest.metrics.ts@@ -0,0 +1,54 @@+// The ingest-path collectors, moved from apps/worker/src/metrics.ts (M8-003)+// onto core's one registry. Names, labels and help strings are V1's; the+// worker-side copies die with apps/worker (P9). Until then V1's consumer and+// incoming-event delegate inject their own counters (both take `metrics` as a+// dependency), so V1's `/metrics` body is unchanged and these are the V2+// bindings.++import client from 'prom-client';+import { registry } from '../../../metrics';++// Kafka event messages reprocessed (same offset redelivered outside a+// rebalance). Should stay ~0. A sustained non-zero rate means the consumer is+// re-delivering messages it already handled — an offset-handling/duplicate bug.+export const kafkaReprocessedTotal = new client.Counter({+ name: 'kafka_events_reprocessed_total',+ help: 'Kafka event messages reprocessed (offset redelivered outside a rebalance)',+ labelNames: ['partition'],+ registers: [registry],+});++// Handler exceptions on the Kafka ingest path, counted per attempt — retries+// included, so the series shows real failure pressure and not just the+// messages that ran out of attempts.+export const kafkaHandlerFailuresTotal = new client.Counter({+ name: 'kafka_events_handler_failures_total',+ help: 'Kafka event handler exceptions (each attempt, retries included)',+ labelNames: ['partition'],+ registers: [registry],+});++// Messages written to the dead-letter topic after their attempts were+// exhausted (or after they failed to parse). One increment per message.+export const kafkaDeadLetteredTotal = new client.Counter({+ name: 'kafka_events_dead_lettered_total',+ help: 'Kafka event messages produced to the dead-letter topic',+ labelNames: ['partition', 'reason'],+ registers: [registry],+});++// The dead-letter produce itself failed. The offset is then left unresolved+// and the message is redelivered, so this is a stuck partition, not a loss.+export const kafkaDeadLetterFailedTotal = new client.Counter({+ name: 'kafka_events_dead_letter_failed_total',+ help: 'Failed attempts to produce a Kafka event message to the dead-letter topic',+ labelNames: ['partition'],+ registers: [registry],+});++export const sessionsStarted = new client.Counter({+ name: 'sessions_started_total',+ help: 'session_start events emitted, by lifecycle kind',+ labelNames: ['kind'], // 'new' | 'boundary'+ registers: [registry],+});diff --git a/pnpm-lock.yaml b/pnpm-lock.yamlindex 33abe4199..d1e8bc1f1 100644--- a/pnpm-lock.yaml+++ b/pnpm-lock.yaml@@ -1169,6 +1169,9 @@ importers:js-yaml:specifier: ^4.1.0version: 4.1.1+ kafkajs:+ specifier: ^2.2.4+ version: 2.2.4lru-cache:specifier: ^11.2.4version: 11.5.1