OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.
ralph(M15-113): Per-module fixes: packages/core/src/modules/ingest - R1, R4, R13, R16 and the findings no rule covers
ralph2f8e9457bd23
diff --git a/apps/api/src/main.ts b/apps/api/src/main.tsindex 07b89d20f..3ca47ea65 100644--- a/apps/api/src/main.ts+++ b/apps/api/src/main.ts@@ -53,6 +53,7 @@ import {createTrpcFetchHandler,dashboardRoutes,debugRoutes,+ disconnectKafka,enableEventsHeartbeat,errorHandler,type HttpCtx,@@ -531,6 +532,10 @@ async function main() {}await Promise.all([workers?.close(), consumer?.stop()]);await deps.producers.close();+ // Core opens the Kafka producer lazily on the first /track and the+ // consumer at boot; `deps.producers` is the BullMQ handle, not this one.+ // Without this an idempotent producer with in-flight batches is dropped.+ await disconnectKafka(config.core);logger.info('Graceful shutdown completed');clearTimeout(forceExit);process.exit(0);diff --git a/packages/core/src/modules/ingest/ingest.routes.ts b/packages/core/src/modules/ingest/ingest.routes.tsindex ad913f4d8..225e5580d 100644--- a/packages/core/src/modules/ingest/ingest.routes.ts+++ b/packages/core/src/modules/ingest/ingest.routes.ts@@ -87,11 +87,17 @@ function respondToOutcome(outcome: TrackOutcome, status: StatusFn) {});case 'profile-not-found':return status(404, { status: 404, message: 'Profile not found' });- default:+ case 'profile-property-not-a-number':return status(400, {status: 400,message: 'Property value is not a number',});+ default: {+ // `outcome` narrows to `never`: an eighth TrackOutcome variant fails the+ // compile here instead of silently getting the 400 above.+ const unhandled: never = outcome;+ return unhandled;+ }}}@@ -142,8 +148,12 @@ function windDownGuard(deps: AppDeps) {};}-export const ingestRoutes = defineRoutes((app, deps: AppDeps) =>- app.guard(+export const ingestRoutes = defineRoutes((app, deps: AppDeps) => {+ // One closure over `deps` for all three routes: this is route-definition+ // work, and three identical closures read as if it were per-request.+ const blockWhenWoundDown = windDownGuard(deps);++ return app.guard({// V1's `duplicateHook`, registered on `preValidation` so it answers// before the client is ever authenticated. A guard hook runs ahead of@@ -190,7 +200,7 @@ export const ingestRoutes = defineRoutes((app, deps: AppDeps) =>{clientAuth: { ingest: validateIngestRequest },body: zTrackHandlerPayload,- beforeHandle: [botGuard, windDownGuard(deps)],+ beforeHandle: [botGuard, blockWhenWoundDown],detail: {tags: TAGS,description:@@ -229,7 +239,7 @@ export const ingestRoutes = defineRoutes((app, deps: AppDeps) =>},{clientAuth: { ingest: validateIngestRequest },- beforeHandle: [botGuard, windDownGuard(deps)],+ beforeHandle: [botGuard, blockWhenWoundDown],detail: {tags: TAGS,description:@@ -272,7 +282,7 @@ export const ingestRoutes = defineRoutes((app, deps: AppDeps) =>recordLegacyEventRequest(client.id);},botGuard,- windDownGuard(deps),+ blockWhenWoundDown,],detail: {tags: LEGACY_EVENT_TAGS,@@ -281,5 +291,5 @@ export const ingestRoutes = defineRoutes((app, deps: AppDeps) =>},})- )-);+ );+});diff --git a/packages/core/src/modules/ingest/ingest.service.ts b/packages/core/src/modules/ingest/ingest.service.tsindex 1b5e6fa05..410c20028 100644--- a/packages/core/src/modules/ingest/ingest.service.ts+++ b/packages/core/src/modules/ingest/ingest.service.ts@@ -94,7 +94,7 @@ export interface IngestTransport {/** What the two context builders below need: the Postgres client the salt* cache keys on, and the idle window every session decision reads. */-type IngestScope = DbScope & Pick<ServiceDeps, 'config'>;+type IngestScope = DbScope & Pick<ServiceDeps, 'config' | 'logger'>;const QUEUE_PAYLOAD_HEADERS = ['user-agent',@@ -113,13 +113,13 @@ const FALLBACK_USER_AGENT = 'unknown/1.0';export function getStringHeaders(headers: IngestHeaders): Record<string, string | undefined> {- return Object.entries(pick(QUEUE_PAYLOAD_HEADERS, headers)).reduce(- (acc, [key, value]) => ({- ...acc,- [key]: value ? String(value) : undefined,- }),- {}- );+ const stringHeaders: Record<string, string | undefined> = {};+ for (const [key, value] of Object.entries(+ pick(QUEUE_PAYLOAD_HEADERS, headers)+ )) {+ stringHeaders[key] = value ? String(value) : undefined;+ }+ return stringHeaders;}function getIdentity(body: ITrackHandlerPayload): IIdentifyPayload | undefined {@@ -273,6 +273,7 @@ async function buildContext(eventTimeMs: timestamp.timestamp,sessionBuffer: buffers.session,sessionTimeoutMs,+ logger: deps.logger,});return {@@ -665,6 +666,7 @@ export async function ingestLegacyEvent(salts,sessionBuffer: transport.buffers.session,sessionTimeoutMs: resolveSessionTimeoutMs(transport.deps.config),+ logger: transport.deps.logger,});const uaInfo = parseUserAgent(ua, request.body?.properties);@@ -918,19 +920,21 @@ export function createIngestService(deps: ServiceDeps,_services: () => Services) {- return {- track: (- request: TrackRequest,- produceIncomingEvent: IncomingEventProducer- ): Promise<TrackOutcome> =>- ingestTrack(request, {- buffers: deps.buffers,- produceIncomingEvent,- deps,- }),- checkBot: (- request: Parameters<typeof checkIngestBot>[1]- ): Promise<BotVerdict> => checkIngestBot(deps, request),- isDuplicate: isDuplicateIngestRequest,- };+ const track = (+ request: TrackRequest,+ produceIncomingEvent: IncomingEventProducer+ ): Promise<TrackOutcome> =>+ ingestTrack(request, {+ buffers: deps.buffers,+ produceIncomingEvent,+ deps,+ });++ const checkBot = (+ request: Parameters<typeof checkIngestBot>[1]+ ): Promise<BotVerdict> => checkIngestBot(deps, request);++ const isDuplicate = isDuplicateIngestRequest;++ return { track, checkBot, isDuplicate };}diff --git a/packages/core/src/modules/ingest/src/device-id.test.ts b/packages/core/src/modules/ingest/src/device-id.test.tsindex 33151f3eb..330ad64a1 100644--- a/packages/core/src/modules/ingest/src/device-id.test.ts+++ b/packages/core/src/modules/ingest/src/device-id.test.ts@@ -13,7 +13,11 @@import { afterEach, describe, expect, it, mock } from 'bun:test';import type { IClickhouseSession } from '../../session/session.service';import { formatClickhouseDate } from '../../session/src/dates';-import { getDeviceId, type SessionBufferReader } from './device-id';+import {+ getDeviceId,+ type SessionBufferReader,+ type SessionResolutionLogger,+} from './device-id';const NOW = new Date('2026-06-08T12:00:00.000Z').getTime();const MINUTE = 60 * 1000;@@ -21,6 +25,13 @@ const SALTS = { current: 'salt-current', previous: 'salt-previous' };/** SESSION_TIMEOUT_MS's default: the caller resolves it from config now. */const SESSION_TIMEOUT_MS = 30 * 60 * 1000;+/** The caller's request logger. Inputs here are an IP and a salt, so a+ * session-read failure must go through pino's redaction, not `console`. */+const stubLogger = () => {+ const error = mock((_details: unknown, _message?: string) => undefined);+ return { error, logger: { error } as unknown as SessionResolutionLogger };+};+const BASE = {projectId: 'proj-1',ip: '1.2.3.4',@@ -28,6 +39,7 @@ const BASE = {salts: SALTS,eventTimeMs: NOW,sessionTimeoutMs: SESSION_TIMEOUT_MS,+ logger: stubLogger().logger,};// withinIdleWindow only reads `id` + `ended_at`; the rest is irrelevant here.@@ -126,4 +138,25 @@ describe('getDeviceId — session resolution', () => {});expect(new Set(deviceIds).size).toBe(2); // distinct current/previous hashes});++ it('logs a failed session read through the caller logger, not console', async () => {+ const { error, logger } = stubLogger();+ const failing = {+ getExistingSession: mock(() => {+ throw new Error('session store unavailable');+ }),+ } as unknown as SessionBufferReader;++ const result = await getDeviceId({+ ...BASE,+ logger,+ overrideDeviceId: 'cookie-abc',+ sessionBuffer: failing,+ });++ expect(error).toHaveBeenCalledTimes(1);+ // Still answers with a deterministic id: the read is best-effort.+ expect(result.deviceId).toBe('cookie-abc');+ expect(result.sessionId).not.toBe('');+ });});diff --git a/packages/core/src/modules/ingest/src/device-id.ts b/packages/core/src/modules/ingest/src/device-id.tsindex 04b8a84e8..c056acbd4 100644--- a/packages/core/src/modules/ingest/src/device-id.ts+++ b/packages/core/src/modules/ingest/src/device-id.ts@@ -4,6 +4,7 @@import crypto from 'node:crypto';import type { SessionBuffer } from '../../../buffers/session-buffer';+import type { Logger } from '../../../logger';import { generateDeviceId } from '../../../shared/profileId';import type { IClickhouseSession } from '../../session/session.service';import { convertClickhouseDateToJs } from '../../session/src/dates';@@ -11,6 +12,10 @@ import { convertClickhouseDateToJs } from '../../session/src/dates';/** Only the read this module needs, so a caller may pass a narrower stub. */export type SessionBufferReader = Pick<SessionBuffer, 'getExistingSession'>;+/** Only the level this module logs at. The inputs here are an IP and a salt,+ * so it must be the redacting pino logger and never `console`. */+export type SessionResolutionLogger = Pick<Logger, 'error'>;+export interface DeviceIdResult {deviceId: string;sessionId: string;@@ -36,6 +41,7 @@ export async function getDeviceId({eventTimeMs,sessionBuffer,sessionTimeoutMs,+ logger,}: {projectId: string;ip: string;@@ -48,6 +54,7 @@ export async function getDeviceId({sessionBuffer: SessionBufferReader;/** The idle window, resolved from config by the caller. */sessionTimeoutMs: number;+ logger: SessionResolutionLogger;}): Promise<DeviceIdResult> {if (overrideDeviceId) {// A caller-supplied device id is stable (no salt rotation), so it's the only@@ -58,6 +65,7 @@ export async function getDeviceId({eventTimeMs: eventTimeMs ?? Date.now(),sessionBuffer,sessionTimeoutMs,+ logger,});}@@ -84,6 +92,7 @@ export async function getDeviceId({eventTimeMs: eventTimeMs ?? Date.now(),sessionBuffer,sessionTimeoutMs,+ logger,});}@@ -112,6 +121,7 @@ async function getInfoFromSession({eventTimeMs,sessionBuffer,sessionTimeoutMs,+ logger,}: {projectId: string;/** Candidate device ids in priority order (e.g. [current, previous] salt@@ -120,6 +130,7 @@ async function getInfoFromSession({eventTimeMs: number;sessionBuffer: SessionBufferReader;sessionTimeoutMs: number;+ logger: SessionResolutionLogger;}): Promise<DeviceIdResult> {const candidates = [...new Set(deviceIds.filter(Boolean))];const primary = candidates[0] ?? '';@@ -140,7 +151,7 @@ async function getInfoFromSession({}}} catch (error) {- console.error('Error resolving session for device id', error);+ logger.error({ err: error }, 'Error resolving session for device id');}return {@@ -171,11 +182,12 @@ async function getInfoFromSession({* Deterministic session id for (projectId, deviceId) within a time window,* with a grace period at the *start* of each window to avoid boundary splits.*- * - windowMs: 30 minutes by default- * - graceMs: 1 minute by default (events in first minute of a bucket map to previous bucket)+ * - windowMs: `DEFAULT_SESSION_WINDOW_MS` (5 minutes) by default+ * - graceMs: `DEFAULT_SESSION_GRACE_MS` (1 minute) by default (events in the+ * grace period at the start of a bucket map to the previous bucket)* - Output: base64url, 128-bit (16 bytes) truncated from SHA-256*/-export function getSessionId(params: {+function getSessionId(params: {projectId: string;deviceId: string;eventMs?: number; // use event timestamp; defaults to Date.now()diff --git a/packages/core/src/modules/ingest/src/kafka.ts b/packages/core/src/modules/ingest/src/kafka.tsindex 01bd14cd0..6423aacee 100644--- a/packages/core/src/modules/ingest/src/kafka.ts+++ b/packages/core/src/modules/ingest/src/kafka.ts@@ -232,6 +232,12 @@ export const createKafkaEventsConsumer = (config.kafka.maxMessagesPerPartition * KAFKA_BYTES_PER_MESSAGE,});consumers.add(consumer);+ // Whoever opens closes: a consumer that disconnected on its own (the events+ // handle's `stop()`) drops out of the set so shutdown never disconnects it+ // twice and the set never retains a dead consumer.+ consumer.on(consumer.events.DISCONNECT, () => {+ consumers.delete(consumer);+ });return consumer;};