Skip to content

OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.

ralph(M15-113): Per-module fixes: packages/core/src/modules/ingest - R1, R4, R13, R16 and the findings no rule covers

ralph2f8e9457bd23
diff --git a/apps/api/src/main.ts b/apps/api/src/main.ts
index 07b89d20f..3ca47ea65 100644
--- a/apps/api/src/main.ts
+++ b/apps/api/src/main.ts
@@ -53,6 +53,7 @@ import {
createTrpcFetchHandler,
dashboardRoutes,
debugRoutes,
+ disconnectKafka,
enableEventsHeartbeat,
errorHandler,
type HttpCtx,
@@ -531,6 +532,10 @@ async function main() {
}
await Promise.all([workers?.close(), consumer?.stop()]);
await deps.producers.close();
+ // Core opens the Kafka producer lazily on the first /track and the
+ // consumer at boot; `deps.producers` is the BullMQ handle, not this one.
+ // Without this an idempotent producer with in-flight batches is dropped.
+ await disconnectKafka(config.core);
logger.info('Graceful shutdown completed');
clearTimeout(forceExit);
process.exit(0);
diff --git a/packages/core/src/modules/ingest/ingest.routes.ts b/packages/core/src/modules/ingest/ingest.routes.ts
index ad913f4d8..225e5580d 100644
--- a/packages/core/src/modules/ingest/ingest.routes.ts
+++ b/packages/core/src/modules/ingest/ingest.routes.ts
@@ -87,11 +87,17 @@ function respondToOutcome(outcome: TrackOutcome, status: StatusFn) {
});
case 'profile-not-found':
return status(404, { status: 404, message: 'Profile not found' });
- default:
+ case 'profile-property-not-a-number':
return status(400, {
status: 400,
message: 'Property value is not a number',
});
+ default: {
+ // `outcome` narrows to `never`: an eighth TrackOutcome variant fails the
+ // compile here instead of silently getting the 400 above.
+ const unhandled: never = outcome;
+ return unhandled;
+ }
}
}
@@ -142,8 +148,12 @@ function windDownGuard(deps: AppDeps) {
};
}
-export const ingestRoutes = defineRoutes((app, deps: AppDeps) =>
- app.guard(
+export const ingestRoutes = defineRoutes((app, deps: AppDeps) => {
+ // One closure over `deps` for all three routes: this is route-definition
+ // work, and three identical closures read as if it were per-request.
+ const blockWhenWoundDown = windDownGuard(deps);
+
+ return app.guard(
{
// V1's `duplicateHook`, registered on `preValidation` so it answers
// before the client is ever authenticated. A guard hook runs ahead of
@@ -190,7 +200,7 @@ export const ingestRoutes = defineRoutes((app, deps: AppDeps) =>
{
clientAuth: { ingest: validateIngestRequest },
body: zTrackHandlerPayload,
- beforeHandle: [botGuard, windDownGuard(deps)],
+ beforeHandle: [botGuard, blockWhenWoundDown],
detail: {
tags: TAGS,
description:
@@ -229,7 +239,7 @@ export const ingestRoutes = defineRoutes((app, deps: AppDeps) =>
},
{
clientAuth: { ingest: validateIngestRequest },
- beforeHandle: [botGuard, windDownGuard(deps)],
+ beforeHandle: [botGuard, blockWhenWoundDown],
detail: {
tags: TAGS,
description:
@@ -272,7 +282,7 @@ export const ingestRoutes = defineRoutes((app, deps: AppDeps) =>
recordLegacyEventRequest(client.id);
},
botGuard,
- windDownGuard(deps),
+ blockWhenWoundDown,
],
detail: {
tags: LEGACY_EVENT_TAGS,
@@ -281,5 +291,5 @@ export const ingestRoutes = defineRoutes((app, deps: AppDeps) =>
},
}
)
- )
-);
+ );
+});
diff --git a/packages/core/src/modules/ingest/ingest.service.ts b/packages/core/src/modules/ingest/ingest.service.ts
index 1b5e6fa05..410c20028 100644
--- a/packages/core/src/modules/ingest/ingest.service.ts
+++ b/packages/core/src/modules/ingest/ingest.service.ts
@@ -94,7 +94,7 @@ export interface IngestTransport {
/** What the two context builders below need: the Postgres client the salt
* cache keys on, and the idle window every session decision reads. */
-type IngestScope = DbScope & Pick<ServiceDeps, 'config'>;
+type IngestScope = DbScope & Pick<ServiceDeps, 'config' | 'logger'>;
const QUEUE_PAYLOAD_HEADERS = [
'user-agent',
@@ -113,13 +113,13 @@ const FALLBACK_USER_AGENT = 'unknown/1.0';
export function getStringHeaders(
headers: IngestHeaders
): Record<string, string | undefined> {
- return Object.entries(pick(QUEUE_PAYLOAD_HEADERS, headers)).reduce(
- (acc, [key, value]) => ({
- ...acc,
- [key]: value ? String(value) : undefined,
- }),
- {}
- );
+ const stringHeaders: Record<string, string | undefined> = {};
+ for (const [key, value] of Object.entries(
+ pick(QUEUE_PAYLOAD_HEADERS, headers)
+ )) {
+ stringHeaders[key] = value ? String(value) : undefined;
+ }
+ return stringHeaders;
}
function getIdentity(body: ITrackHandlerPayload): IIdentifyPayload | undefined {
@@ -273,6 +273,7 @@ async function buildContext(
eventTimeMs: timestamp.timestamp,
sessionBuffer: buffers.session,
sessionTimeoutMs,
+ logger: deps.logger,
});
return {
@@ -665,6 +666,7 @@ export async function ingestLegacyEvent(
salts,
sessionBuffer: transport.buffers.session,
sessionTimeoutMs: resolveSessionTimeoutMs(transport.deps.config),
+ logger: transport.deps.logger,
});
const uaInfo = parseUserAgent(ua, request.body?.properties);
@@ -918,19 +920,21 @@ export function createIngestService(
deps: ServiceDeps,
_services: () => Services
) {
- return {
- track: (
- request: TrackRequest,
- produceIncomingEvent: IncomingEventProducer
- ): Promise<TrackOutcome> =>
- ingestTrack(request, {
- buffers: deps.buffers,
- produceIncomingEvent,
- deps,
- }),
- checkBot: (
- request: Parameters<typeof checkIngestBot>[1]
- ): Promise<BotVerdict> => checkIngestBot(deps, request),
- isDuplicate: isDuplicateIngestRequest,
- };
+ const track = (
+ request: TrackRequest,
+ produceIncomingEvent: IncomingEventProducer
+ ): Promise<TrackOutcome> =>
+ ingestTrack(request, {
+ buffers: deps.buffers,
+ produceIncomingEvent,
+ deps,
+ });
+
+ const checkBot = (
+ request: Parameters<typeof checkIngestBot>[1]
+ ): Promise<BotVerdict> => checkIngestBot(deps, request);
+
+ const isDuplicate = isDuplicateIngestRequest;
+
+ return { track, checkBot, isDuplicate };
}
diff --git a/packages/core/src/modules/ingest/src/device-id.test.ts b/packages/core/src/modules/ingest/src/device-id.test.ts
index 33151f3eb..330ad64a1 100644
--- a/packages/core/src/modules/ingest/src/device-id.test.ts
+++ b/packages/core/src/modules/ingest/src/device-id.test.ts
@@ -13,7 +13,11 @@
import { afterEach, describe, expect, it, mock } from 'bun:test';
import type { IClickhouseSession } from '../../session/session.service';
import { formatClickhouseDate } from '../../session/src/dates';
-import { getDeviceId, type SessionBufferReader } from './device-id';
+import {
+ getDeviceId,
+ type SessionBufferReader,
+ type SessionResolutionLogger,
+} from './device-id';
const NOW = new Date('2026-06-08T12:00:00.000Z').getTime();
const MINUTE = 60 * 1000;
@@ -21,6 +25,13 @@ const SALTS = { current: 'salt-current', previous: 'salt-previous' };
/** SESSION_TIMEOUT_MS's default: the caller resolves it from config now. */
const SESSION_TIMEOUT_MS = 30 * 60 * 1000;
+/** The caller's request logger. Inputs here are an IP and a salt, so a
+ * session-read failure must go through pino's redaction, not `console`. */
+const stubLogger = () => {
+ const error = mock((_details: unknown, _message?: string) => undefined);
+ return { error, logger: { error } as unknown as SessionResolutionLogger };
+};
+
const BASE = {
projectId: 'proj-1',
ip: '1.2.3.4',
@@ -28,6 +39,7 @@ const BASE = {
salts: SALTS,
eventTimeMs: NOW,
sessionTimeoutMs: SESSION_TIMEOUT_MS,
+ logger: stubLogger().logger,
};
// withinIdleWindow only reads `id` + `ended_at`; the rest is irrelevant here.
@@ -126,4 +138,25 @@ describe('getDeviceId — session resolution', () => {
});
expect(new Set(deviceIds).size).toBe(2); // distinct current/previous hashes
});
+
+ it('logs a failed session read through the caller logger, not console', async () => {
+ const { error, logger } = stubLogger();
+ const failing = {
+ getExistingSession: mock(() => {
+ throw new Error('session store unavailable');
+ }),
+ } as unknown as SessionBufferReader;
+
+ const result = await getDeviceId({
+ ...BASE,
+ logger,
+ overrideDeviceId: 'cookie-abc',
+ sessionBuffer: failing,
+ });
+
+ expect(error).toHaveBeenCalledTimes(1);
+ // Still answers with a deterministic id: the read is best-effort.
+ expect(result.deviceId).toBe('cookie-abc');
+ expect(result.sessionId).not.toBe('');
+ });
});
diff --git a/packages/core/src/modules/ingest/src/device-id.ts b/packages/core/src/modules/ingest/src/device-id.ts
index 04b8a84e8..c056acbd4 100644
--- a/packages/core/src/modules/ingest/src/device-id.ts
+++ b/packages/core/src/modules/ingest/src/device-id.ts
@@ -4,6 +4,7 @@
import crypto from 'node:crypto';
import type { SessionBuffer } from '../../../buffers/session-buffer';
+import type { Logger } from '../../../logger';
import { generateDeviceId } from '../../../shared/profileId';
import type { IClickhouseSession } from '../../session/session.service';
import { convertClickhouseDateToJs } from '../../session/src/dates';
@@ -11,6 +12,10 @@ import { convertClickhouseDateToJs } from '../../session/src/dates';
/** Only the read this module needs, so a caller may pass a narrower stub. */
export type SessionBufferReader = Pick<SessionBuffer, 'getExistingSession'>;
+/** Only the level this module logs at. The inputs here are an IP and a salt,
+ * so it must be the redacting pino logger and never `console`. */
+export type SessionResolutionLogger = Pick<Logger, 'error'>;
+
export interface DeviceIdResult {
deviceId: string;
sessionId: string;
@@ -36,6 +41,7 @@ export async function getDeviceId({
eventTimeMs,
sessionBuffer,
sessionTimeoutMs,
+ logger,
}: {
projectId: string;
ip: string;
@@ -48,6 +54,7 @@ export async function getDeviceId({
sessionBuffer: SessionBufferReader;
/** The idle window, resolved from config by the caller. */
sessionTimeoutMs: number;
+ logger: SessionResolutionLogger;
}): Promise<DeviceIdResult> {
if (overrideDeviceId) {
// A caller-supplied device id is stable (no salt rotation), so it's the only
@@ -58,6 +65,7 @@ export async function getDeviceId({
eventTimeMs: eventTimeMs ?? Date.now(),
sessionBuffer,
sessionTimeoutMs,
+ logger,
});
}
@@ -84,6 +92,7 @@ export async function getDeviceId({
eventTimeMs: eventTimeMs ?? Date.now(),
sessionBuffer,
sessionTimeoutMs,
+ logger,
});
}
@@ -112,6 +121,7 @@ async function getInfoFromSession({
eventTimeMs,
sessionBuffer,
sessionTimeoutMs,
+ logger,
}: {
projectId: string;
/** Candidate device ids in priority order (e.g. [current, previous] salt
@@ -120,6 +130,7 @@ async function getInfoFromSession({
eventTimeMs: number;
sessionBuffer: SessionBufferReader;
sessionTimeoutMs: number;
+ logger: SessionResolutionLogger;
}): Promise<DeviceIdResult> {
const candidates = [...new Set(deviceIds.filter(Boolean))];
const primary = candidates[0] ?? '';
@@ -140,7 +151,7 @@ async function getInfoFromSession({
}
}
} catch (error) {
- console.error('Error resolving session for device id', error);
+ logger.error({ err: error }, 'Error resolving session for device id');
}
return {
@@ -171,11 +182,12 @@ async function getInfoFromSession({
* Deterministic session id for (projectId, deviceId) within a time window,
* with a grace period at the *start* of each window to avoid boundary splits.
*
- * - windowMs: 30 minutes by default
- * - graceMs: 1 minute by default (events in first minute of a bucket map to previous bucket)
+ * - windowMs: `DEFAULT_SESSION_WINDOW_MS` (5 minutes) by default
+ * - graceMs: `DEFAULT_SESSION_GRACE_MS` (1 minute) by default (events in the
+ * grace period at the start of a bucket map to the previous bucket)
* - Output: base64url, 128-bit (16 bytes) truncated from SHA-256
*/
-export function getSessionId(params: {
+function getSessionId(params: {
projectId: string;
deviceId: string;
eventMs?: number; // use event timestamp; defaults to Date.now()
diff --git a/packages/core/src/modules/ingest/src/kafka.ts b/packages/core/src/modules/ingest/src/kafka.ts
index 01bd14cd0..6423aacee 100644
--- a/packages/core/src/modules/ingest/src/kafka.ts
+++ b/packages/core/src/modules/ingest/src/kafka.ts
@@ -232,6 +232,12 @@ export const createKafkaEventsConsumer = (
config.kafka.maxMessagesPerPartition * KAFKA_BYTES_PER_MESSAGE,
});
consumers.add(consumer);
+ // Whoever opens closes: a consumer that disconnected on its own (the events
+ // handle's `stop()`) drops out of the set so shutdown never disconnects it
+ // twice and the set never retains a dead consumer.
+ consumer.on(consumer.events.DISCONNECT, () => {
+ consumers.delete(consumer);
+ });
return consumer;
};