Skip to content

OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.

ralph(M8-002): ingest module: /track + hooks (client-auth, duplicate, isBot) + bots into core

ralphd5cf585418cc
diff --git a/apps/api/package.json b/apps/api/package.json
index 46becd36f..3af22cc73 100644
--- a/apps/api/package.json
+++ b/apps/api/package.json
@@ -7,7 +7,6 @@
"testing": "API_PORT=3333 pnpm dev",
"start": "dotenv -e ../../.env node dist/index.js",
"build": "rm -rf dist && tsdown",
- "gen:bots": "jiti scripts/get-bots.ts && biome format --write src/bots/bots.ts",
"test:manage": "jiti scripts/test-manage-api.ts",
"e2e:sessions": "LOG_LEVEL=warn dotenv -e ../../.env -- jiti e2e/session-e2e.ts",
"e2e:sessions:stress": "LOG_LEVEL=warn dotenv -e ../../.env -- jiti e2e/session-stress.ts",
@@ -55,12 +54,10 @@
"devDependencies": {
"@faker-js/faker": "^9.0.1",
"@openpanel/tsconfig": "workspace:*",
- "@types/js-yaml": "^4.0.9",
"@types/ramda": "^0.30.2",
"@types/source-map-support": "^0.5.10",
"@types/uuid": "^10.0.0",
"@types/ws": "^8.5.14",
- "js-yaml": "^4.1.0",
"tsdown": "0.14.2",
"typescript": "catalog:",
"vitest": "^1.0.0"
diff --git a/apps/api/src/controllers/event.controller.test.ts b/apps/api/src/controllers/event.controller.test.ts
index 6e23036f2..d7aee7a58 100644
--- a/apps/api/src/controllers/event.controller.test.ts
+++ b/apps/api/src/controllers/event.controller.test.ts
@@ -9,8 +9,9 @@
* - legacyEventUsageHook increments openpanel_legacy_event_requests_total,
* labelled by client id, and never mints a label for an unauthenticated
* request.
- * - validateSdkRequest still accepts the `mixan-*` headers (ADR-015 entry 5 is
- * pending the same usage metric, so the fallback stays for now).
+ * The `mixan-*` header fallback moved with the validator itself into
+ * @openpanel/core's ingest module (M8-002) — see
+ * packages/core/src/modules/ingest/src/client-auth.test.ts.
*
* `@openpanel/db` is partially mocked (importActual + overrides) because it
* imports `@openpanel/queue`, which the produce assertions replace.
@@ -25,7 +26,6 @@ import { beforeEach, describe, expect, it, vi } from 'vitest';
import { postEvent } from './event.controller';
import { legacyEventUsageHook } from '@/hooks/legacy-event.hook';
import { legacyEventRequestsTotal } from '@/metrics';
-import { validateSdkRequest } from '@/utils/auth';
vi.mock('@openpanel/queue', async () => {
const actual =
@@ -44,7 +44,7 @@ vi.mock('@openpanel/core', async () => {
vi.mock('@openpanel/db', async () => {
const actual =
await vi.importActual<typeof import('@openpanel/db')>('@openpanel/db');
- return { ...actual, getSalts: vi.fn(), getClientByIdCached: vi.fn() };
+ return { ...actual, getSalts: vi.fn() };
});
const PROJECT_ID = 'legacy-event-project';
@@ -214,32 +214,3 @@ describe('legacyEventUsageHook', () => {
});
});
});
-
-describe('mixan-* header fallback', () => {
- it('authenticates a client sent as mixan-client-id (ADR-015 entry 5 deferred)', async () => {
- const { getClientByIdCached } = await import('@openpanel/db');
- vi.mocked(getClientByIdCached).mockResolvedValue({
- id: CLIENT_ID,
- projectId: PROJECT_ID,
- // Keeps this test about the header fallback rather than re-proving
- // secret verification, which the auth contracts already cover.
- ignoreCorsAndSecret: true,
- project: {
- id: PROJECT_ID,
- filters: [],
- allowUnsafeRevenueTracking: false,
- },
- } as unknown as Awaited<ReturnType<typeof getClientByIdCached>>);
-
- const request = makeRequest(legacyBody(), {
- 'mixan-client-id': CLIENT_ID,
- 'mixan-client-secret': 'legacy-secret',
- });
- delete (request as { client?: unknown }).client;
-
- const client = await validateSdkRequest(request);
-
- expect(client.id).toBe(CLIENT_ID);
- expect(request.clientSecretAuth).toBe(true);
- });
-});
diff --git a/apps/api/src/controllers/event.controller.ts b/apps/api/src/controllers/event.controller.ts
index 914983de7..77646cca2 100644
--- a/apps/api/src/controllers/event.controller.ts
+++ b/apps/api/src/controllers/event.controller.ts
@@ -1,6 +1,14 @@
import { generateId } from '@openpanel/common';
-import { getAsnInfo, getGeoLocation, parseUserAgent } from '@openpanel/core';
-import { getSalts } from '@openpanel/db';
+import {
+ applyBotSuspicion,
+ getAsnInfo,
+ getDeviceId,
+ getGeoLocation,
+ getStringHeaders,
+ getTimestamp,
+ parseUserAgent,
+} from '@openpanel/core';
+import { getSalts, sessionBuffer } from '@openpanel/db';
import {
type EventsQueuePayloadIncomingEvent,
produceIncomingEvent,
@@ -8,9 +16,6 @@ import {
import type { DeprecatedPostEventPayload } from '@openpanel/validation';
import type { FastifyReply, FastifyRequest } from 'fastify';
import { v4 as uuid } from 'uuid';
-import { getStringHeaders, getTimestamp } from './track.controller';
-import { applyBotSuspicion } from '@/bots/suspicion';
-import { getDeviceId } from '@/utils/ids';
export async function postEvent(
request: FastifyRequest<{
@@ -42,6 +47,7 @@ export async function postEvent(
ip,
ua,
salts,
+ sessionBuffer,
});
const uaInfo = parseUserAgent(ua, request.body?.properties);
diff --git a/apps/api/src/controllers/track.controller.test.ts b/apps/api/src/controllers/track.controller.test.ts
deleted file mode 100644
index 0fd3850e2..000000000
--- a/apps/api/src/controllers/track.controller.test.ts
+++ /dev/null
@@ -1,250 +0,0 @@
-/**
- * Tests for:
- * - getOverrideDeviceId — the guard around a track event's caller-supplied
- * `__deviceId` (it becomes a Redis key segment, a BullMQ jobId and a ClickHouse
- * device_id, so it must be trimmed, non-empty, and length-bounded).
- * - handleReplay — replay files a chunk under the session id the SDK echoes back;
- * it needs no device resolution and trusts the client-sent session id.
- * - handler — the single events transport (ADR-004): every tracked event goes to
- * Kafka via produceIncomingEvent, with no branch and no fallback queue.
- *
- * `@openpanel/db` and `@openpanel/queue` are partially mocked (importActual +
- * overrides) rather than replaced: they import each other, and the replay test
- * below spies on the real `replayBuffer`.
- */
-
-import { getAsnInfo, getGeoLocation } from '@openpanel/core';
-import {
- getSalts,
- replayBuffer,
- sessionBuffer,
- upsertProfile,
-} from '@openpanel/db';
-import { produceIncomingEvent } from '@openpanel/queue';
-import type {
- IReplayPayload,
- ITrackHandlerPayload,
-} from '@openpanel/validation';
-import type { FastifyReply, FastifyRequest } from 'fastify';
-import { beforeEach, describe, expect, it, vi } from 'vitest';
-import { getOverrideDeviceId, handleReplay, handler } from './track.controller';
-
-vi.mock('@openpanel/queue', async () => {
- const actual =
- await vi.importActual<typeof import('@openpanel/queue')>(
- '@openpanel/queue'
- );
- return { ...actual, produceIncomingEvent: vi.fn() };
-});
-
-vi.mock('@openpanel/core', async () => {
- const actual =
- await vi.importActual<typeof import('@openpanel/core')>('@openpanel/core');
- return { ...actual, getGeoLocation: vi.fn(), getAsnInfo: vi.fn() };
-});
-
-vi.mock('@openpanel/db', async () => {
- const actual =
- await vi.importActual<typeof import('@openpanel/db')>('@openpanel/db');
- return { ...actual, getSalts: vi.fn(), upsertProfile: vi.fn() };
-});
-
-const track = (properties?: Record<string, unknown>): ITrackHandlerPayload =>
- ({
- type: 'track',
- payload: { name: 'page_view', properties },
- }) as ITrackHandlerPayload;
-
-describe('getOverrideDeviceId', () => {
- it('returns a trimmed device id', () => {
- expect(getOverrideDeviceId(track({ __deviceId: ' cookie-abc ' }))).toBe(
- 'cookie-abc'
- );
- });
-
- it('returns the device id as-is when already clean', () => {
- expect(getOverrideDeviceId(track({ __deviceId: 'cookie-abc' }))).toBe(
- 'cookie-abc'
- );
- });
-
- it('ignores empty / whitespace-only values', () => {
- expect(getOverrideDeviceId(track({ __deviceId: '' }))).toBeUndefined();
- expect(getOverrideDeviceId(track({ __deviceId: ' ' }))).toBeUndefined();
- });
-
- it('ignores non-string values', () => {
- expect(getOverrideDeviceId(track({ __deviceId: 123 }))).toBeUndefined();
- expect(getOverrideDeviceId(track({}))).toBeUndefined();
- expect(getOverrideDeviceId(track(undefined))).toBeUndefined();
- });
-
- it('ignores pathologically long values (cap is 64)', () => {
- expect(
- getOverrideDeviceId(track({ __deviceId: 'x'.repeat(65) }))
- ).toBeUndefined();
- expect(getOverrideDeviceId(track({ __deviceId: 'x'.repeat(64) }))).toBe(
- 'x'.repeat(64)
- );
- });
-
- it('returns undefined for non-track payload types', () => {
- expect(
- getOverrideDeviceId({
- type: 'identify',
- payload: { profileId: 'p1', properties: { __deviceId: 'cookie' } },
- } as ITrackHandlerPayload)
- ).toBeUndefined();
- });
-});
-
-const chunk = (): IReplayPayload => ({
- chunk_index: 0,
- events_count: 1,
- is_full_snapshot: true,
- started_at: '2026-06-08T12:00:00.000Z',
- ended_at: '2026-06-08T12:00:01.000Z',
- payload: '[]',
-});
-
-describe('handleReplay', () => {
- beforeEach(() => {
- vi.restoreAllMocks();
- });
-
- it('files the chunk under the session id provided by the caller', async () => {
- const add = vi.spyOn(replayBuffer, 'add').mockResolvedValue(undefined);
-
- await handleReplay(chunk(), {
- projectId: 'proj-1',
- sessionId: 'sess-issued-123',
- });
-
- expect(add).toHaveBeenCalledTimes(1);
- expect(add).toHaveBeenCalledWith(
- expect.objectContaining({
- project_id: 'proj-1',
- session_id: 'sess-issued-123',
- })
- );
- });
-
- it('throws when the session id is missing', async () => {
- const add = vi.spyOn(replayBuffer, 'add').mockResolvedValue(undefined);
-
- await expect(
- handleReplay(chunk(), { projectId: 'proj-1', sessionId: undefined })
- ).rejects.toThrow('Session ID is required for replay');
- expect(add).not.toHaveBeenCalled();
- });
-});
-
-const PROJECT_ID = 'proj-kafka';
-const DEVICE_ID = 'device-kafka-1';
-
-const makeRequest = (body: ITrackHandlerPayload) =>
- ({
- body,
- client: { projectId: PROJECT_ID },
- clientIp: '1.2.3.4',
- clientSecretAuth: false,
- timestamp: new Date('2026-06-08T12:00:00.000Z').getTime(),
- headers: { 'user-agent': 'Mozilla/5.0 (Macintosh) Chrome/148.0.0.0' },
- }) as unknown as FastifyRequest<{ Body: ITrackHandlerPayload }>;
-
-const makeReply = () => {
- const reply = {
- statusCode: 0,
- body: undefined as unknown,
- status(code: number) {
- reply.statusCode = code;
- return reply;
- },
- send(payload: unknown) {
- reply.body = payload;
- return reply;
- },
- };
- return reply as unknown as FastifyReply & typeof reply;
-};
-
-describe('handler — events transport', () => {
- beforeEach(() => {
- vi.clearAllMocks();
- vi.mocked(getSalts).mockResolvedValue({
- current: 'salt-current',
- previous: 'salt-previous',
- });
- vi.mocked(getGeoLocation).mockResolvedValue({
- country: 'SE',
- city: 'Stockholm',
- region: 'AB',
- longitude: 18,
- latitude: 59,
- });
- vi.mocked(getAsnInfo).mockResolvedValue({
- asn: undefined,
- org: undefined,
- isHosting: false,
- });
- vi.mocked(produceIncomingEvent).mockResolvedValue(undefined);
- vi.spyOn(sessionBuffer, 'getExistingSession').mockResolvedValue(null);
- });
-
- it('produces every tracked event to Kafka — the only transport', async () => {
- await handler(
- makeRequest({
- type: 'track',
- payload: {
- name: 'page_view',
- properties: { __deviceId: DEVICE_ID },
- },
- } as ITrackHandlerPayload),
- makeReply()
- );
-
- expect(produceIncomingEvent).toHaveBeenCalledTimes(1);
- const [queueData, partitionKey] =
- vi.mocked(produceIncomingEvent).mock.calls[0]!;
- expect(queueData).toMatchObject({
- projectId: PROJECT_ID,
- deviceId: DEVICE_ID,
- event: { name: 'page_view' },
- });
- // Browser events partition on the device id, which is what preserves
- // per-device mutual exclusion in the consumer.
- expect(partitionKey).toBe(DEVICE_ID);
- });
-
- it('surfaces a produce failure instead of falling back to another queue', async () => {
- vi.mocked(produceIncomingEvent).mockRejectedValue(
- new Error('kafka unavailable')
- );
-
- await expect(
- handler(
- makeRequest({
- type: 'track',
- payload: { name: 'page_view', properties: { __deviceId: DEVICE_ID } },
- } as ITrackHandlerPayload),
- makeReply()
- )
- ).rejects.toThrow('kafka unavailable');
-
- expect(produceIncomingEvent).toHaveBeenCalledTimes(1);
- });
-
- it('does not produce for non-event payload types', async () => {
- vi.mocked(upsertProfile).mockResolvedValue(undefined as never);
-
- await handler(
- makeRequest({
- type: 'identify',
- payload: { profileId: 'p1', properties: {} },
- } as ITrackHandlerPayload),
- makeReply()
- );
-
- expect(produceIncomingEvent).not.toHaveBeenCalled();
- });
-});
diff --git a/apps/api/src/controllers/track.controller.ts b/apps/api/src/controllers/track.controller.ts
index 2d877ebf1..fe1a1dd2c 100644
--- a/apps/api/src/controllers/track.controller.ts
+++ b/apps/api/src/controllers/track.controller.ts
@@ -1,411 +1,59 @@
-import { generateId } from '@openpanel/common';
import {
- type AsnInfo,
- type GeoLocation,
- generateDeviceId,
- getAsnInfo,
- getGeoLocation,
- parseUserAgent,
+ fetchDeviceIdentity,
+ ingestTrack,
+ type TrackOutcome,
} from '@openpanel/core';
-import {
- convertClickhouseDateToJs,
- getProfileById,
- getSalts,
- groupBuffer,
- replayBuffer,
- SESSION_TIMEOUT_MS,
- sessionBuffer,
- upsertProfile,
-} from '@openpanel/db';
-import {
- type EventsQueuePayloadIncomingEvent,
- produceIncomingEvent,
-} from '@openpanel/queue';
-import type {
- IAssignGroupPayload,
- IDecrementPayload,
- IGroupPayload,
- IIdentifyPayload,
- IIncrementPayload,
- IReplayPayload,
- ITrackHandlerPayload,
- ITrackPayload,
-} from '@openpanel/validation';
+import { groupBuffer, replayBuffer, sessionBuffer } from '@openpanel/db';
+import { produceIncomingEvent } from '@openpanel/queue';
+import type { ITrackHandlerPayload } from '@openpanel/validation';
import type { FastifyReply, FastifyRequest } from 'fastify';
-import { assocPath, pathOr, pick } from 'ramda';
-import { v4 as uuid } from 'uuid';
-import { applyBotSuspicion, stripBotProperties } from '@/bots/suspicion';
import { HttpError } from '@/utils/errors';
-import { getDeviceId } from '@/utils/ids';
-
-export function getStringHeaders(headers: FastifyRequest['headers']) {
- return Object.entries(
- pick(
- [
- 'user-agent',
- 'openpanel-sdk-name',
- 'openpanel-sdk-version',
- 'openpanel-client-id',
- 'request-id',
- ],
- headers
- )
- ).reduce(
- (acc, [key, value]) => ({
- ...acc,
- [key]: value ? String(value) : undefined,
- }),
- {}
- );
-}
-
-function getIdentity(body: ITrackHandlerPayload): IIdentifyPayload | undefined {
- if (body.type === 'track') {
- const identity = body.payload.properties?.__identify as
- | IIdentifyPayload
- | undefined;
-
- if (identity) {
- return identity;
- }
-
- return body.payload.profileId
- ? {
- profileId: String(body.payload.profileId),
- }
- : undefined;
- }
-
- return undefined;
-}
-
-const MAX_OVERRIDE_DEVICE_ID_LENGTH = 64;
-
-function sanitizeOverrideDeviceId(raw: unknown): string | undefined {
- if (typeof raw !== 'string') {
- return undefined;
- }
- const trimmed = raw.trim();
- if (!trimmed || trimmed.length > MAX_OVERRIDE_DEVICE_ID_LENGTH) {
- return undefined;
- }
- return trimmed;
-}
-
-// Resolve a caller-supplied device id from a track event's `properties.__deviceId`.
-export function getOverrideDeviceId(
- body: ITrackHandlerPayload
-): string | undefined {
- if (body.type !== 'track') {
- return undefined;
- }
- return sanitizeOverrideDeviceId(body.payload?.properties?.__deviceId);
-}
-
-export function getTimestamp(
- timestamp: FastifyRequest['timestamp'],
- payload: ITrackHandlerPayload['payload']
-) {
- const safeTimestamp = timestamp || Date.now();
- const userDefinedTimestamp =
- 'properties' in payload
- ? (payload?.properties?.__timestamp as string | undefined)
- : undefined;
-
- if (!userDefinedTimestamp) {
- return { timestamp: safeTimestamp, isTimestampFromThePast: false };
- }
-
- const clientTimestamp = new Date(userDefinedTimestamp);
- const clientTimestampNumber = clientTimestamp.getTime();
-
- // Constants for time validation
- const ONE_MINUTE_MS = 60 * 1000;
- const FIFTEEN_MINUTES_MS = 15 * ONE_MINUTE_MS;
-
- // Use safeTimestamp if invalid or more than 1 minute in the future
- if (
- Number.isNaN(clientTimestampNumber) ||
- clientTimestampNumber > safeTimestamp + ONE_MINUTE_MS
- ) {
- return { timestamp: safeTimestamp, isTimestampFromThePast: false };
- }
-
- // isTimestampFromThePast is true only if timestamp is older than 15 minutes
- const isTimestampFromThePast =
- clientTimestampNumber < safeTimestamp - FIFTEEN_MINUTES_MS;
-
- return {
- timestamp: clientTimestampNumber,
- isTimestampFromThePast,
- };
-}
-
-interface TrackContext {
- projectId: string;
- ip: string;
- ua?: string;
- // Whitelisted subset (getStringHeaders) that ships with the queue payload.
- headers: Record<string, string | undefined>;
- // Full raw request headers — bot signals need sec-ch-ua / sec-fetch-* /
- // accept-language, which the whitelist above deliberately drops.
- requestHeaders: FastifyRequest['headers'];
- clientSecretAuth: boolean;
- timestamp: { value: number; isFromPast: boolean };
- identity?: IIdentifyPayload;
- deviceId: string;
- sessionId: string;
- geo: GeoLocation;
- asnInfo: AsnInfo;
-}
-
-async function buildContext(
- request: FastifyRequest<{
- Body: ITrackHandlerPayload;
- }>,
- validatedBody: ITrackHandlerPayload
-): Promise<TrackContext> {
- const projectId = request.client?.projectId;
- if (!projectId) {
- throw new HttpError('Missing projectId', { status: 400 });
- }
-
- const timestamp = getTimestamp(request.timestamp, validatedBody.payload);
- const ip =
- validatedBody.type === 'track' && validatedBody.payload.properties?.__ip
- ? (validatedBody.payload.properties.__ip as string)
- : request.clientIp;
- const ua = request.headers['user-agent'] ?? 'unknown/1.0';
-
- const headers = getStringHeaders(request.headers);
- const identity = getIdentity(validatedBody);
- const profileId = identity?.profileId;
-
- if (profileId && validatedBody.type === 'track') {
- validatedBody.payload.profileId = profileId;
- }
-
- const overrideDeviceId = getOverrideDeviceId(validatedBody);
-
- // Get geo location (needed for track and identify) + ASN (bot detection).
- // Both hit the same MaxMind readers keyed on the same IP and are cached, so
- // resolving them together adds no meaningful latency.
- const [geo, asnInfo, salts] = await Promise.all([
- getGeoLocation(ip),
- getAsnInfo(ip),
- getSalts(),
- ]);
-
- const deviceIdResult = await getDeviceId({
- projectId,
- ip,
- ua,
- salts,
- overrideDeviceId,
- eventTimeMs: timestamp.timestamp,
- });
-
- return {
- projectId,
- ip,
- ua,
- headers,
- requestHeaders: request.headers,
- clientSecretAuth: request.clientSecretAuth ?? false,
- timestamp: {
- value: timestamp.timestamp,
- isFromPast: timestamp.isTimestampFromThePast,
- },
- identity,
- deviceId: deviceIdResult.deviceId,
- sessionId: deviceIdResult.sessionId,
- geo,
- asnInfo,
- };
-}
-
-async function handleTrack(
- payload: ITrackPayload,
- context: TrackContext
-): Promise<void> {
- const { projectId, deviceId, geo, headers, timestamp, sessionId } = context;
-
- const uaInfo = parseUserAgent(headers['user-agent'], payload.properties);
- // Mark (never block) likely bot traffic with __bot / __bot_reasons props.
- payload.properties = applyBotSuspicion(payload.properties, {
- asnInfo: context.asnInfo,
- headers: context.requestHeaders,
- clientSecretAuth: context.clientSecretAuth,
- isServer: uaInfo.isServer,
- });
- const groupId = uaInfo.isServer
- ? payload.profileId
- ? `${projectId}:${payload.profileId}`
- : undefined
- : deviceId;
- const promises: Promise<unknown>[] = [];
-
- // If we have more than one property in the identity object, we should identify the user
- // Otherwise its only a profileId and we should not identify the user
- if (context.identity && Object.keys(context.identity).length > 1) {
- promises.push(handleIdentify(context.identity, context));
- }
-
- const queueData: EventsQueuePayloadIncomingEvent['payload'] = {
- // The id of the ClickHouse row this event becomes, minted here so a
- // redelivered Kafka message is a duplicate we can recognise.
- id: uuid(),
- projectId,
- headers,
- event: {
- ...payload,
- groups: payload.groups ?? [],
- timestamp: timestamp.value,
- isTimestampFromThePast: timestamp.isFromPast,
- },
- uaInfo,
- geo,
- deviceId,
- sessionId,
- };
-
- const partitionKey = groupId || generateId();
-
- promises.push(produceIncomingEvent(queueData, partitionKey));
-
- await Promise.all(promises);
-}
-
-async function handleIdentify(
- payload: IIdentifyPayload,
- context: TrackContext
-): Promise<void> {
- const { projectId, geo, ua } = context;
- // Profiles must not carry forged bot verdicts either.
- stripBotProperties(payload.properties);
- const uaInfo = parseUserAgent(ua, payload.properties);
- await upsertProfile({
- ...payload,
- id: payload.profileId,
- isExternal: true,
- projectId,
- properties: {
- ...(payload.properties ?? {}),
- country: geo.country,
- city: geo.city,
- region: geo.region,
- longitude: geo.longitude,
- latitude: geo.latitude,
- os: uaInfo.os,
- os_version: uaInfo.osVersion,
- browser: uaInfo.browser,
- browser_version: uaInfo.browserVersion,
- device: uaInfo.device,
- brand: uaInfo.brand,
- model: uaInfo.model,
- },
- });
-}
-
-async function adjustProfileProperty(
- payload: IIncrementPayload | IDecrementPayload,
- projectId: string,
- direction: 1 | -1
-): Promise<void> {
- const { profileId, property, value } = payload;
- const profile = await getProfileById(String(profileId), projectId);
- if (!profile) {
- throw new HttpError('Profile not found', { status: 404 });
- }
-
- const parsed = Number.parseInt(
- pathOr<string>('0', property.split('.'), profile.properties),
- 10
- );
-
- if (Number.isNaN(parsed)) {
- throw new HttpError('Property value is not a number', { status: 400 });
- }
-
- profile.properties = assocPath(
- property.split('.'),
- parsed + direction * (value || 1),
- profile.properties
- );
-
- await upsertProfile({
- id: profile.id,
- projectId,
- properties: profile.properties,
- isExternal: true,
- });
-}
-
-async function handleIncrement(
- payload: IIncrementPayload,
- context: TrackContext
-): Promise<void> {
- await adjustProfileProperty(payload, context.projectId, 1);
-}
-
-async function handleDecrement(
- payload: IDecrementPayload,
- context: TrackContext
-): Promise<void> {
- await adjustProfileProperty(payload, context.projectId, -1);
-}
-
-// Replay only needs the server-issued session id (the SDK echoes it back). Trust
-// it — scoped to the authed project, unguessable, same trust level as event data.
-export async function handleReplay(
- payload: IReplayPayload,
- { projectId, sessionId }: { projectId: string; sessionId: string | undefined }
-): Promise<void> {
- if (!sessionId) {
- throw new HttpError('Session ID is required for replay', { status: 400 });
- }
- const row = {
- project_id: projectId,
- session_id: sessionId,
- chunk_index: payload.chunk_index,
- started_at: payload.started_at,
- ended_at: payload.ended_at,
- events_count: payload.events_count,
- is_full_snapshot: payload.is_full_snapshot,
- payload: payload.payload,
- };
- await replayBuffer.add(row);
-}
-
-async function handleGroup(
- payload: IGroupPayload,
- context: TrackContext
-): Promise<void> {
- const { id, type, name, properties = {} } = payload;
- await groupBuffer.add({
- id,
- projectId: context.projectId,
- type,
- name,
- properties,
- });
-}
-
-async function handleAssignGroup(
- payload: IAssignGroupPayload,
- context: TrackContext
-): Promise<void> {
- const profileId = payload.profileId ?? context.deviceId;
- if (!profileId) {
- return;
+// Dissolved into @openpanel/core's ingest module (M8-002): the whole pipeline
+// — context building, the seven payload handlers, device/session resolution
+// and the Kafka produce — moved to ingest.service.ts. These controllers stay
+// (DELEGATE PATTERN) as the live Fastify handlers: they resolve
+// `request.client`, hand the pipeline its transport, and translate the
+// outcome back into the exact replies and `HttpError`s V1 produced.
+//
+// The producer is passed in rather than imported by core: @openpanel/queue
+// imports @openpanel/core for its logger, so core cannot import it back
+// (M8-003 moves the producer into core).
+
+const ingestBuffers = {
+ session: sessionBuffer,
+ replay: replayBuffer,
+ group: groupBuffer,
+};
+
+function replyToOutcome(outcome: TrackOutcome, reply: FastifyReply) {
+ switch (outcome.status) {
+ case 'ok':
+ return reply.status(200).send({
+ deviceId: outcome.deviceId,
+ sessionId: outcome.sessionId,
+ });
+ case 'alias-not-supported':
+ return reply.status(400).send({
+ status: 400,
+ error: 'Bad Request',
+ message: 'Alias is not supported',
+ });
+ case 'invalid-type':
+ return reply.status(400).send({
+ status: 400,
+ error: 'Bad Request',
+ message: 'Invalid type',
+ });
+ case 'missing-project-id':
+ throw new HttpError('Missing projectId', { status: 400 });
+ case 'replay-missing-session-id':
+ throw new HttpError('Session ID is required for replay', { status: 400 });
+ case 'profile-not-found':
+ throw new HttpError('Profile not found', { status: 404 });
+ default:
+ throw new HttpError('Property value is not a number', { status: 400 });
}
- await upsertProfile({
- id: String(profileId),
- projectId: context.projectId,
- isExternal: !!payload.profileId,
- groups: payload.groupIds,
- });
}
export async function handler(
@@ -414,149 +62,47 @@ export async function handler(
}>,
reply: FastifyReply
) {
- const validatedBody = request.body;
-
- // Handle alias (not supported)
- if (validatedBody.type === 'alias') {
- return reply.status(400).send({
- status: 400,
- error: 'Bad Request',
- message: 'Alias is not supported',
- });
- }
-
- // Build request context
- const context = await buildContext(request, validatedBody);
-
- // Dispatch to appropriate handler
- switch (validatedBody.type) {
- case 'track':
- await handleTrack(validatedBody.payload, context);
- break;
- case 'identify':
- await handleIdentify(validatedBody.payload, context);
- break;
- case 'increment':
- await handleIncrement(validatedBody.payload, context);
- break;
- case 'decrement':
- await handleDecrement(validatedBody.payload, context);
- break;
- case 'replay': {
- // BACKCOMPAT(replay-sessionid): TEMPORARY legacy branch, remove when new SDK is fully deployed
- if (!validatedBody.payload.sessionId) {
- await handleReplay(validatedBody.payload, {
- projectId: context.projectId,
- sessionId: context.sessionId,
- });
- break;
- }
-
- await handleReplay(validatedBody.payload, {
- projectId: context.projectId,
- sessionId: validatedBody.payload.sessionId,
- });
- break;
- }
- case 'group':
- await handleGroup(validatedBody.payload, context);
- break;
- case 'assign_group':
- await handleAssignGroup(validatedBody.payload, context);
- break;
- default:
- return reply.status(400).send({
- status: 400,
- error: 'Bad Request',
- message: 'Invalid type',
- });
- }
+ const outcome = await ingestTrack(
+ {
+ projectId: request.client?.projectId,
+ clientIp: request.clientIp,
+ headers: request.headers,
+ clientSecretAuth: request.clientSecretAuth ?? false,
+ timestamp: request.timestamp,
+ body: request.body,
+ },
+ { buffers: ingestBuffers, produceIncomingEvent }
+ );
- reply.status(200).send({
- deviceId: context.deviceId,
- sessionId: context.sessionId,
- });
+ return replyToOutcome(outcome, reply);
}
export async function fetchDeviceId(
request: FastifyRequest,
reply: FastifyReply
) {
- const salts = await getSalts();
- const projectId = request.client?.projectId;
- if (!projectId) {
- return reply.status(400).send('No projectId');
- }
-
- const ip = request.clientIp;
- if (!ip) {
- return reply.status(400).send('Missing ip address');
- }
-
- const ua = request.headers['user-agent'];
- if (!ua) {
- return reply.status(400).send('Missing header: user-agent');
- }
-
- const currentDeviceId = generateDeviceId({
- salt: salts.current,
- origin: projectId,
- ip,
- ua,
- });
- const previousDeviceId = generateDeviceId({
- salt: salts.previous,
- origin: projectId,
- ip,
- ua,
- });
-
- try {
- const [current, previous] = await Promise.all([
- sessionBuffer.getExistingSession({
- projectId,
- deviceId: currentDeviceId,
- }),
- sessionBuffer.getExistingSession({
- projectId,
- deviceId: previousDeviceId,
- }),
- ]);
-
- // Blob has no TTL — only treat the session as "current" if its last
- // event is within the idle window. Otherwise the SDK should ask the
- // server to start a fresh session id on the next event.
- const now = Date.now();
- const isLive = (s: typeof current) =>
- !!s &&
- now - convertClickhouseDateToJs(s.ended_at).getTime() <
- SESSION_TIMEOUT_MS;
-
- if (current && isLive(current)) {
- return reply.status(200).send({
- deviceId: currentDeviceId,
- sessionId: current.id,
- message: 'current session exists for this device id',
- });
- }
+ const identity = await fetchDeviceIdentity(
+ {
+ projectId: request.client?.projectId,
+ clientIp: request.clientIp,
+ headers: request.headers,
+ },
+ { session: sessionBuffer },
+ request.log
+ );
- if (previous && isLive(previous)) {
+ switch (identity.status) {
+ case 'missing-project-id':
+ return reply.status(400).send('No projectId');
+ case 'missing-ip':
+ return reply.status(400).send('Missing ip address');
+ case 'missing-user-agent':
+ return reply.status(400).send('Missing header: user-agent');
+ default:
return reply.status(200).send({
- deviceId: previousDeviceId,
- sessionId: previous.id,
- message: 'previous session exists for this device id',
+ deviceId: identity.deviceId,
+ sessionId: identity.sessionId,
+ message: identity.message,
});
- }
- } catch (error) {
- request.log.error(
- { err: error },
- 'Error getting session end GET /track/device-id'
- );
}
-
- return reply.status(200).send({
- deviceId: currentDeviceId,
- sessionId: '',
- message: 'No session exists for this device id',
- });
}
diff --git a/apps/api/src/hooks/client.hook.ts b/apps/api/src/hooks/client.hook.ts
index b3a9611c8..8a87d14ef 100644
--- a/apps/api/src/hooks/client.hook.ts
+++ b/apps/api/src/hooks/client.hook.ts
@@ -1,15 +1,15 @@
-import { SdkAuthError, validateSdkRequest } from '@/utils/auth';
import type {
DeprecatedPostEventPayload,
ITrackHandlerPayload,
} from '@openpanel/validation';
import type { FastifyReply, FastifyRequest } from 'fastify';
+import { SdkAuthError, validateSdkRequest } from '@/utils/auth';
export async function clientHook(
req: FastifyRequest<{
Body: ITrackHandlerPayload | DeprecatedPostEventPayload;
}>,
- reply: FastifyReply,
+ reply: FastifyReply
) {
try {
const client = await validateSdkRequest(req);
diff --git a/apps/api/src/hooks/duplicate.hook.ts b/apps/api/src/hooks/duplicate.hook.ts
index 154211721..6b0d312eb 100644
--- a/apps/api/src/hooks/duplicate.hook.ts
+++ b/apps/api/src/hooks/duplicate.hook.ts
@@ -1,55 +1,25 @@
+import { isDuplicateIngestRequest } from '@openpanel/core';
import type {
DeprecatedPostEventPayload,
ITrackHandlerPayload,
} from '@openpanel/validation';
import type { FastifyReply, FastifyRequest } from 'fastify';
-import { isDuplicatedEvent } from '@/utils/deduplicate';
+// Delegates to @openpanel/core's ingest module (M8-002).
export async function duplicateHook(
req: FastifyRequest<{
Body: ITrackHandlerPayload | DeprecatedPostEventPayload;
}>,
reply: FastifyReply
) {
- const ip = req.clientIp;
- const origin = req.headers.origin;
- const clientId = req.headers['openpanel-client-id'];
- const body = req?.body;
- const isTrackPayload = getIsTrackPayload(req);
- const isReplay = isTrackPayload && req.body.type === 'replay';
- const shouldCheck = ip && origin && clientId && !isReplay;
- const isDuplicate = shouldCheck
- ? await isDuplicatedEvent({
- ip,
- origin,
- payload: body,
- projectId: clientId as string,
- })
- : false;
+ const isDuplicate = await isDuplicateIngestRequest({
+ method: req.method,
+ clientIp: req.clientIp,
+ headers: req.headers,
+ body: req.body,
+ });
if (isDuplicate) {
return reply.status(200).send('Duplicate event');
}
}
-
-function getIsTrackPayload(
- req: FastifyRequest<{
- Body: ITrackHandlerPayload | DeprecatedPostEventPayload;
- }>
-): req is FastifyRequest<{
- Body: ITrackHandlerPayload;
-}> {
- if (req.method !== 'POST') {
- return false;
- }
-
- if (!req.body) {
- return false;
- }
-
- if (typeof req.body !== 'object' || Array.isArray(req.body)) {
- return false;
- }
-
- return 'type' in req.body;
-}
diff --git a/apps/api/src/hooks/is-bot.hook.test.ts b/apps/api/src/hooks/is-bot.hook.test.ts
deleted file mode 100644
index 4f1bba10f..000000000
--- a/apps/api/src/hooks/is-bot.hook.test.ts
+++ /dev/null
@@ -1,87 +0,0 @@
-/**
- * Tests for isBotHook — the preHandler that filters bot traffic on ingestion.
- *
- * The key behaviour guarded here: requests authenticated with a client secret
- * (server-side SDKs) are never treated as bots, regardless of user agent. Bot
- * detection only applies to public/frontend (origin-authenticated) traffic.
- */
-
-import type { FastifyReply, FastifyRequest } from 'fastify';
-import { beforeEach, describe, expect, it, vi } from 'vitest';
-
-const isBot = vi.fn();
-const createBotEvent = vi.fn();
-
-vi.mock('@/bots', () => ({ isBot: (ua: string) => isBot(ua) }));
-vi.mock('@openpanel/db', () => ({
- createBotEvent: (...args: unknown[]) => createBotEvent(...args),
-}));
-
-const { isBotHook } = await import('./is-bot.hook');
-
-function makeReply() {
- const status = vi.fn();
- const send = vi.fn();
- const reply = { status, send };
- status.mockReturnValue(reply);
- send.mockReturnValue(reply);
- return { reply: reply as unknown as FastifyReply, status, send };
-}
-
-function makeReq(overrides: Partial<FastifyRequest> = {}) {
- return {
- headers: { 'user-agent': 'Googlebot/2.1' },
- client: { projectId: 'proj-1' },
- body: { type: 'track', payload: { properties: { path: '/home' } } },
- ...overrides,
- } as unknown as FastifyRequest;
-}
-
-beforeEach(() => {
- isBot.mockReset();
- createBotEvent.mockReset();
-});
-
-describe('isBotHook', () => {
- it('skips bot detection entirely for client-secret (server-side) traffic', async () => {
- const req = makeReq({ clientSecretAuth: true });
- const { reply, status } = makeReply();
-
- await isBotHook(req as never, reply);
-
- expect(isBot).not.toHaveBeenCalled();
- expect(createBotEvent).not.toHaveBeenCalled();
- expect(status).not.toHaveBeenCalled();
- });
-
- it('records a bot event and responds 202 for public bot traffic', async () => {
- isBot.mockResolvedValue({ name: 'Googlebot', type: 'Search bot' });
- const req = makeReq();
- const { reply, status } = makeReply();
-
- await isBotHook(req as never, reply);
-
- expect(isBot).toHaveBeenCalledWith('Googlebot/2.1');
- expect(createBotEvent).toHaveBeenCalledWith(
- expect.objectContaining({
- name: 'Googlebot',
- type: 'Search bot',
- projectId: 'proj-1',
- path: '/home',
- }),
- );
- expect(status).toHaveBeenCalledWith(202);
- });
-
- it('passes legitimate public traffic through untouched', async () => {
- isBot.mockResolvedValue(null);
- const req = makeReq({ headers: { 'user-agent': 'node' } as never });
- const { reply, status } = makeReply();
-
- const result = await isBotHook(req as never, reply);
-
- expect(createBotEvent).not.toHaveBeenCalled();
- expect(status).not.toHaveBeenCalled();
- expect(result).toBeUndefined();
- });
-});
diff --git a/apps/api/src/hooks/is-bot.hook.ts b/apps/api/src/hooks/is-bot.hook.ts
index a787d71cb..ad1ce7af0 100644
--- a/apps/api/src/hooks/is-bot.hook.ts
+++ b/apps/api/src/hooks/is-bot.hook.ts
@@ -1,57 +1,26 @@
-import { createBotEvent } from '@openpanel/db';
+import { checkIngestBot } from '@openpanel/core';
import type {
DeprecatedPostEventPayload,
ITrackHandlerPayload,
} from '@openpanel/validation';
import type { FastifyReply, FastifyRequest } from 'fastify';
-import { isBot } from '@/bots';
+// Delegates to @openpanel/core's ingest module (M8-002), which owns the
+// client-secret exemption, the bot lookup and the bot-event write.
export async function isBotHook(
req: FastifyRequest<{
Body: ITrackHandlerPayload | DeprecatedPostEventPayload;
}>,
reply: FastifyReply
) {
- // Requests authenticated with a client secret come from server-side SDKs
- // (node, php, go, rust, java, python, …). That auth is a far stronger signal
- // of legitimate first-party traffic than the user agent, so never treat them
- // as bots — bot detection is for public/frontend (origin-authenticated)
- // traffic.
- if (req.clientSecretAuth) {
- return;
- }
-
- const bot = req.headers['user-agent']
- ? await isBot(req.headers['user-agent'])
- : null;
-
- if (bot && req.client?.projectId) {
- if ('type' in req.body && req.body.type === 'track') {
- const path = (req.body.payload.properties?.__path ||
- req.body.payload.properties?.path) as string | undefined;
- if (path) {
- await createBotEvent({
- ...bot,
- projectId: req.client.projectId,
- path: path ?? '',
- createdAt: new Date(),
- });
- }
- // Handle deprecated events (v1)
- } else if ('name' in req.body && 'properties' in req.body) {
- const path = (req.body.properties?.__path || req.body.properties?.path) as
- | string
- | undefined;
- if (path) {
- await createBotEvent({
- ...bot,
- projectId: req.client.projectId,
- path: path ?? '',
- createdAt: new Date(),
- });
- }
- }
+ const bot = await checkIngestBot({
+ headers: req.headers,
+ clientSecretAuth: req.clientSecretAuth ?? false,
+ projectId: req.client?.projectId,
+ body: req.body,
+ });
+ if (bot) {
return reply.status(202).send({ bot });
}
}
diff --git a/apps/api/src/main.ts b/apps/api/src/main.ts
index 181765a9d..bced7c8e8 100644
--- a/apps/api/src/main.ts
+++ b/apps/api/src/main.ts
@@ -21,6 +21,7 @@ import {
queues,
registerBufferMetrics,
} from '@openpanel/core';
+import { produceIncomingEvent } from '@openpanel/queue';
import pino from 'pino';
import { type Config, loadConfig } from './config/env';
@@ -111,6 +112,9 @@ function buildDeps(): AppDeps {
clients: undefined,
buffers: createBuffers(bufferDeps(producers)),
producers,
+ // @openpanel/queue's Kafka producer, injected because core cannot import
+ // it back (M8-002). M8-003 replaces it with core's own.
+ produceIncomingEvent,
logger,
config: { selfHosted: config.SELF_HOSTED },
};
diff --git a/apps/api/src/utils/auth.ts b/apps/api/src/utils/auth.ts
index 3b1a6a74a..8a5f4f37a 100644
--- a/apps/api/src/utils/auth.ts
+++ b/apps/api/src/utils/auth.ts
@@ -1,37 +1,30 @@
-import { verifyPassword } from '@openpanel/core';
+import {
+ type IngestAuthErrorPayload,
+ validateIngestRequest,
+ verifyPassword,
+} from '@openpanel/core';
import type { IServiceClientWithProject } from '@openpanel/db';
import { ClientType, getClientByIdCached } from '@openpanel/db';
-import { getCache } from '@openpanel/redis';
import type {
DeprecatedPostEventPayload,
- IProjectFilterIp,
- IProjectFilterProfileId,
ITrackHandlerPayload,
} from '@openpanel/validation';
import type { FastifyRequest, RawRequestDefaultExpression } from 'fastify';
-import { path } from 'ramda';
-
-const cleanDomain = (domain: string) =>
- domain
- .replace('www.', '')
- .replace(/https?:\/\//, '')
- .replace(/\/$/, '');
+// `validateSdkRequest` dissolved into @openpanel/core's ingest module
+// (M8-002): the rules live in modules/ingest/src/client-auth.ts and this is
+// the DELEGATE, kept so the Fastify hook keeps its signature and its
+// `req.clientSecretAuth` side channel. The other three validators stay here —
+// they belong to the client module, not to ingest, and no task has moved them.
+//
+// The error class stays HERE on purpose. core returns a refusal rather than
+// throwing, so the stack trace is captured in this bundle chunk instead of in
+// the ~11 MB core chunk whose ~20 MB source map `source-map-support` would
+// then parse — ~1.2s, once, on the first failed ingest auth.
export class SdkAuthError extends Error {
- payload: {
- clientId?: string;
- clientSecret?: string;
- origin?: string;
- };
+ payload: IngestAuthErrorPayload;
- constructor(
- message: string,
- payload: {
- clientId?: string;
- clientSecret?: string;
- origin?: string;
- }
- ) {
+ constructor(message: string, payload: IngestAuthErrorPayload) {
super(message);
this.name = 'SdkAuthError';
this.message = message;
@@ -44,135 +37,23 @@ export async function validateSdkRequest(
Body: ITrackHandlerPayload | DeprecatedPostEventPayload;
}>
): Promise<IServiceClientWithProject> {
- const { headers, clientIp } = req;
- const clientIdNew = headers['openpanel-client-id'] as string;
- const clientIdOld = headers['mixan-client-id'] as string;
- const clientSecretNew = headers['openpanel-client-secret'] as string;
- const clientSecretOld = headers['mixan-client-secret'] as string;
- const clientIdFromBody = path<string | undefined>(['clientId'], req.body);
- const clientSecretFromBody = path<string | undefined>(
- ['clientSecret'],
- req.body
- );
- const clientId = clientIdNew || clientIdOld || clientIdFromBody;
- const clientSecret =
- clientSecretNew || clientSecretOld || clientSecretFromBody;
- const origin = headers.origin;
-
- if (clientSecret) {
+ const outcome = await validateIngestRequest({
+ headers: req.headers,
+ clientIp: req.clientIp,
+ body: req.body,
+ });
+
+ // V1 set this at parse time, before validation could refuse — so it is set
+ // on both branches.
+ if (outcome.secretPresented) {
req.clientSecretAuth = true;
}
- const createError = (message: string) =>
- new SdkAuthError(message, {
- clientId,
- clientSecret:
- typeof clientSecret === 'string'
- ? `${clientSecret.slice(0, 5)}...${clientSecret.slice(-5)}`
- : 'none',
- origin,
- });
-
- if (!clientId) {
- throw createError('Ingestion: Missing client id');
- }
-
- if (
- !/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/.test(
- clientId
- )
- ) {
- throw createError('Ingestion: Client ID must be a valid UUIDv4');
- }
-
- const client = await getClientByIdCached(clientId);
-
- if (!client) {
- throw createError('Ingestion: Invalid client id');
- }
-
- if (!client.project) {
- throw createError('Ingestion: Client has no project');
- }
-
- // Filter out blocked IPs
- const ipFilter = client.project.filters.filter(
- (filter): filter is IProjectFilterIp => filter.type === 'ip'
- );
- if (ipFilter.some((filter) => filter.ip === clientIp)) {
- throw createError('Ingestion: IP address is blocked by project filter');
- }
-
- // Filter out blocked profile ids
- const profileFilter = client.project.filters.filter(
- (filter): filter is IProjectFilterProfileId => filter.type === 'profile_id'
- );
- const profileId =
- path<string | undefined>(['payload', 'profileId'], req.body) || // Track handler
- path<string | undefined>(['profileId'], req.body); // Event handler
-
- if (profileFilter.some((filter) => filter.profileId === profileId)) {
- throw createError('Ingestion: Profile id is blocked by project filter');
- }
-
- const revenue =
- path(['payload', 'properties', '__revenue'], req.body) ??
- path(['properties', '__revenue'], req.body);
-
- // Only allow revenue tracking if it was sent with a client secret
- // or if the project has allowUnsafeRevenueTracking enabled
- if (
- !(client.project.allowUnsafeRevenueTracking || clientSecret) &&
- typeof revenue !== 'undefined'
- ) {
- throw createError(
- 'Ingestion: Revenue tracking is not allowed without a client secret'
- );
- }
-
- if (client.ignoreCorsAndSecret) {
- return client;
- }
-
- if (client.project.cors) {
- const domainAllowed = client.project.cors.find((domain) => {
- const cleanedDomain = cleanDomain(domain);
- // support wildcard domains `*.foo.com`
- if (cleanedDomain.includes('*')) {
- const regex = new RegExp(
- `${cleanedDomain.replaceAll('.', '\\.').replaceAll('*', '.+?')}`
- );
-
- return regex.test(origin || '');
- }
-
- if (cleanedDomain === cleanDomain(origin || '')) {
- return true;
- }
- });
-
- if (domainAllowed) {
- return client;
- }
-
- if (client.project.cors.includes('*') && origin) {
- return client;
- }
- }
-
- if (client.secret && clientSecret) {
- const isVerified = await getCache(
- `client:auth:${clientId}:${Buffer.from(clientSecret).toString('base64')}`,
- 60 * 5,
- async () => await verifyPassword(clientSecret, client.secret!),
- true
- );
- if (isVerified) {
- return client;
- }
+ if (!outcome.ok) {
+ throw new SdkAuthError(outcome.message, outcome.payload);
}
- throw createError('Ingestion: Invalid cors or secret');
+ return outcome.client;
}
export async function validateExportRequest(
diff --git a/apps/api/src/utils/deduplicate.ts b/apps/api/src/utils/deduplicate.ts
deleted file mode 100644
index d45ac9ba5..000000000
--- a/apps/api/src/utils/deduplicate.ts
+++ /dev/null
@@ -1,34 +0,0 @@
-import { getLock } from '@openpanel/redis';
-import fastJsonStableHash from 'fast-json-stable-hash';
-
-export async function isDuplicatedEvent({
- ip,
- origin,
- payload,
- projectId,
-}: {
- ip: string;
- origin: string;
- payload: Record<string, any>;
- projectId: string;
-}) {
- const locked = await getLock(
- `fastify:deduplicate:${fastJsonStableHash.hash(
- {
- ...payload,
- ip,
- origin,
- projectId,
- },
- 'md5',
- )}`,
- '1',
- 100,
- );
-
- if (locked) {
- return false;
- }
-
- return true;
-}
diff --git a/apps/api/src/utils/ids.test.ts b/apps/api/src/utils/ids.test.ts
deleted file mode 100644
index e6d8e952c..000000000
--- a/apps/api/src/utils/ids.test.ts
+++ /dev/null
@@ -1,110 +0,0 @@
-/**
- * Tests for getDeviceId's session resolution against the device-keyed session
- * store. The id is reused from the live blob only while it's within the idle
- * window; a lingering (past-window) blob must NOT be reused — see the guard in
- * ids.ts. Override device ids resolve through the same path with a single read.
- *
- * We spy on the real `sessionBuffer.getExistingSession` singleton (no Redis),
- * and use the real `formatClickhouseDate` so `ended_at` round-trips exactly.
- */
-
-import type { IClickhouseSession } from '@openpanel/db';
-import { formatClickhouseDate, sessionBuffer } from '@openpanel/db';
-import { afterEach, describe, expect, it, vi } from 'vitest';
-import { getDeviceId } from './ids';
-
-const NOW = new Date('2026-06-08T12:00:00.000Z').getTime();
-const MINUTE = 60 * 1000;
-const SALTS = { current: 'salt-current', previous: 'salt-previous' };
-const BASE = {
- projectId: 'proj-1',
- ip: '1.2.3.4',
- ua: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/148.0.0.0',
- salts: SALTS,
- eventTimeMs: NOW,
-};
-
-// withinIdleWindow only reads `id` + `ended_at`; the rest is irrelevant here.
-const fakeSession = (id: string, endedAtMs: number): IClickhouseSession =>
- ({
- id,
- ended_at: formatClickhouseDate(new Date(endedAtMs)),
- }) as unknown as IClickhouseSession;
-
-afterEach(() => {
- vi.restoreAllMocks();
-});
-
-describe('getDeviceId — session resolution', () => {
- it('mints a deterministic, stable id when no session exists', async () => {
- const spy = vi
- .spyOn(sessionBuffer, 'getExistingSession')
- .mockResolvedValue(null);
-
- const a = await getDeviceId({ ...BASE, overrideDeviceId: 'cookie-abc' });
- const b = await getDeviceId({ ...BASE, overrideDeviceId: 'cookie-abc' });
-
- expect(a.sessionId).toBeTruthy();
- expect(b.sessionId).toBe(a.sessionId); // same window → same id
- expect(a.deviceId).toBe('cookie-abc');
- expect(spy).toHaveBeenCalled();
- });
-
- it('reuses the live session id when within the idle window', async () => {
- vi.spyOn(sessionBuffer, 'getExistingSession').mockResolvedValue(
- fakeSession('sess-live', NOW - MINUTE)
- );
-
- const result = await getDeviceId({
- ...BASE,
- overrideDeviceId: 'cookie-abc',
- });
-
- expect(result).toEqual({ deviceId: 'cookie-abc', sessionId: 'sess-live' });
- });
-
- it('does NOT reuse a session that has lingered past the idle window', async () => {
- vi.spyOn(sessionBuffer, 'getExistingSession').mockResolvedValue(
- // ended 31 min ago — past the 30 min timeout; the reaper just hasn't
- // closed it yet (blobs have no TTL).
- fakeSession('sess-stale', NOW - 31 * MINUTE)
- );
-
- const result = await getDeviceId({
- ...BASE,
- overrideDeviceId: 'cookie-abc',
- });
-
- expect(result.deviceId).toBe('cookie-abc');
- expect(result.sessionId).toBeTruthy();
- expect(result.sessionId).not.toBe('sess-stale'); // a fresh id, not the stale one
- });
-
- it('reads the store once for an override (no redundant previous lookup)', async () => {
- const spy = vi
- .spyOn(sessionBuffer, 'getExistingSession')
- .mockResolvedValue(null);
-
- await getDeviceId({ ...BASE, overrideDeviceId: 'cookie-abc' });
-
- expect(spy).toHaveBeenCalledTimes(1);
- expect(spy).toHaveBeenCalledWith({
- projectId: 'proj-1',
- deviceId: 'cookie-abc',
- });
- });
-
- it('checks both current and previous salt windows for internal ids', async () => {
- const spy = vi
- .spyOn(sessionBuffer, 'getExistingSession')
- .mockResolvedValue(null);
-
- await getDeviceId(BASE); // no override → IP+UA hashing
-
- expect(spy).toHaveBeenCalledTimes(2);
- const deviceIds = spy.mock.calls.map((c) =>
- 'deviceId' in c[0] ? c[0].deviceId : ''
- );
- expect(new Set(deviceIds).size).toBe(2); // distinct current/previous hashes
- });
-});
diff --git a/packages/constants/index.ts b/packages/constants/index.ts
index be2dce2bf..f52900566 100644
--- a/packages/constants/index.ts
+++ b/packages/constants/index.ts
@@ -1,4 +1,7 @@
-export const RESERVED_EVENT_NAMES = ['session_start', 'session_end'] as const;
+// Moved into @openpanel/core's ingest module (M8-002, ADR-008's module map:
+// ingest owns "C" — the reserved names are enforced by the same zod refine
+// that lives there). Re-exported here for existing @openpanel/constants
+// importers — same shape as `ProjectTypeNames` below.
export type { EmailCategory } from '@openpanel/core/modules/email/email.constants';
// Moved into @openpanel/core's email module (M6-004, ADR-008's module map:
@@ -7,6 +10,7 @@ export type { EmailCategory } from '@openpanel/core/modules/email/email.constant
// email-preferences pages) — same shape as `ProjectTypeNames`'s re-export
// since M6-002. Keys must match the template `category` in @openpanel/email.
export { emailCategories } from '@openpanel/core/modules/email/email.constants';
+export { RESERVED_EVENT_NAMES } from '@openpanel/core/modules/ingest/ingest.constants';
// Moved into @openpanel/core's project module (M6-002, ADR-008's module
// map: project owns "C"). Re-exported here for existing
// @openpanel/constants importers — same shape as
diff --git a/packages/core/package.json b/packages/core/package.json
index 2098fbef4..fa3a1d6a4 100644
--- a/packages/core/package.json
+++ b/packages/core/package.json
@@ -13,6 +13,7 @@
"test": "bun test --isolate",
"codegen": "bun scripts/download-geo.ts && bun scripts/get-datacenter-asns.ts",
"gen:asns": "bun scripts/get-datacenter-asns.ts",
+ "gen:bots": "bun scripts/get-bots.ts && biome format --write ./src/modules/ingest/src/bots/bots.ts",
"gen:referrers": "bun scripts/get-referrers.ts && biome format --write ./src/shared/referrers/index.ts"
},
"dependencies": {
@@ -48,7 +49,9 @@
"date-fns": "^3.3.1",
"elysia": "1.4.30",
"fast-deep-equal": "^3.1.3",
+ "fast-json-stable-hash": "^1.0.3",
"ipaddr.js": "^2.5.0",
+ "js-yaml": "^4.1.0",
"lru-cache": "^11.2.4",
"mathjs": "^12.3.2",
"nanoid": "^5.1.6",
@@ -66,15 +69,18 @@
"undici": "^8.10.0",
"unzipper": "^0.12.3",
"url-metadata": "^5.4.1",
+ "uuid": "^9.0.1",
"zod": "catalog:"
},
"devDependencies": {
"@clickhouse/client": "^1.18.5",
"@openpanel/tsconfig": "workspace:*",
+ "@types/js-yaml": "^4.0.9",
"@types/qrcode": "^1.5.6",
"@types/ramda": "^0.31.1",
"@types/sqlstring": "^2.3.2",
"@types/unzipper": "^0.10.10",
+ "@types/uuid": "^10.0.0",
"bun-types": "1.4.0",
"tar": "^7.4.3",
"typescript": "catalog:"
diff --git a/apps/api/scripts/get-bots.ts b/packages/core/scripts/get-bots.ts
similarity index 93%
rename from apps/api/scripts/get-bots.ts
rename to packages/core/scripts/get-bots.ts
index 5d8281339..a36eeb4e8 100644
--- a/apps/api/scripts/get-bots.ts
+++ b/packages/core/scripts/get-bots.ts
@@ -1,14 +1,14 @@
import fs from 'node:fs';
-import path from 'node:path';
-import { dirname } from 'node:path';
+import path, { dirname } from 'node:path';
import { fileURLToPath } from 'node:url';
const __filename = fileURLToPath(import.meta.url);
const __dirname = dirname(__filename);
+
import yaml from 'js-yaml';
// Regex special characters that indicate we need actual regex
-const regexSpecialChars = /[|^$.*+?(){}\[\]\\]/;
+const regexSpecialChars = /[|^$.*+?(){}[\]\\]/;
// User agents that must NEVER be treated as bots, even though the upstream
// device-detector list flags them. Its "Generic Bot" pattern lumps bare
@@ -40,7 +40,7 @@ export function stripAllowlistedTokens(regex: string): string {
const trailingPipe = match.endsWith('|') ? '|' : '';
return `^(?:${kept.join('|')})$${trailingPipe}`;
- },
+ }
);
}
@@ -63,14 +63,14 @@ async function main() {
// Get document, or throw exception on error
try {
const data = await fetch(
- 'https://raw.githubusercontent.com/matomo-org/device-detector/master/regexes/bots.yml',
+ 'https://raw.githubusercontent.com/matomo-org/device-detector/master/regexes/bots.yml'
).then((res) => res.text());
const parsedData = yaml.load(data) as any[];
const transformedBots = transformBots(parsedData);
fs.writeFileSync(
- path.resolve(__dirname, '../src/bots/bots.ts'),
+ path.resolve(__dirname, '../src/modules/ingest/src/bots/bots.ts'),
[
'// This file is generated by the script get-bots.ts',
'',
@@ -80,11 +80,11 @@ async function main() {
'export default bots;',
'',
].join('\n'),
- 'utf-8',
+ 'utf-8'
);
console.log(
- `✅ Generated bots.ts with ${transformedBots.length} bot entries`,
+ `✅ Generated bots.ts with ${transformedBots.length} bot entries`
);
const regexCount = transformedBots.filter((b) => 'regex' in b).length;
const includesCount = transformedBots.filter((b) => 'includes' in b).length;
diff --git a/packages/core/src/context.test.ts b/packages/core/src/context.test.ts
index 628f390dc..928b7b8db 100644
--- a/packages/core/src/context.test.ts
+++ b/packages/core/src/context.test.ts
@@ -68,6 +68,7 @@ function stubDeps(): {
return surface;
},
},
+ produceIncomingEvent: () => Promise.resolve(),
logger: stubLogger(),
config: { selfHosted: true },
},
diff --git a/packages/core/src/context.ts b/packages/core/src/context.ts
index c738f9632..1f1f7f4c5 100644
--- a/packages/core/src/context.ts
+++ b/packages/core/src/context.ts
@@ -8,6 +8,7 @@ import type { SessionValidationResult } from '@openpanel/db';
import type { Buffers } from './buffers/create-buffers';
import type { QueueProducerHandle, QueueProducers } from './jobs.registry';
import type { Logger } from './logger';
+import type { IncomingEventProducer } from './modules/ingest/src/incoming-event';
import { createServices, type Services } from './services';
import type { CookieJar, CookieOptions } from './shared/cookie';
@@ -48,6 +49,13 @@ export interface AppDeps {
clients: ServiceClients;
buffers: Buffers;
producers: QueueProducerHandle;
+ /**
+ * The events transport. `produceIncomingEvent` still lives in
+ * @openpanel/queue, which imports core for its logger, so core cannot
+ * import it back (M8-002); main.ts supplies it and M8-003 replaces it with
+ * core's own Kafka producer.
+ */
+ produceIncomingEvent: IncomingEventProducer;
logger: Logger;
config: RuntimeFlags;
}
diff --git a/packages/core/src/http/auth.ts b/packages/core/src/http/auth.ts
index ff1297d85..9eb21cfa8 100644
--- a/packages/core/src/http/auth.ts
+++ b/packages/core/src/http/auth.ts
@@ -40,6 +40,7 @@
import { Elysia } from 'elysia';
import type { AppDeps } from '../context';
+import { toIngestHeaders } from '../modules/ingest/src/headers';
import { authenticateClient, type ClientAuthOptions } from './client-auth';
import { requestContext } from './context';
@@ -64,8 +65,13 @@ export function authMacros(deps: AppDeps) {
},
},
clientAuth: (options: ClientAuthOptions) => ({
- async resolve({ ctx, status }) {
- const client = await authenticateClient(deps, ctx.headers, options);
+ async resolve({ body, ctx, status }) {
+ const client = await authenticateClient(
+ deps,
+ toIngestHeaders(ctx.headers),
+ options,
+ { ip: ctx.ip, body }
+ );
if (!client) {
return status(401, INVALID_CLIENT_MESSAGE);
}
diff --git a/packages/core/src/http/client-auth.ts b/packages/core/src/http/client-auth.ts
index dbf346916..60e0a7c73 100644
--- a/packages/core/src/http/client-auth.ts
+++ b/packages/core/src/http/client-auth.ts
@@ -5,14 +5,19 @@
// and the MCP copy — that differ only by the accepted `ClientType` set and by
// the ingest extension. They collapse into this one function (ADR-011 A-i).
//
-// P8 fills the body in, moving it onto `services.client.authenticate` and
-// porting the ingest branch verbatim: the `ignoreCorsAndSecret` short-circuit,
-// the unanchored wildcard origin regex, CORS-OR-secret ordering, the
-// ip/profile_id project filters, the `__revenue` gate, the body-field
-// credential fallback and the 5-minute verify cache. The signature and the
-// macro over it are what P2 fixes.
+// M8-002 filled in the INGEST branch — `modules/ingest/src/client-auth.ts`
+// holds V1's `validateSdkRequest` verbatim, and this file adapts it onto the
+// principal. The `allow`-list tiers (`validateExportRequest`,
+// `validateImportRequest`, `validateManageRequest`, MCP's `token: 'basic'`)
+// are NOT filled in yet: they belong to the client module, not to ingest, and
+// stay the stub that answers 401 — the same NAMED GAP every non-ingest
+// `publicApiRoutes` module's header already records.
import type { AppDeps } from '../context';
+import {
+ type IngestHeaders,
+ validateIngestRequest,
+} from '../modules/ingest/src/client-auth';
/** Prisma's `ClientType` enum, by value. Moves to client.constants.ts in P7. */
export type ClientType = 'read' | 'write' | 'root';
@@ -42,10 +47,39 @@ export interface ClientAuthOptions {
token?: 'basic';
}
-export function authenticateClient(
+/** What the ingest tier needs beyond the headers: V1 reads the attribution ip
+ * and the body (credential fallback, profile filter, `__revenue` gate). */
+export interface ClientAuthRequest {
+ ip: string;
+ body: unknown;
+}
+
+export async function authenticateClient(
_deps: AppDeps,
- _headers: Headers,
- _options: ClientAuthOptions
+ headers: IngestHeaders,
+ options: ClientAuthOptions,
+ request?: ClientAuthRequest
): Promise<AuthenticatedClient | null> {
- return Promise.resolve(null);
+ if (!options.ingest) {
+ return null;
+ }
+
+ const outcome = await validateIngestRequest({
+ headers,
+ clientIp: request?.ip,
+ body: request?.body,
+ });
+
+ if (!outcome.ok) {
+ return null;
+ }
+
+ const { client } = outcome;
+ return {
+ id: client.id,
+ projectId: client.projectId,
+ organizationId: client.organizationId,
+ type: client.type,
+ secretPresented: outcome.secretPresented,
+ };
}
diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts
index edc3aeed9..79253497d 100644
--- a/packages/core/src/index.ts
+++ b/packages/core/src/index.ts
@@ -1115,6 +1115,42 @@ export {
runImportJob,
updateImportStatus,
} from './modules/import/import.service';
+// The ingestion pipeline (M8-002). apps/api's /track controller, its three
+// route hooks and the legacy /event controller are thin delegates over these
+// — the same functions core's own `ingestRoutes` calls.
+export type {
+ BotMatch,
+ BotSuspicion,
+ DeviceIdentity,
+ IncomingEventPayload,
+ IncomingEventProducer,
+ IngestAuthErrorPayload,
+ IngestAuthOutcome,
+ IngestBuffers,
+ IngestHeaders,
+ IngestService,
+ IngestTransport,
+ TrackOutcome,
+ TrackRequest,
+} from './modules/ingest/ingest.service';
+export {
+ applyBotSuspicion,
+ checkIngestBot,
+ detectBot,
+ fetchDeviceIdentity,
+ getDeviceId,
+ getOverrideDeviceId,
+ getStringHeaders,
+ getTimestamp,
+ handleReplay,
+ ingestTrack,
+ isBot,
+ isDuplicatedEvent,
+ isDuplicateIngestRequest,
+ stripBotProperties,
+ summarizeBotSignals,
+ validateIngestRequest,
+} from './modules/ingest/ingest.service';
// Dissolved from @openpanel/db's services/insights* + referrer-spikes.service
// (M5-001) — apps/worker's insight job files and packages/trpc's insight
// router call these directly, the same way V1 reaches every other dissolved
diff --git a/packages/core/src/jobs/workers.test.ts b/packages/core/src/jobs/workers.test.ts
index 87274d36e..4abe9c4c7 100644
--- a/packages/core/src/jobs/workers.test.ts
+++ b/packages/core/src/jobs/workers.test.ts
@@ -30,6 +30,7 @@ function stubDeps(): AppDeps {
clients: {},
buffers: {} as Buffers,
producers: createRecordingProducers(queues),
+ produceIncomingEvent: () => Promise.resolve(),
logger: stubLogger(),
config: { selfHosted: true },
};
diff --git a/packages/core/src/modules/cohort/cohort.jobs.test.ts b/packages/core/src/modules/cohort/cohort.jobs.test.ts
index 676c3e2ca..c4aa5ec2d 100644
--- a/packages/core/src/modules/cohort/cohort.jobs.test.ts
+++ b/packages/core/src/modules/cohort/cohort.jobs.test.ts
@@ -35,6 +35,7 @@ function stubJobCtx(cohort: Partial<CohortService>, jobName = 'test'): JobCtx {
insight: {} as Services['insight'],
gsc: {} as Services['gsc'],
import: {} as Services['import'],
+ ingest: {} as Services['ingest'],
cohort: cohort as CohortService,
organization: {} as Services['organization'],
onboarding: {} as Services['onboarding'],
diff --git a/packages/core/src/modules/gsc/gsc.jobs.test.ts b/packages/core/src/modules/gsc/gsc.jobs.test.ts
index cd33a4488..13f14f933 100644
--- a/packages/core/src/modules/gsc/gsc.jobs.test.ts
+++ b/packages/core/src/modules/gsc/gsc.jobs.test.ts
@@ -31,6 +31,7 @@ function stubJobCtx(gsc: Partial<GscService>, jobName = 'test'): JobCtx {
insight: {} as Services['insight'],
cohort: {} as Services['cohort'],
import: {} as Services['import'],
+ ingest: {} as Services['ingest'],
gsc: gsc as GscService,
organization: {} as Services['organization'],
onboarding: {} as Services['onboarding'],
@@ -132,6 +133,7 @@ test('gscSync fans out one gscProjectSync enqueue per connected project', async
insight: {} as Services['insight'],
cohort: {} as Services['cohort'],
import: {} as Services['import'],
+ ingest: {} as Services['ingest'],
gsc: {
listConnectionsForSync: async () => [
{ projectId: 'p1' },
diff --git a/packages/core/src/modules/import/import.jobs.test.ts b/packages/core/src/modules/import/import.jobs.test.ts
index da8feca35..2fa70b13b 100644
--- a/packages/core/src/modules/import/import.jobs.test.ts
+++ b/packages/core/src/modules/import/import.jobs.test.ts
@@ -30,6 +30,7 @@ function stubJobCtx(importService: Partial<ImportService>): JobCtx {
notification: {} as Services['notification'],
insight: {} as Services['insight'],
cohort: {} as Services['cohort'],
+ ingest: {} as Services['ingest'],
gsc: {} as Services['gsc'],
import: importService as ImportService,
organization: {} as Services['organization'],
diff --git a/packages/core/src/modules/ingest/ingest.constants.ts b/packages/core/src/modules/ingest/ingest.constants.ts
new file mode 100644
index 000000000..17b5293e4
--- /dev/null
+++ b/packages/core/src/modules/ingest/ingest.constants.ts
@@ -0,0 +1,398 @@
+// The ingestion wire contract (M8-002, ADR-008's module map: ingest owns "C").
+// Moved from packages/validation/src/track.validation.ts,
+// packages/validation/src/event-blocklist.ts and @openpanel/constants'
+// RESERVED_EVENT_NAMES — all three enforce the same thing (what a caller may
+// name an event and what a track body may contain), so they are one file here.
+// packages/validation and packages/constants re-export this; the SDK deep-import
+// retarget is P11.
+//
+// Isomorphic by rule: zod, another *.constants.ts, or nothing.
+
+import { z } from 'zod';
+
+/** Event names the ingestion pipeline mints itself; a caller may not send them. */
+export const RESERVED_EVENT_NAMES = ['session_start', 'session_end'] as const;
+
+const MAX_EVENT_LENGTH = 80;
+
+// Substrings that indicate attack/spam payloads
+const BLOCKED_SUBSTRINGS = [
+ // === Security Scanner Domains ===
+ 'oastify.com',
+ 'burpcollaborator',
+ 'interact.sh',
+ 'oast.me',
+
+ // === SQL Injection ===
+ 'pg_sleep',
+ 'waitfor delay',
+ 'xp_dirtree',
+ 'load_file(',
+ 'extractvalue(',
+ 'dbms_pipe.receive_message',
+ 'union select',
+
+ // === Command Injection ===
+ 'nslookup ',
+ '/bin/sleep',
+ '/bin/bash',
+ 'cmd.exe',
+ 'wget+http',
+ 'wget http',
+ 'chmod+777',
+ 'chmod 777',
+
+ // === Java/Code Execution ===
+ 'processbuilder',
+ 'runtime.getruntime',
+ 'java.lang.processbuilder',
+ 'eval-stdin.php',
+
+ // === Path Traversal ===
+ '../',
+ '..\\',
+ '%2e%2e',
+ '%u002e%u002e',
+ '/etc/passwd',
+ '/etc/shadow',
+ 'win.ini',
+ 'system.ini',
+
+ // === Template/SSTI Injection ===
+ '${',
+ '%{',
+
+ // === XXE / XML Attacks ===
+ '<!doctype',
+ '<!entity',
+ '<xi:include',
+ 'xsi:schemalocation',
+
+ // === SMTP Header Injection ===
+ '\r\n',
+ 'bcc:',
+
+ // === Common File Scanning (paths as events) ===
+ 'phpinfo.php',
+ 'wp-config.php',
+ '.git/config',
+ '.env.backup',
+ '.env.bak',
+ '/vendor/phpunit/',
+
+ // === Malware/Botnet Indicators ===
+ 'mozi.m',
+ '/setup.cgi?',
+ '/cgi-bin/',
+
+ // === Ruby Object Inspection Leaks ===
+ '#<article:0x',
+ '#<video:0x',
+ '#<brand:0x',
+
+ // === SQL Injection Patterns ===
+ 'exec master.dbo',
+ 'declare @',
+ "' and '",
+ "' or '",
+ "') or ",
+ "')and ",
+];
+
+// Patterns that indicate the "event" is actually a URL path being scanned
+const PATH_SCAN_PATTERNS = [
+ /^\/[a-z_-]+\.(php|env|yml|yaml|json|xml|config|ini|bak|sql|log)/i,
+ /^\/\.[a-z]/i, // Hidden files like /.env, /.git
+ /^\/(wp-|wordpress)/i, // WordPress scanning
+ /^\/phpmyadmin/i,
+ /^\/.+\.php$/i, // Any .php path
+];
+
+/**
+ * Check if an event name should be blocked
+ * @param name - The event name to check
+ * @returns true if the event name should be blocked, false otherwise
+ */
+export function isBlockedEventName(name: string): boolean {
+ // Length check - attack payloads are often very long
+ if (name.length > MAX_EVENT_LENGTH) {
+ return true;
+ }
+
+ // Contains newlines (always suspicious for event names)
+ if (name.includes('\n') || name.includes('\r')) {
+ return true;
+ }
+
+ // Substring blocklist (case-insensitive)
+ const lower = name.toLowerCase();
+ if (BLOCKED_SUBSTRINGS.some((blocked) => lower.includes(blocked))) {
+ return true;
+ }
+
+ // Path scanning patterns
+ if (PATH_SCAN_PATTERNS.some((pattern) => pattern.test(name))) {
+ return true;
+ }
+
+ return false;
+}
+
+// ----- Hand-written types (source of truth) -----
+//
+// These interfaces are duplicated in code that ships in our SDK type
+// declarations. We hand-write them (instead of using `z.infer<…>`) so:
+// 1. Generated `.d.ts` files for the SDKs are readable plain TypeScript
+// with no `import type { ITrackPayload } from '@openpanel/validation'`
+// lines (the package isn't published).
+// 2. The interfaces stay clean — no zod internals leaking through.
+// Each schema below has `satisfies z.ZodType<…>` attached so a drift
+// between the interface and the schema fails to compile.
+
+export type IProfileId = string | number;
+
+export interface IGroupPayload {
+ id: string;
+ type: string;
+ name: string;
+ properties?: Record<string, unknown>;
+}
+
+export interface IAssignGroupPayload {
+ groupIds: string[];
+ profileId?: IProfileId;
+}
+
+export interface ITrackPayload {
+ name: string;
+ properties?: Record<string, unknown>;
+ profileId?: IProfileId;
+ groups?: string[] | null;
+}
+
+export interface IIdentifyPayload {
+ profileId: IProfileId;
+ firstName?: string;
+ lastName?: string;
+ email?: string;
+ avatar?: string;
+ properties?: Record<string, unknown>;
+}
+
+export interface IIncrementPayload {
+ profileId: IProfileId;
+ property: string;
+ value?: number;
+}
+
+export interface IDecrementPayload {
+ profileId: IProfileId;
+ property: string;
+ value?: number;
+}
+
+export interface IAliasPayload {
+ profileId: IProfileId;
+ alias: string;
+}
+
+export interface IReplayPayload {
+ chunk_index: number;
+ events_count: number;
+ is_full_snapshot: boolean;
+ started_at: string;
+ ended_at: string;
+ payload: string;
+ // Server-issued session id (from a prior /track response) the SDK echoes back,
+ // so the chunk is filed under the right session without device resolution.
+ sessionId?: string;
+}
+
+export type ITrackHandlerPayload =
+ | { type: 'track'; payload: ITrackPayload }
+ | { type: 'identify'; payload: IIdentifyPayload }
+ | { type: 'increment'; payload: IIncrementPayload }
+ | { type: 'decrement'; payload: IDecrementPayload }
+ | { type: 'alias'; payload: IAliasPayload }
+ | { type: 'replay'; payload: IReplayPayload }
+ | { type: 'group'; payload: IGroupPayload }
+ | { type: 'assign_group'; payload: IAssignGroupPayload };
+
+// ----- Schemas (each `satisfies` its hand-written interface) -----
+
+export const zProfileId = z.union([
+ z.string().min(1),
+ z.number(),
+]) satisfies z.ZodType<IProfileId>;
+
+export const zGroupPayload = z.object({
+ id: z.string().min(1),
+ type: z.string().min(1),
+ name: z.string().min(1),
+ properties: z.record(z.string(), z.unknown()).optional(),
+}) satisfies z.ZodType<IGroupPayload>;
+
+export const zAssignGroupPayload = z.object({
+ groupIds: z.array(z.string().min(1)),
+ profileId: zProfileId.optional(),
+}) satisfies z.ZodType<IAssignGroupPayload>;
+
+export const zTrackPayload = z
+ .object({
+ name: z.string().min(1),
+ properties: z.record(z.string(), z.unknown()).optional(),
+ profileId: zProfileId.optional(),
+ groups: z.array(z.string().min(1)).nullish(),
+ })
+ .refine(
+ (data) => !(RESERVED_EVENT_NAMES as readonly string[]).includes(data.name),
+ {
+ message: `Event name cannot be one of the reserved names: ${RESERVED_EVENT_NAMES.join(', ')}`,
+ path: ['name'],
+ }
+ )
+ .refine((data) => !isBlockedEventName(data.name), {
+ message: 'Event name contains blocked content',
+ path: ['name'],
+ })
+ .refine(
+ (data) => {
+ if (data.name !== 'revenue') {
+ return true;
+ }
+ const revenue = data.properties?.__revenue;
+ if (revenue === undefined || revenue === null) {
+ return true;
+ }
+ const isInt = Number.isInteger(revenue);
+ if (isInt && Number(revenue) < 0) {
+ return false;
+ }
+ return isInt;
+ },
+ {
+ message: '__revenue must be an integer (no floats or strings)',
+ path: ['properties', '__revenue'],
+ }
+ ) satisfies z.ZodType<ITrackPayload>;
+
+export const zIdentifyPayload = z.object({
+ profileId: zProfileId,
+ firstName: z.string().optional(),
+ lastName: z.string().optional(),
+ email: z.string().email().optional(),
+ avatar: z.string().url().optional(),
+ properties: z.record(z.string(), z.unknown()).optional(),
+}) satisfies z.ZodType<IIdentifyPayload>;
+
+export const zIncrementPayload = z.object({
+ profileId: zProfileId,
+ property: z.string().min(1),
+ value: z.number().positive().optional(),
+}) satisfies z.ZodType<IIncrementPayload>;
+
+export const zDecrementPayload = z.object({
+ profileId: zProfileId,
+ property: z.string().min(1),
+ value: z.number().positive().optional(),
+}) satisfies z.ZodType<IDecrementPayload>;
+
+export const zAliasPayload = z.object({
+ profileId: zProfileId,
+ alias: z.string().min(1),
+}) satisfies z.ZodType<IAliasPayload>;
+
+export const zReplayPayload = z.object({
+ chunk_index: z.number().int().min(0).max(65_535),
+ events_count: z.number().int().min(1),
+ is_full_snapshot: z.boolean(),
+ started_at: z.string().datetime(),
+ ended_at: z.string().datetime(),
+ payload: z.string().max(1_048_576 * 2), // 2MB max
+ sessionId: z.string().max(64).optional(),
+}) satisfies z.ZodType<IReplayPayload>;
+
+export const zTrackHandlerPayload = z.discriminatedUnion('type', [
+ z
+ .object({
+ type: z.enum(['track']),
+ payload: zTrackPayload,
+ })
+ .meta({ title: 'Track' }),
+ z
+ .object({
+ type: z.enum(['identify']),
+ payload: zIdentifyPayload,
+ })
+ .meta({ title: 'Identify' }),
+ z
+ .object({
+ type: z.enum(['increment']),
+ payload: zIncrementPayload,
+ })
+ .meta({ title: 'Increment' }),
+ z
+ .object({
+ type: z.enum(['decrement']),
+ payload: zDecrementPayload,
+ })
+ .meta({ title: 'Decrement' }),
+ z
+ .object({
+ type: z.enum(['alias']),
+ payload: zAliasPayload,
+ })
+ .meta({ title: 'Alias' }),
+ z
+ .object({
+ type: z.enum(['replay']),
+ payload: zReplayPayload,
+ })
+ .meta({ title: 'Replay' }),
+ z
+ .object({
+ type: z.enum(['group']),
+ payload: zGroupPayload,
+ })
+ .meta({ title: 'Group' }),
+ z
+ .object({
+ type: z.enum(['assign_group']),
+ payload: zAssignGroupPayload,
+ })
+ .meta({ title: 'Assign Group' }),
+]) satisfies z.ZodType<ITrackHandlerPayload>;
+
+// Deprecated types for beta version of the SDKs
+
+export interface DeprecatedOpenpanelEventOptions {
+ profileId?: string;
+}
+
+export interface DeprecatedPostEventPayload {
+ name: string;
+ timestamp: string;
+ profileId?: string;
+ properties?: Record<string, unknown> & DeprecatedOpenpanelEventOptions;
+}
+
+export interface DeprecatedUpdateProfilePayload {
+ profileId: string;
+ firstName?: string;
+ lastName?: string;
+ email?: string;
+ avatar?: string;
+ properties?: Record<string, unknown>;
+}
+
+export interface DeprecatedIncrementProfilePayload {
+ profileId: string;
+ property: string;
+ value: number;
+}
+
+export interface DeprecatedDecrementProfilePayload {
+ profileId?: string;
+ property: string;
+ value: number;
+}
diff --git a/packages/core/src/modules/ingest/ingest.routes.ts b/packages/core/src/modules/ingest/ingest.routes.ts
new file mode 100644
index 000000000..ef004a509
--- /dev/null
+++ b/packages/core/src/modules/ingest/ingest.routes.ts
@@ -0,0 +1,196 @@
+// Ported from apps/api/src/routes/track.router.ts + track.controller.ts
+// (M8-002). V1's Fastify router stays the LIVE route (DELEGATE PATTERN) and
+// its controller and hooks delegate into ingest.service.ts — the same
+// functions this file calls.
+//
+// THE HOOK ORDER IS THE CONTRACT (ADR-002 "behaviour that must be preserved
+// explicitly" 5): duplicate -> clientAuth -> isBot. Here it holds by
+// lifecycle phase rather than by registration luck — measured on Elysia
+// 1.4.30, a route declaring all three runs
+// `derive -> transform -> macro resolve -> beforeHandle`. So the duplicate
+// check is a `transform` (V1's `preValidation`), `clientAuth` is the macro,
+// and the bot check is a `beforeHandle`, which is what lets it read
+// `client.secretPresented` the way V1's `isBotHook` reads the
+// `req.clientSecretAuth` side channel `validateSdkRequest` sets.
+//
+// Both routes carry the whole chain because V1's `fastify.addHook` calls are
+// plugin-scoped, so `GET /track/device-id` goes through it too.
+//
+// NAMED GAPS, same as profile.routes.ts: not yet reachable — main.ts does not
+// mount `publicApiRoutes` until a real `AppDeps` exists. V1's
+// `subscriptionHook` (the wind-down gate) is NOT ported here: it belongs to
+// the subscription/organization modules and reads `process.env.SELF_HOSTED`,
+// which core does not do. It stays live on V1's router.
+
+import type { AppDeps, HttpCtx } from '../../context';
+import { defineRoutes } from '../../http/define';
+import { zTrackHandlerPayload } from './ingest.constants';
+import {
+ checkIngestBot,
+ fetchDeviceIdentity,
+ ingestTrack,
+ isDuplicateIngestRequest,
+ type TrackOutcome,
+} from './ingest.service';
+import { toIngestHeaders } from './src/headers';
+
+const TAGS = ['Track'];
+const DUPLICATE_BODY = 'Duplicate event';
+
+type StatusFn = (code: number, body?: unknown) => unknown;
+
+// V1's error handler turns each thrown HttpError into this body; core has no
+// such handler, so the mapping is explicit and the codes and messages are
+// V1's.
+function respondToOutcome(outcome: TrackOutcome, status: StatusFn) {
+ switch (outcome.status) {
+ case 'ok':
+ return { deviceId: outcome.deviceId, sessionId: outcome.sessionId };
+ case 'alias-not-supported':
+ return status(400, {
+ status: 400,
+ error: 'Bad Request',
+ message: 'Alias is not supported',
+ });
+ case 'invalid-type':
+ return status(400, {
+ status: 400,
+ error: 'Bad Request',
+ message: 'Invalid type',
+ });
+ case 'missing-project-id':
+ return status(400, { status: 400, message: 'Missing projectId' });
+ case 'replay-missing-session-id':
+ return status(400, {
+ status: 400,
+ message: 'Session ID is required for replay',
+ });
+ case 'profile-not-found':
+ return status(404, { status: 404, message: 'Profile not found' });
+ default:
+ return status(400, {
+ status: 400,
+ message: 'Property value is not a number',
+ });
+ }
+}
+
+// V1's `isBotHook`: a `preHandler` registered after `clientHook`, so it can
+// read the `clientSecretAuth` side channel. Here it is a route-level
+// `beforeHandle`, which runs after the macro resolved the principal.
+async function botGuard({
+ body,
+ client,
+ ctx,
+ status,
+}: {
+ body: unknown;
+ client: { projectId: string | null; secretPresented: boolean };
+ ctx: HttpCtx;
+ status: StatusFn;
+}) {
+ const bot = await checkIngestBot({
+ headers: toIngestHeaders(ctx.headers),
+ clientSecretAuth: client.secretPresented,
+ projectId: client.projectId,
+ body,
+ });
+ if (bot) {
+ return status(202, { bot });
+ }
+}
+
+export const ingestRoutes = defineRoutes((app, deps: AppDeps) =>
+ app.guard(
+ {
+ // V1's `duplicateHook`, registered on `preValidation` so it answers
+ // before the client is ever authenticated. A guard hook runs ahead of
+ // the `clientAuth` macro's resolve and is scoped to the two routes
+ // below — measured on Elysia 1.4.30:
+ // derive -> guard beforeHandle -> macro resolve -> route beforeHandle.
+ async beforeHandle({ body, ctx, request, status }) {
+ if (
+ await isDuplicateIngestRequest({
+ method: request.method,
+ clientIp: ctx.ip,
+ headers: toIngestHeaders(ctx.headers),
+ body,
+ })
+ ) {
+ return status(200, DUPLICATE_BODY);
+ }
+ },
+ },
+ (routes) =>
+ routes
+ .post(
+ '/track',
+ async ({ body, client, ctx, status, timestamp }) => {
+ const outcome = await ingestTrack(
+ {
+ projectId: client.projectId,
+ clientIp: ctx.ip,
+ headers: toIngestHeaders(ctx.headers),
+ clientSecretAuth: client.secretPresented,
+ timestamp,
+ body,
+ },
+ {
+ buffers: ctx.buffers,
+ produceIncomingEvent: deps.produceIncomingEvent,
+ }
+ );
+
+ return respondToOutcome(outcome, status);
+ },
+ {
+ clientAuth: { ingest: true },
+ body: zTrackHandlerPayload,
+ beforeHandle: botGuard,
+ detail: {
+ tags: TAGS,
+ description:
+ 'Ingest a tracking event (track, identify, group, increment, decrement, replay).',
+ },
+ }
+ )
+ .get(
+ '/track/device-id',
+ async ({ client, ctx, status }) => {
+ const identity = await fetchDeviceIdentity(
+ {
+ projectId: client.projectId,
+ clientIp: ctx.ip,
+ headers: toIngestHeaders(ctx.headers),
+ },
+ ctx.buffers,
+ ctx.logger
+ );
+
+ switch (identity.status) {
+ case 'missing-project-id':
+ return status(400, 'No projectId');
+ case 'missing-ip':
+ return status(400, 'Missing ip address');
+ case 'missing-user-agent':
+ return status(400, 'Missing header: user-agent');
+ default:
+ return {
+ deviceId: identity.deviceId,
+ sessionId: identity.sessionId,
+ message: identity.message,
+ };
+ }
+ },
+ {
+ clientAuth: { ingest: true },
+ beforeHandle: botGuard,
+ detail: {
+ tags: TAGS,
+ description:
+ 'Get or generate a stable device ID and session ID for the current visitor.',
+ },
+ }
+ )
+ )
+);
diff --git a/packages/core/src/modules/ingest/ingest.service.test.ts b/packages/core/src/modules/ingest/ingest.service.test.ts
new file mode 100644
index 000000000..3eb73260d
--- /dev/null
+++ b/packages/core/src/modules/ingest/ingest.service.test.ts
@@ -0,0 +1,249 @@
+/**
+ * Tests for:
+ * - getOverrideDeviceId — the guard around a track event's caller-supplied
+ * `__deviceId` (it becomes a Redis key segment, a BullMQ jobId and a ClickHouse
+ * device_id, so it must be trimmed, non-empty, and length-bounded).
+ * - handleReplay — replay files a chunk under the session id the SDK echoes back;
+ * it needs no device resolution and trusts the client-sent session id.
+ * - ingestTrack — the single events transport (ADR-004): every tracked event goes
+ * to Kafka via the injected producer, with no branch and no fallback queue.
+ *
+ * Moved from apps/api/src/controllers/track.controller.test.ts with M8-002.
+ * The producer and the buffers are arguments now, so V1's `vi.mock` of
+ * @openpanel/queue and its `vi.spyOn(replayBuffer)` become plain stubs — same
+ * assertions. Geo/ASN and the salts are still module-level, so those keep a
+ * `mock.module` (unhoisted, hence the `await import` in `beforeAll`).
+ */
+
+import { beforeAll, beforeEach, describe, expect, it, mock } from 'bun:test';
+import type { Buffers } from '../../buffers/create-buffers';
+import type { GeoLocation } from '../../clients/geo';
+import type { IReplayPayload, ITrackHandlerPayload } from './ingest.constants';
+
+const getGeoLocation = mock(
+ async (): Promise<GeoLocation> => ({
+ country: 'SE',
+ city: 'Stockholm',
+ region: 'AB',
+ longitude: 18,
+ latitude: 59,
+ })
+);
+const getAsnInfo = mock(async () => ({
+ asn: undefined,
+ org: undefined,
+ isHosting: false,
+}));
+const getSalts = mock(async () => ({
+ current: 'salt-current',
+ previous: 'salt-previous',
+}));
+const identifyProfile = mock(async () => undefined);
+const upsertProfile = mock(async () => undefined);
+const getProfileById = mock(async () => null);
+
+mock.module('../../clients/geo', () => ({ getGeoLocation, getAsnInfo }));
+mock.module('@openpanel/db/src/services/salt.service', () => ({ getSalts }));
+
+let getOverrideDeviceId: typeof import('./ingest.service').getOverrideDeviceId;
+let handleReplay: typeof import('./ingest.service').handleReplay;
+let ingestTrack: typeof import('./ingest.service').ingestTrack;
+
+// The profile module is spread and then overridden by name: event.service
+// imports more of it than ingest.service does, so a factory returning only
+// the three overrides makes the rest vanish under `--isolate`. Overriding
+// `identifyProfile` (not just `upsertProfile`) is what keeps the identify
+// path off Postgres and the profile buffer — profile.service calls its own
+// local `upsertProfile`, not the exported binding.
+beforeAll(async () => {
+ const profile = await import('../profile/profile.service');
+ mock.module('../profile/profile.service', () => ({
+ ...profile,
+ identifyProfile,
+ upsertProfile,
+ getProfileById,
+ }));
+ ({ getOverrideDeviceId, handleReplay, ingestTrack } = await import(
+ './ingest.service'
+ ));
+});
+
+const track = (properties?: Record<string, unknown>): ITrackHandlerPayload =>
+ ({
+ type: 'track',
+ payload: { name: 'page_view', properties },
+ }) as ITrackHandlerPayload;
+
+describe('getOverrideDeviceId', () => {
+ it('returns a trimmed device id', () => {
+ expect(getOverrideDeviceId(track({ __deviceId: ' cookie-abc ' }))).toBe(
+ 'cookie-abc'
+ );
+ });
+
+ it('returns the device id as-is when already clean', () => {
+ expect(getOverrideDeviceId(track({ __deviceId: 'cookie-abc' }))).toBe(
+ 'cookie-abc'
+ );
+ });
+
+ it('ignores empty / whitespace-only values', () => {
+ expect(getOverrideDeviceId(track({ __deviceId: '' }))).toBeUndefined();
+ expect(getOverrideDeviceId(track({ __deviceId: ' ' }))).toBeUndefined();
+ });
+
+ it('ignores non-string values', () => {
+ expect(getOverrideDeviceId(track({ __deviceId: 123 }))).toBeUndefined();
+ expect(getOverrideDeviceId(track({}))).toBeUndefined();
+ expect(getOverrideDeviceId(track(undefined))).toBeUndefined();
+ });
+
+ it('ignores pathologically long values (cap is 64)', () => {
+ expect(
+ getOverrideDeviceId(track({ __deviceId: 'x'.repeat(65) }))
+ ).toBeUndefined();
+ expect(getOverrideDeviceId(track({ __deviceId: 'x'.repeat(64) }))).toBe(
+ 'x'.repeat(64)
+ );
+ });
+
+ it('returns undefined for non-track payload types', () => {
+ expect(
+ getOverrideDeviceId({
+ type: 'identify',
+ payload: { profileId: 'p1', properties: { __deviceId: 'cookie' } },
+ } as ITrackHandlerPayload)
+ ).toBeUndefined();
+ });
+});
+
+const chunk = (): IReplayPayload => ({
+ chunk_index: 0,
+ events_count: 1,
+ is_full_snapshot: true,
+ started_at: '2026-06-08T12:00:00.000Z',
+ ended_at: '2026-06-08T12:00:01.000Z',
+ payload: '[]',
+});
+
+describe('handleReplay', () => {
+ it('files the chunk under the session id provided by the caller', async () => {
+ const add = mock((_row: Record<string, unknown>) => Promise.resolve());
+
+ const failure = await handleReplay(
+ chunk(),
+ { projectId: 'proj-1', sessionId: 'sess-issued-123' },
+ { add } as unknown as Buffers['replay']
+ );
+
+ expect(failure).toBeNull();
+ expect(add).toHaveBeenCalledTimes(1);
+ expect(add.mock.calls[0]?.[0]).toMatchObject({
+ project_id: 'proj-1',
+ session_id: 'sess-issued-123',
+ });
+ });
+
+ it('refuses the chunk when the session id is missing', async () => {
+ const add = mock(async () => undefined);
+
+ const failure = await handleReplay(
+ chunk(),
+ { projectId: 'proj-1', sessionId: undefined },
+ { add } as unknown as Buffers['replay']
+ );
+
+ expect(failure).toEqual({ status: 'replay-missing-session-id' });
+ expect(add).not.toHaveBeenCalled();
+ });
+});
+
+const PROJECT_ID = 'proj-kafka';
+const DEVICE_ID = 'device-kafka-1';
+
+const getExistingSession = mock(async () => null);
+
+const transport = (
+ produceIncomingEvent: ReturnType<typeof mock>
+): Parameters<typeof ingestTrack>[1] =>
+ ({
+ buffers: {
+ session: { getExistingSession },
+ replay: { add: mock(async () => undefined) },
+ group: { add: mock(async () => undefined) },
+ },
+ produceIncomingEvent,
+ }) as unknown as Parameters<typeof ingestTrack>[1];
+
+const request = (body: ITrackHandlerPayload) => ({
+ projectId: PROJECT_ID,
+ clientIp: '1.2.3.4',
+ clientSecretAuth: false,
+ timestamp: new Date('2026-06-08T12:00:00.000Z').getTime(),
+ headers: { 'user-agent': 'Mozilla/5.0 (Macintosh) Chrome/148.0.0.0' },
+ body,
+});
+
+describe('ingestTrack — events transport', () => {
+ beforeEach(() => {
+ getExistingSession.mockClear();
+ identifyProfile.mockClear();
+ });
+
+ it('produces every tracked event to Kafka — the only transport', async () => {
+ const produce = mock(async () => undefined);
+
+ await ingestTrack(
+ request({
+ type: 'track',
+ payload: { name: 'page_view', properties: { __deviceId: DEVICE_ID } },
+ } as ITrackHandlerPayload),
+ transport(produce)
+ );
+
+ expect(produce).toHaveBeenCalledTimes(1);
+ const [queueData, partitionKey] = produce.mock.calls[0] as unknown as [
+ Record<string, unknown>,
+ string,
+ ];
+ expect(queueData).toMatchObject({
+ projectId: PROJECT_ID,
+ deviceId: DEVICE_ID,
+ event: { name: 'page_view' },
+ });
+ // Browser events partition on the device id, which is what preserves
+ // per-device mutual exclusion in the consumer.
+ expect(partitionKey).toBe(DEVICE_ID);
+ });
+
+ it('surfaces a produce failure instead of falling back to another queue', async () => {
+ const produce = mock(() => Promise.reject(new Error('kafka unavailable')));
+
+ await expect(
+ ingestTrack(
+ request({
+ type: 'track',
+ payload: { name: 'page_view', properties: { __deviceId: DEVICE_ID } },
+ } as ITrackHandlerPayload),
+ transport(produce)
+ )
+ ).rejects.toThrow('kafka unavailable');
+
+ expect(produce).toHaveBeenCalledTimes(1);
+ });
+
+ it('does not produce for non-event payload types', async () => {
+ const produce = mock(async () => undefined);
+
+ await ingestTrack(
+ request({
+ type: 'identify',
+ payload: { profileId: 'p1', properties: {} },
+ } as ITrackHandlerPayload),
+ transport(produce)
+ );
+
+ expect(produce).not.toHaveBeenCalled();
+ expect(identifyProfile).toHaveBeenCalledTimes(1);
+ });
+});
diff --git a/packages/core/src/modules/ingest/ingest.service.ts b/packages/core/src/modules/ingest/ingest.service.ts
new file mode 100644
index 000000000..06b8a0651
--- /dev/null
+++ b/packages/core/src/modules/ingest/ingest.service.ts
@@ -0,0 +1,744 @@
+// The ingestion pipeline (M8-002). Ported from apps/api's
+// track.controller.ts, its three route hooks (duplicate / client / is-bot)
+// and utils/ids.ts. V1's Fastify controller and hooks stay the LIVE route
+// (DELEGATE PATTERN) and call the functions below; ingest.routes.ts is the
+// V2 half over the same functions.
+//
+// Nothing here throws for a caller error: each transport owns its own status
+// codes and bodies, so the pipeline returns a `TrackOutcome` and V1 maps it
+// back onto the exact `HttpError`s and replies it produced before.
+//
+// The Kafka producer is INJECTED. `produceIncomingEvent` still lives in
+// @openpanel/queue, which imports @openpanel/core for its logger — core
+// cannot import it back without a real package cycle (see
+// cohort.service.ts's header). M8-003 moves the producer into core and the
+// injection point becomes an AppDeps field.
+
+import { generateId } from '@openpanel/common';
+import { assocPath, pathOr, pick } from 'ramda';
+import { v4 as uuid } from 'uuid';
+import type { Buffers } from '../../buffers/create-buffers';
+import { SESSION_TIMEOUT_MS } from '../../buffers/session-buffer';
+import {
+ type AsnInfo,
+ type GeoLocation,
+ getAsnInfo,
+ getGeoLocation,
+} from '../../clients/geo';
+import type { Logger } from '../../logger';
+import type { ServiceDeps } from '../../services';
+import { parseUserAgent } from '../../shared/parser-user-agent';
+import { generateDeviceId } from '../../shared/profileId';
+import { createBotEvent } from '../event/event.service';
+import {
+ getProfileById,
+ identifyProfile,
+ upsertProfile,
+} from '../profile/profile.service';
+import { convertClickhouseDateToJs } from '../session/src/dates';
+import type {
+ IAssignGroupPayload,
+ IDecrementPayload,
+ IGroupPayload,
+ IIdentifyPayload,
+ IIncrementPayload,
+ IReplayPayload,
+ ITrackHandlerPayload,
+ ITrackPayload,
+} from './ingest.constants';
+import { isBot } from './src/bots/detect';
+import { applyBotSuspicion, stripBotProperties } from './src/bots/suspicion';
+import { isDuplicatedEvent } from './src/deduplicate';
+import { getDeviceId } from './src/device-id';
+import { headerValue, type IngestHeaders } from './src/headers';
+import type {
+ IncomingEventPayload,
+ IncomingEventProducer,
+} from './src/incoming-event';
+
+export type { BotMatch } from './src/bots/detect';
+export { detectBot, isBot } from './src/bots/detect';
+export {
+ applyBotSuspicion,
+ BOT_CATEGORY_THRESHOLD,
+ type BotSuspicion,
+ stripBotProperties,
+ summarizeBotSignals,
+} from './src/bots/suspicion';
+export {
+ type IngestAuthErrorPayload,
+ type IngestAuthOutcome,
+ validateIngestRequest,
+} from './src/client-auth';
+export { isDuplicatedEvent } from './src/deduplicate';
+export { getDeviceId } from './src/device-id';
+export type { IngestHeaders } from './src/headers';
+export type {
+ IncomingEventPayload,
+ IncomingEventProducer,
+} from './src/incoming-event';
+
+/** The buffers this module touches. A subset of `AppDeps.buffers`. */
+export type IngestBuffers = Pick<Buffers, 'session' | 'replay' | 'group'>;
+
+export interface IngestTransport {
+ buffers: IngestBuffers;
+ produceIncomingEvent: IncomingEventProducer;
+}
+
+const QUEUE_PAYLOAD_HEADERS = [
+ 'user-agent',
+ 'openpanel-sdk-name',
+ 'openpanel-sdk-version',
+ 'openpanel-client-id',
+ 'request-id',
+];
+
+const MAX_OVERRIDE_DEVICE_ID_LENGTH = 64;
+const ONE_MINUTE_MS = 60 * 1000;
+const FIFTEEN_MINUTES_MS = 15 * ONE_MINUTE_MS;
+const FALLBACK_USER_AGENT = 'unknown/1.0';
+
+function loadSalts() {
+ return import('@openpanel/db/src/services/salt.service');
+}
+
+/** The whitelisted subset that ships with the queue payload. */
+export function getStringHeaders(
+ headers: IngestHeaders
+): Record<string, string | undefined> {
+ return Object.entries(pick(QUEUE_PAYLOAD_HEADERS, headers)).reduce(
+ (acc, [key, value]) => ({
+ ...acc,
+ [key]: value ? String(value) : undefined,
+ }),
+ {}
+ );
+}
+
+function getIdentity(body: ITrackHandlerPayload): IIdentifyPayload | undefined {
+ if (body.type === 'track') {
+ const identity = body.payload.properties?.__identify as
+ | IIdentifyPayload
+ | undefined;
+
+ if (identity) {
+ return identity;
+ }
+
+ return body.payload.profileId
+ ? { profileId: String(body.payload.profileId) }
+ : undefined;
+ }
+
+ return undefined;
+}
+
+function sanitizeOverrideDeviceId(raw: unknown): string | undefined {
+ if (typeof raw !== 'string') {
+ return undefined;
+ }
+ const trimmed = raw.trim();
+ if (!trimmed || trimmed.length > MAX_OVERRIDE_DEVICE_ID_LENGTH) {
+ return undefined;
+ }
+ return trimmed;
+}
+
+/** Resolve a caller-supplied device id from `properties.__deviceId`. */
+export function getOverrideDeviceId(
+ body: ITrackHandlerPayload
+): string | undefined {
+ if (body.type !== 'track') {
+ return undefined;
+ }
+ return sanitizeOverrideDeviceId(body.payload?.properties?.__deviceId);
+}
+
+export function getTimestamp(
+ timestamp: number | undefined,
+ payload: ITrackHandlerPayload['payload']
+): { timestamp: number; isTimestampFromThePast: boolean } {
+ const safeTimestamp = timestamp || Date.now();
+ const userDefinedTimestamp =
+ 'properties' in payload
+ ? (payload?.properties?.__timestamp as string | undefined)
+ : undefined;
+
+ if (!userDefinedTimestamp) {
+ return { timestamp: safeTimestamp, isTimestampFromThePast: false };
+ }
+
+ const clientTimestamp = new Date(userDefinedTimestamp);
+ const clientTimestampNumber = clientTimestamp.getTime();
+
+ // Use safeTimestamp if invalid or more than 1 minute in the future
+ if (
+ Number.isNaN(clientTimestampNumber) ||
+ clientTimestampNumber > safeTimestamp + ONE_MINUTE_MS
+ ) {
+ return { timestamp: safeTimestamp, isTimestampFromThePast: false };
+ }
+
+ // isTimestampFromThePast is true only if timestamp is older than 15 minutes
+ const isTimestampFromThePast =
+ clientTimestampNumber < safeTimestamp - FIFTEEN_MINUTES_MS;
+
+ return { timestamp: clientTimestampNumber, isTimestampFromThePast };
+}
+
+export interface TrackContext {
+ projectId: string;
+ ip: string;
+ ua?: string;
+ // Whitelisted subset (getStringHeaders) that ships with the queue payload.
+ headers: Record<string, string | undefined>;
+ // Full raw request headers — bot signals need sec-ch-ua / sec-fetch-* /
+ // accept-language, which the whitelist above deliberately drops.
+ requestHeaders: IngestHeaders;
+ clientSecretAuth: boolean;
+ timestamp: { value: number; isFromPast: boolean };
+ identity?: IIdentifyPayload;
+ deviceId: string;
+ sessionId: string;
+ geo: GeoLocation;
+ asnInfo: AsnInfo;
+}
+
+/** What a transport hands the pipeline, after its own auth ran. */
+export interface TrackRequest {
+ projectId: string | null | undefined;
+ clientIp: string;
+ headers: IngestHeaders;
+ clientSecretAuth: boolean;
+ /** The request-arrival timestamp (V1's `timestampHook`). */
+ timestamp: number | undefined;
+ body: ITrackHandlerPayload;
+}
+
+export type TrackOutcome =
+ | { status: 'ok'; deviceId: string; sessionId: string }
+ | { status: 'alias-not-supported' }
+ | { status: 'invalid-type' }
+ | { status: 'missing-project-id' }
+ | { status: 'replay-missing-session-id' }
+ | { status: 'profile-not-found' }
+ | { status: 'profile-property-not-a-number' };
+
+async function buildContext(
+ request: TrackRequest,
+ buffers: IngestBuffers
+): Promise<TrackContext> {
+ const projectId = request.projectId as string;
+ const timestamp = getTimestamp(request.timestamp, request.body.payload);
+ const ip =
+ request.body.type === 'track' && request.body.payload.properties?.__ip
+ ? (request.body.payload.properties.__ip as string)
+ : request.clientIp;
+ const ua = headerValue(request.headers, 'user-agent') ?? FALLBACK_USER_AGENT;
+
+ const headers = getStringHeaders(request.headers);
+ const identity = getIdentity(request.body);
+ const profileId = identity?.profileId;
+
+ if (profileId && request.body.type === 'track') {
+ request.body.payload.profileId = profileId;
+ }
+
+ const overrideDeviceId = getOverrideDeviceId(request.body);
+
+ // Get geo location (needed for track and identify) + ASN (bot detection).
+ // Both hit the same MaxMind readers keyed on the same IP and are cached, so
+ // resolving them together adds no meaningful latency.
+ const { getSalts } = await loadSalts();
+ const [geo, asnInfo, salts] = await Promise.all([
+ getGeoLocation(ip),
+ getAsnInfo(ip),
+ getSalts(),
+ ]);
+
+ const deviceIdResult = await getDeviceId({
+ projectId,
+ ip,
+ ua,
+ salts,
+ overrideDeviceId,
+ eventTimeMs: timestamp.timestamp,
+ sessionBuffer: buffers.session,
+ });
+
+ return {
+ projectId,
+ ip,
+ ua,
+ headers,
+ requestHeaders: request.headers,
+ clientSecretAuth: request.clientSecretAuth,
+ timestamp: {
+ value: timestamp.timestamp,
+ isFromPast: timestamp.isTimestampFromThePast,
+ },
+ identity,
+ deviceId: deviceIdResult.deviceId,
+ sessionId: deviceIdResult.sessionId,
+ geo,
+ asnInfo,
+ };
+}
+
+async function handleTrack(
+ payload: ITrackPayload,
+ context: TrackContext,
+ transport: IngestTransport
+): Promise<void> {
+ const { projectId, deviceId, geo, headers, timestamp, sessionId } = context;
+
+ const uaInfo = parseUserAgent(headers['user-agent'], payload.properties);
+ // Mark (never block) likely bot traffic with __bot / __bot_reasons props.
+ payload.properties = applyBotSuspicion(payload.properties, {
+ asnInfo: context.asnInfo,
+ headers: context.requestHeaders,
+ clientSecretAuth: context.clientSecretAuth,
+ isServer: uaInfo.isServer,
+ });
+ const groupId = uaInfo.isServer
+ ? payload.profileId
+ ? `${projectId}:${payload.profileId}`
+ : undefined
+ : deviceId;
+ const promises: Promise<unknown>[] = [];
+
+ // If we have more than one property in the identity object, we should identify the user
+ // Otherwise its only a profileId and we should not identify the user
+ if (context.identity && Object.keys(context.identity).length > 1) {
+ promises.push(handleIdentify(context.identity, context));
+ }
+
+ const queueData: IncomingEventPayload = {
+ // The id of the ClickHouse row this event becomes, minted here so a
+ // redelivered Kafka message is a duplicate we can recognise.
+ id: uuid(),
+ projectId,
+ headers,
+ event: {
+ ...payload,
+ groups: payload.groups ?? [],
+ timestamp: timestamp.value,
+ isTimestampFromThePast: timestamp.isFromPast,
+ },
+ uaInfo,
+ geo,
+ deviceId,
+ sessionId,
+ };
+
+ const partitionKey = groupId || generateId();
+
+ promises.push(transport.produceIncomingEvent(queueData, partitionKey));
+
+ await Promise.all(promises);
+}
+
+async function handleIdentify(
+ payload: IIdentifyPayload,
+ context: TrackContext
+): Promise<void> {
+ // Profiles must not carry forged bot verdicts either.
+ stripBotProperties(payload.properties);
+ const userAgent = parseUserAgent(context.ua, payload.properties);
+ await identifyProfile(context.projectId, payload, {
+ geo: context.geo,
+ userAgent,
+ });
+}
+
+async function adjustProfileProperty(
+ payload: IIncrementPayload | IDecrementPayload,
+ projectId: string,
+ direction: 1 | -1
+): Promise<TrackOutcome | null> {
+ const { profileId, property, value } = payload;
+ const profile = await getProfileById(String(profileId), projectId);
+ if (!profile) {
+ return { status: 'profile-not-found' };
+ }
+
+ const parsed = Number.parseInt(
+ pathOr<string>('0', property.split('.'), profile.properties),
+ 10
+ );
+
+ if (Number.isNaN(parsed)) {
+ return { status: 'profile-property-not-a-number' };
+ }
+
+ await upsertProfile({
+ id: profile.id,
+ projectId,
+ properties: assocPath(
+ property.split('.'),
+ parsed + direction * (value || 1),
+ profile.properties
+ ),
+ isExternal: true,
+ });
+
+ return null;
+}
+
+/** Replay only needs the server-issued session id (the SDK echoes it back).
+ * Trust it — scoped to the authed project, unguessable, same trust level as
+ * event data. */
+export async function handleReplay(
+ payload: IReplayPayload,
+ {
+ projectId,
+ sessionId,
+ }: { projectId: string; sessionId: string | undefined },
+ replayBuffer: IngestBuffers['replay']
+): Promise<TrackOutcome | null> {
+ if (!sessionId) {
+ return { status: 'replay-missing-session-id' };
+ }
+
+ await replayBuffer.add({
+ project_id: projectId,
+ session_id: sessionId,
+ chunk_index: payload.chunk_index,
+ started_at: payload.started_at,
+ ended_at: payload.ended_at,
+ events_count: payload.events_count,
+ is_full_snapshot: payload.is_full_snapshot,
+ payload: payload.payload,
+ });
+ return null;
+}
+
+async function handleGroup(
+ payload: IGroupPayload,
+ context: TrackContext,
+ groupBuffer: IngestBuffers['group']
+): Promise<void> {
+ const { id, type, name, properties = {} } = payload;
+ await groupBuffer.add({
+ id,
+ projectId: context.projectId,
+ type,
+ name,
+ properties,
+ });
+}
+
+async function handleAssignGroup(
+ payload: IAssignGroupPayload,
+ context: TrackContext
+): Promise<void> {
+ const profileId = payload.profileId ?? context.deviceId;
+ if (!profileId) {
+ return;
+ }
+ await upsertProfile({
+ id: String(profileId),
+ projectId: context.projectId,
+ isExternal: !!payload.profileId,
+ groups: payload.groupIds,
+ });
+}
+
+/** `POST /track`. The dispatch order is V1's, including that an unknown type
+ * still resolves geo/device before it is refused. */
+export async function ingestTrack(
+ request: TrackRequest,
+ transport: IngestTransport
+): Promise<TrackOutcome> {
+ const body = request.body;
+
+ if (body.type === 'alias') {
+ return { status: 'alias-not-supported' };
+ }
+
+ if (!request.projectId) {
+ return { status: 'missing-project-id' };
+ }
+
+ const context = await buildContext(request, transport.buffers);
+
+ switch (body.type) {
+ case 'track':
+ await handleTrack(body.payload, context, transport);
+ break;
+ case 'identify':
+ await handleIdentify(body.payload, context);
+ break;
+ case 'increment': {
+ const failure = await adjustProfileProperty(
+ body.payload,
+ context.projectId,
+ 1
+ );
+ if (failure) {
+ return failure;
+ }
+ break;
+ }
+ case 'decrement': {
+ const failure = await adjustProfileProperty(
+ body.payload,
+ context.projectId,
+ -1
+ );
+ if (failure) {
+ return failure;
+ }
+ break;
+ }
+ case 'replay': {
+ // BACKCOMPAT(replay-sessionid): TEMPORARY legacy branch, remove when new SDK is fully deployed
+ const failure = await handleReplay(
+ body.payload,
+ {
+ projectId: context.projectId,
+ sessionId: body.payload.sessionId || context.sessionId,
+ },
+ transport.buffers.replay
+ );
+ if (failure) {
+ return failure;
+ }
+ break;
+ }
+ case 'group':
+ await handleGroup(body.payload, context, transport.buffers.group);
+ break;
+ case 'assign_group':
+ await handleAssignGroup(body.payload, context);
+ break;
+ default:
+ return { status: 'invalid-type' };
+ }
+
+ return {
+ status: 'ok',
+ deviceId: context.deviceId,
+ sessionId: context.sessionId,
+ };
+}
+
+export type DeviceIdentity =
+ | { status: 'ok'; deviceId: string; sessionId: string; message: string }
+ | { status: 'missing-project-id' }
+ | { status: 'missing-ip' }
+ | { status: 'missing-user-agent' };
+
+/** `GET /track/device-id`. */
+export async function fetchDeviceIdentity(
+ request: {
+ projectId: string | null | undefined;
+ clientIp: string | undefined;
+ headers: IngestHeaders;
+ },
+ buffers: Pick<IngestBuffers, 'session'>,
+ logger: Logger
+): Promise<DeviceIdentity> {
+ const { getSalts } = await loadSalts();
+ const salts = await getSalts();
+ const projectId = request.projectId;
+ if (!projectId) {
+ return { status: 'missing-project-id' };
+ }
+
+ const ip = request.clientIp;
+ if (!ip) {
+ return { status: 'missing-ip' };
+ }
+
+ const ua = headerValue(request.headers, 'user-agent');
+ if (!ua) {
+ return { status: 'missing-user-agent' };
+ }
+
+ const currentDeviceId = generateDeviceId({
+ salt: salts.current,
+ origin: projectId,
+ ip,
+ ua,
+ });
+ const previousDeviceId = generateDeviceId({
+ salt: salts.previous,
+ origin: projectId,
+ ip,
+ ua,
+ });
+
+ try {
+ const [current, previous] = await Promise.all([
+ buffers.session.getExistingSession({
+ projectId,
+ deviceId: currentDeviceId,
+ }),
+ buffers.session.getExistingSession({
+ projectId,
+ deviceId: previousDeviceId,
+ }),
+ ]);
+
+ // Blob has no TTL — only treat the session as "current" if its last
+ // event is within the idle window. Otherwise the SDK should ask the
+ // server to start a fresh session id on the next event.
+ const now = Date.now();
+
+ if (current && isLiveSession(current, now)) {
+ return {
+ status: 'ok',
+ deviceId: currentDeviceId,
+ sessionId: current.id,
+ message: 'current session exists for this device id',
+ };
+ }
+
+ if (previous && isLiveSession(previous, now)) {
+ return {
+ status: 'ok',
+ deviceId: previousDeviceId,
+ sessionId: previous.id,
+ message: 'previous session exists for this device id',
+ };
+ }
+ } catch (error) {
+ logger.error(
+ { err: error },
+ 'Error getting session end GET /track/device-id'
+ );
+ }
+
+ return {
+ status: 'ok',
+ deviceId: currentDeviceId,
+ sessionId: '',
+ message: 'No session exists for this device id',
+ };
+}
+
+/** The `duplicateHook`: the web SDK can fire the same event twice. */
+export async function isDuplicateIngestRequest(request: {
+ method: string;
+ clientIp: string | undefined;
+ headers: IngestHeaders;
+ body: unknown;
+}): Promise<boolean> {
+ const ip = request.clientIp;
+ const origin = headerValue(request.headers, 'origin');
+ const clientId = headerValue(request.headers, 'openpanel-client-id');
+ const isReplay = isTrackBody(request) && request.body.type === 'replay';
+ const shouldCheck = ip && origin && clientId && !isReplay;
+
+ if (!shouldCheck) {
+ return false;
+ }
+
+ return await isDuplicatedEvent({
+ ip,
+ origin,
+ payload: request.body as Record<string, unknown>,
+ projectId: clientId,
+ });
+}
+
+function isTrackBody(request: {
+ method: string;
+ body: unknown;
+}): request is { method: string; body: ITrackHandlerPayload } {
+ if (request.method !== 'POST') {
+ return false;
+ }
+ if (!request.body) {
+ return false;
+ }
+ if (typeof request.body !== 'object' || Array.isArray(request.body)) {
+ return false;
+ }
+ return 'type' in request.body;
+}
+
+export type BotVerdict = { name: string; type: string } | null;
+
+/**
+ * The `isBotHook`. Requests authenticated with a client secret come from
+ * server-side SDKs (node, php, go, rust, java, python, …). That auth is a far
+ * stronger signal of legitimate first-party traffic than the user agent, so
+ * never treat them as bots — bot detection is for public/frontend
+ * (origin-authenticated) traffic.
+ */
+export async function checkIngestBot(request: {
+ headers: IngestHeaders;
+ clientSecretAuth: boolean;
+ projectId: string | null | undefined;
+ body: unknown;
+}): Promise<BotVerdict> {
+ if (request.clientSecretAuth) {
+ return null;
+ }
+
+ const userAgent = headerValue(request.headers, 'user-agent');
+ const bot = userAgent ? await isBot(userAgent) : null;
+
+ if (!(bot && request.projectId)) {
+ return null;
+ }
+
+ const path = getBotEventPath(request.body);
+ if (path) {
+ await createBotEvent({
+ ...bot,
+ projectId: request.projectId,
+ path,
+ createdAt: new Date(),
+ });
+ }
+
+ return bot;
+}
+
+function getBotEventPath(body: unknown): string | undefined {
+ if (!body || typeof body !== 'object') {
+ return undefined;
+ }
+ if ('type' in body && (body as ITrackHandlerPayload).type === 'track') {
+ const properties = (
+ body as { payload?: { properties?: Record<string, unknown> } }
+ ).payload?.properties;
+ return (properties?.__path || properties?.path) as string | undefined;
+ }
+ // Handle deprecated events (v1)
+ if ('name' in body && 'properties' in body) {
+ const properties = (body as { properties?: Record<string, unknown> })
+ .properties;
+ return (properties?.__path || properties?.path) as string | undefined;
+ }
+ return undefined;
+}
+
+function isLiveSession(session: { ended_at: string }, now: number): boolean {
+ return (
+ now - convertClickhouseDateToJs(session.ended_at).getTime() <
+ SESSION_TIMEOUT_MS
+ );
+}
+
+export interface IngestService {
+ track(
+ request: TrackRequest,
+ produceIncomingEvent: IncomingEventProducer
+ ): Promise<TrackOutcome>;
+ checkBot: typeof checkIngestBot;
+ isDuplicate: typeof isDuplicateIngestRequest;
+}
+
+export function createIngestService(deps: ServiceDeps): IngestService {
+ return {
+ track: (request, produceIncomingEvent) =>
+ ingestTrack(request, { buffers: deps.buffers, produceIncomingEvent }),
+ checkBot: checkIngestBot,
+ isDuplicate: isDuplicateIngestRequest,
+ };
+}
diff --git a/packages/core/src/modules/ingest/src/bot-check.test.ts b/packages/core/src/modules/ingest/src/bot-check.test.ts
new file mode 100644
index 000000000..52f0e0fd1
--- /dev/null
+++ b/packages/core/src/modules/ingest/src/bot-check.test.ts
@@ -0,0 +1,84 @@
+/**
+ * Tests for checkIngestBot — the bot filter V1 runs as `isBotHook` on
+ * ingestion (apps/api/src/hooks/is-bot.hook.test.ts, moved with M8-002).
+ *
+ * The key behaviour guarded here: requests authenticated with a client secret
+ * (server-side SDKs) are never treated as bots, regardless of user agent. Bot
+ * detection only applies to public/frontend (origin-authenticated) traffic.
+ */
+
+import { beforeAll, beforeEach, describe, expect, it, mock } from 'bun:test';
+import type { BotMatch } from './bots/detect';
+
+const isBot = mock(async (_ua: string): Promise<BotMatch | null> => null);
+const createBotEvent = mock(async (_payload: unknown) => undefined);
+
+let checkIngestBot: typeof import('../ingest.service').checkIngestBot;
+
+// Spread the real modules and override by name: ingest.service re-exports
+// `detectBot` and reaches more of event.service than `createBotEvent`, and a
+// factory returning only the overrides makes those exports vanish under
+// `--isolate`.
+beforeAll(async () => {
+ const detect = await import('./bots/detect');
+ const eventService = await import('../../event/event.service');
+ mock.module('./bots/detect', () => ({ ...detect, isBot }));
+ mock.module('../../event/event.service', () => ({
+ ...eventService,
+ createBotEvent,
+ }));
+ ({ checkIngestBot } = await import('../ingest.service'));
+});
+
+const baseRequest = {
+ headers: { 'user-agent': 'Googlebot/2.1' },
+ clientSecretAuth: false,
+ projectId: 'proj-1',
+ body: { type: 'track', payload: { properties: { path: '/home' } } },
+};
+
+beforeEach(() => {
+ isBot.mockReset();
+ createBotEvent.mockReset();
+});
+
+describe('checkIngestBot', () => {
+ it('skips bot detection entirely for client-secret (server-side) traffic', async () => {
+ const bot = await checkIngestBot({
+ ...baseRequest,
+ clientSecretAuth: true,
+ });
+
+ expect(isBot).not.toHaveBeenCalled();
+ expect(createBotEvent).not.toHaveBeenCalled();
+ expect(bot).toBeNull();
+ });
+
+ it('records a bot event and reports the bot for public bot traffic', async () => {
+ isBot.mockResolvedValue({ name: 'Googlebot', type: 'Search bot' });
+
+ const bot = await checkIngestBot(baseRequest);
+
+ expect(isBot).toHaveBeenCalledWith('Googlebot/2.1');
+ expect(createBotEvent).toHaveBeenCalledTimes(1);
+ expect(createBotEvent.mock.calls[0]?.[0]).toMatchObject({
+ name: 'Googlebot',
+ type: 'Search bot',
+ projectId: 'proj-1',
+ path: '/home',
+ });
+ expect(bot).toEqual({ name: 'Googlebot', type: 'Search bot' });
+ });
+
+ it('passes legitimate public traffic through untouched', async () => {
+ isBot.mockResolvedValue(null);
+
+ const bot = await checkIngestBot({
+ ...baseRequest,
+ headers: { 'user-agent': 'node' },
+ });
+
+ expect(createBotEvent).not.toHaveBeenCalled();
+ expect(bot).toBeNull();
+ });
+});
diff --git a/apps/api/src/bots/bots.readme.md b/packages/core/src/modules/ingest/src/bots/bots.readme.md
similarity index 100%
rename from apps/api/src/bots/bots.readme.md
rename to packages/core/src/modules/ingest/src/bots/bots.readme.md
diff --git a/apps/api/src/bots/bots.ts b/packages/core/src/modules/ingest/src/bots/bots.ts
similarity index 100%
rename from apps/api/src/bots/bots.ts
rename to packages/core/src/modules/ingest/src/bots/bots.ts
diff --git a/apps/api/src/bots/index.test.ts b/packages/core/src/modules/ingest/src/bots/detect.test.ts
similarity index 96%
rename from apps/api/src/bots/index.test.ts
rename to packages/core/src/modules/ingest/src/bots/detect.test.ts
index 624655b2c..f62214c0d 100644
--- a/apps/api/src/bots/index.test.ts
+++ b/packages/core/src/modules/ingest/src/bots/detect.test.ts
@@ -10,8 +10,8 @@
* identifiers, not the rest of the bot patterns.
*/
-import { describe, expect, it } from 'vitest';
-import { detectBot } from './index';
+import { describe, expect, it } from 'bun:test';
+import { detectBot } from './detect';
describe('detectBot', () => {
describe('does not flag legitimate runtime/client user agents', () => {
diff --git a/apps/api/src/bots/index.ts b/packages/core/src/modules/ingest/src/bots/detect.ts
similarity index 100%
rename from apps/api/src/bots/index.ts
rename to packages/core/src/modules/ingest/src/bots/detect.ts
diff --git a/apps/api/scripts/get-bots.test.ts b/packages/core/src/modules/ingest/src/bots/generator.test.ts
similarity index 91%
rename from apps/api/scripts/get-bots.test.ts
rename to packages/core/src/modules/ingest/src/bots/generator.test.ts
index 64def0b2f..7b6fb4c56 100644
--- a/apps/api/scripts/get-bots.test.ts
+++ b/packages/core/src/modules/ingest/src/bots/generator.test.ts
@@ -7,8 +7,11 @@
* traffic. These run without a network call (main() is guarded on direct run).
*/
-import { describe, expect, it } from 'vitest';
-import { ALLOWLISTED_BOT_UA_TOKENS, stripAllowlistedTokens } from './get-bots';
+import { describe, expect, it } from 'bun:test';
+import {
+ ALLOWLISTED_BOT_UA_TOKENS,
+ stripAllowlistedTokens,
+} from '../../../../../scripts/get-bots';
describe('stripAllowlistedTokens', () => {
it('removes allowlisted identifiers from an anchored exact-match group', () => {
@@ -18,7 +21,7 @@ describe('stripAllowlistedTokens', () => {
it('preserves the genuinely-suspicious branches', () => {
const result = stripAllowlistedTokens(
- '^(?:chrome|node|Node\\.js|Zeus|ZmEu)$',
+ '^(?:chrome|node|Node\\.js|Zeus|ZmEu)$'
);
expect(result).toContain('Zeus');
expect(result).toContain('ZmEu');
diff --git a/apps/api/src/bots/header-signals.test.ts b/packages/core/src/modules/ingest/src/bots/header-signals.test.ts
similarity index 96%
rename from apps/api/src/bots/header-signals.test.ts
rename to packages/core/src/modules/ingest/src/bots/header-signals.test.ts
index 1a90b04c6..c7eebabfb 100644
--- a/apps/api/src/bots/header-signals.test.ts
+++ b/packages/core/src/modules/ingest/src/bots/header-signals.test.ts
@@ -1,4 +1,4 @@
-import { describe, expect, it } from 'vitest';
+import { describe, expect, it } from 'bun:test';
import { detectHeaderAnomalies } from './header-signals';
// Representative UA strings.
@@ -34,7 +34,7 @@ describe('detectHeaderAnomalies', () => {
detectHeaderAnomalies({
'user-agent': UA.chromeDesktop,
...realBrowserHeaders,
- }),
+ })
).toEqual([]);
});
@@ -45,7 +45,7 @@ describe('detectHeaderAnomalies', () => {
'sec-fetch-mode': 'cors',
'sec-fetch-site': 'cross-site',
'accept-language': 'en-US,en;q=0.5',
- }),
+ })
).toEqual([]);
});
@@ -56,7 +56,7 @@ describe('detectHeaderAnomalies', () => {
'sec-fetch-mode': 'cors',
'sec-fetch-site': 'cross-site',
'accept-language': 'en-US,en;q=0.9',
- }),
+ })
).toEqual([]);
});
@@ -73,7 +73,10 @@ describe('detectHeaderAnomalies', () => {
});
describe('does not apply to non-browser clients', () => {
- for (const [name, ua] of Object.entries({ curl: UA.curl, go: UA.goClient })) {
+ for (const [name, ua] of Object.entries({
+ curl: UA.curl,
+ go: UA.goClient,
+ })) {
it(`${name} → no reasons (handled by isServer/auth, not headers)`, () => {
expect(detectHeaderAnomalies({ 'user-agent': ua })).toEqual([]);
});
diff --git a/apps/api/src/bots/header-signals.ts b/packages/core/src/modules/ingest/src/bots/header-signals.ts
similarity index 100%
rename from apps/api/src/bots/header-signals.ts
rename to packages/core/src/modules/ingest/src/bots/header-signals.ts
diff --git a/apps/api/src/bots/suspicion.test.ts b/packages/core/src/modules/ingest/src/bots/suspicion.test.ts
similarity index 97%
rename from apps/api/src/bots/suspicion.test.ts
rename to packages/core/src/modules/ingest/src/bots/suspicion.test.ts
index 3214603ea..d7b4020af 100644
--- a/apps/api/src/bots/suspicion.test.ts
+++ b/packages/core/src/modules/ingest/src/bots/suspicion.test.ts
@@ -1,5 +1,5 @@
-import type { AsnInfo } from '@openpanel/core';
-import { describe, expect, it } from 'vitest';
+import { describe, expect, it } from 'bun:test';
+import type { AsnInfo } from '../../../../clients/geo';
import {
applyBotSuspicion,
stripBotProperties,
diff --git a/apps/api/src/bots/suspicion.ts b/packages/core/src/modules/ingest/src/bots/suspicion.ts
similarity index 98%
rename from apps/api/src/bots/suspicion.ts
rename to packages/core/src/modules/ingest/src/bots/suspicion.ts
index 4bc13134a..076a53ef2 100644
--- a/apps/api/src/bots/suspicion.ts
+++ b/packages/core/src/modules/ingest/src/bots/suspicion.ts
@@ -9,7 +9,7 @@
// Multiple reasons within the same category (two header anomalies) count once,
// so a single spoofed request can't inflate itself over the threshold.
-import type { AsnInfo } from '@openpanel/core';
+import type { AsnInfo } from '../../../../clients/geo';
import { detectHeaderAnomalies } from './header-signals';
export const BOT_CATEGORY_THRESHOLD = 2;
diff --git a/packages/core/src/modules/ingest/src/client-auth.test.ts b/packages/core/src/modules/ingest/src/client-auth.test.ts
new file mode 100644
index 000000000..b8c7b4b65
--- /dev/null
+++ b/packages/core/src/modules/ingest/src/client-auth.test.ts
@@ -0,0 +1,167 @@
+/**
+ * Tests for validateIngestRequest — V1's `validateSdkRequest`, moved with
+ * M8-002.
+ *
+ * The `mixan-*` case comes from apps/api/src/controllers/event.controller.test.ts:
+ * ADR-015 entry 5 is pending the `/event` usage metric, so the legacy header
+ * fallback stays. The `secretPresented` assertion is what V1 asserted as
+ * `request.clientSecretAuth` — the side channel the bot check reads.
+ */
+
+import { beforeAll, beforeEach, describe, expect, it, mock } from 'bun:test';
+
+const CLIENT_ID = '11111111-1111-4111-8111-111111111111';
+const PROJECT_ID = 'legacy-event-project';
+
+const client = {
+ id: CLIENT_ID,
+ projectId: PROJECT_ID,
+ organizationId: 'org-1',
+ type: 'write',
+ secret: null,
+ // Keeps this test about the header fallback rather than re-proving secret
+ // verification, which the auth contracts already cover.
+ ignoreCorsAndSecret: true,
+ project: {
+ id: PROJECT_ID,
+ filters: [],
+ allowUnsafeRevenueTracking: false,
+ cors: [],
+ },
+};
+
+const getClientByIdCached = mock(async (_id: string) => client);
+
+mock.module('../../client/client.service', () => ({ getClientByIdCached }));
+
+let validateIngestRequest: typeof import('./client-auth').validateIngestRequest;
+
+beforeAll(async () => {
+ ({ validateIngestRequest } = await import('./client-auth'));
+});
+
+beforeEach(() => {
+ getClientByIdCached.mockClear();
+});
+
+const body = { name: 'legacy_event', properties: { foo: 'bar' } };
+
+describe('validateIngestRequest', () => {
+ it('authenticates a client sent as mixan-client-id (ADR-015 entry 5 deferred)', async () => {
+ const result = await validateIngestRequest({
+ headers: {
+ 'mixan-client-id': CLIENT_ID,
+ 'mixan-client-secret': 'legacy-secret',
+ },
+ clientIp: '203.0.113.10',
+ body,
+ });
+
+ expect(getClientByIdCached).toHaveBeenCalledWith(CLIENT_ID);
+ expect(result.ok && result.client.id).toBe(CLIENT_ID);
+ expect(result.secretPresented).toBe(true);
+ });
+
+ it('accepts the openpanel-* headers and reports no secret when none was sent', async () => {
+ const result = await validateIngestRequest({
+ headers: { 'openpanel-client-id': CLIENT_ID },
+ clientIp: '203.0.113.10',
+ body,
+ });
+
+ expect(result.ok && result.client.id).toBe(CLIENT_ID);
+ expect(result.secretPresented).toBe(false);
+ });
+
+ it('falls back to the credentials carried in the body', async () => {
+ const result = await validateIngestRequest({
+ headers: {},
+ clientIp: '203.0.113.10',
+ body: { ...body, clientId: CLIENT_ID, clientSecret: 'from-body' },
+ });
+
+ expect(result.ok && result.client.id).toBe(CLIENT_ID);
+ expect(result.secretPresented).toBe(true);
+ });
+
+ // A refusal is RETURNED, never thrown: the caller builds the error so its
+ // stack is captured outside core's bundle chunk (see client-auth.ts).
+ it('refuses a missing client id without throwing', async () => {
+ const result = await validateIngestRequest({
+ headers: {},
+ clientIp: '1.2.3.4',
+ body,
+ });
+
+ expect(result).toMatchObject({
+ ok: false,
+ message: 'Ingestion: Missing client id',
+ });
+ expect(getClientByIdCached).not.toHaveBeenCalled();
+ });
+
+ it('refuses a client id that is not a UUIDv4', async () => {
+ const result = await validateIngestRequest({
+ headers: { 'openpanel-client-id': 'not-a-uuid' },
+ clientIp: '1.2.3.4',
+ body,
+ });
+
+ expect(result).toMatchObject({
+ ok: false,
+ message: 'Ingestion: Client ID must be a valid UUIDv4',
+ });
+ });
+
+ it('redacts the presented secret in the refusal payload', async () => {
+ const result = await validateIngestRequest({
+ headers: {
+ 'openpanel-client-id': 'not-a-uuid',
+ 'openpanel-client-secret': 'abcdefghijklmnop',
+ },
+ clientIp: '1.2.3.4',
+ body,
+ });
+
+ expect(result.ok).toBe(false);
+ expect(result.ok === false && result.payload.clientSecret).toBe(
+ 'abcde...lmnop'
+ );
+ expect(result.secretPresented).toBe(true);
+ });
+
+ it('refuses revenue tracking without a client secret', async () => {
+ const result = await validateIngestRequest({
+ headers: { 'openpanel-client-id': CLIENT_ID },
+ clientIp: '1.2.3.4',
+ body: { ...body, properties: { __revenue: 100 } },
+ });
+
+ expect(result).toMatchObject({
+ ok: false,
+ message:
+ 'Ingestion: Revenue tracking is not allowed without a client secret',
+ });
+ });
+
+ it('blocks an ip listed in the project filters', async () => {
+ getClientByIdCached.mockResolvedValueOnce({
+ ...client,
+ project: {
+ ...client.project,
+ filters: [{ type: 'ip', ip: '203.0.113.10' }],
+ },
+ } as unknown as typeof client);
+
+ const result = await validateIngestRequest({
+ headers: { 'openpanel-client-id': CLIENT_ID },
+ clientIp: '203.0.113.10',
+ body,
+ });
+
+ expect(result).toMatchObject({
+ ok: false,
+ message: 'Ingestion: IP address is blocked by project filter',
+ });
+ });
+});
diff --git a/packages/core/src/modules/ingest/src/client-auth.ts b/packages/core/src/modules/ingest/src/client-auth.ts
new file mode 100644
index 000000000..089b43544
--- /dev/null
+++ b/packages/core/src/modules/ingest/src/client-auth.ts
@@ -0,0 +1,214 @@
+// The ingest tier of caller authentication (ADR-011 A-i's "ingest extension",
+// ported verbatim from apps/api/src/utils/auth.ts's `validateSdkRequest`).
+// apps/api's `clientHook` is a thin delegate over this; core's `clientAuth`
+// macro reaches it through http/client-auth.ts when a route asks for
+// `{ ingest: true }`.
+//
+// Everything here is behaviour V1 relies on and none of it is decoration: the
+// `ignoreCorsAndSecret` short-circuit, the unanchored wildcard origin regex,
+// CORS-before-secret ordering, the ip and profile_id project filters, the
+// `__revenue` gate, the body-field credential fallback, the `mixan-*` header
+// fallback (ADR-015 entry 5 is deferred, not taken) and the 5-minute verify
+// cache whose key holds base64(<plaintext secret>) — a recorded finding
+// ADR-011 leaves to Carl, not to a port.
+
+import { getCache } from '@openpanel/redis';
+import type {
+ IProjectFilterIp,
+ IProjectFilterProfileId,
+} from '@openpanel/validation';
+import { path } from 'ramda';
+import { verifyPassword } from '../../../shared/crypto';
+import { headerValue, type IngestHeaders } from './headers';
+
+export type { IngestHeaders } from './headers';
+
+import {
+ getClientByIdCached,
+ type IServiceClientWithProject,
+} from '../../client/client.service';
+
+const CLIENT_ID_UUID_REGEX =
+ /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/;
+const VERIFY_CACHE_SECONDS = 60 * 5;
+const REDACTED_SECRET_EDGE_LENGTH = 5;
+const DOMAIN_PROTOCOL = /https?:\/\//;
+const DOMAIN_TRAILING_SLASH = /\/$/;
+const DOMAIN_DOT = /\./g;
+const DOMAIN_WILDCARD = /\*/g;
+
+export interface IngestAuthErrorPayload {
+ clientId?: string;
+ clientSecret?: string;
+ origin?: string;
+}
+
+/**
+ * A refusal is RETURNED, never thrown.
+ *
+ * Two reasons. It keeps the transport in charge of its own status codes, as
+ * `TrackOutcome` does. And it keeps the error OBJECT out of this bundle:
+ * apps/api's dist splits core into one ~11 MB chunk with a ~20 MB source map,
+ * and `source-map-support` parses that map the first time a stack trace
+ * crosses it — measured at ~1.2s on the first failed ingest auth, on the hot
+ * path, which the SDK wire contract catches as a phantom retry delay.
+ * `SdkAuthError` is therefore constructed by the caller.
+ */
+export type IngestAuthOutcome =
+ | {
+ ok: true;
+ client: IServiceClientWithProject;
+ /**
+ * Whether a client secret was PRESENTED, set before it is verified. V1
+ * sets `req.clientSecretAuth` at parse time and the bot hook reads it
+ * afterwards as "this is a server-side SDK, never a bot" — the side
+ * channel survives as a field on the outcome (ADR-011 A-i).
+ */
+ secretPresented: boolean;
+ }
+ | {
+ ok: false;
+ message: string;
+ payload: IngestAuthErrorPayload;
+ secretPresented: boolean;
+ };
+
+const cleanDomain = (domain: string) =>
+ domain
+ .replace('www.', '')
+ .replace(DOMAIN_PROTOCOL, '')
+ .replace(DOMAIN_TRAILING_SLASH, '');
+
+function isOriginAllowed(cors: string[], origin: string | undefined): boolean {
+ const domainAllowed = cors.find((domain) => {
+ const cleanedDomain = cleanDomain(domain);
+ // support wildcard domains `*.foo.com`
+ if (cleanedDomain.includes('*')) {
+ const regex = new RegExp(
+ `${cleanedDomain.replace(DOMAIN_DOT, '\\.').replace(DOMAIN_WILDCARD, '.+?')}`
+ );
+
+ return regex.test(origin || '');
+ }
+
+ return cleanedDomain === cleanDomain(origin || '');
+ });
+
+ if (domainAllowed) {
+ return true;
+ }
+
+ return cors.includes('*') && !!origin;
+}
+
+export async function validateIngestRequest({
+ headers,
+ clientIp,
+ body,
+}: {
+ headers: IngestHeaders;
+ clientIp: string | undefined;
+ body: unknown;
+}): Promise<IngestAuthOutcome> {
+ const clientIdNew = headerValue(headers, 'openpanel-client-id');
+ const clientIdOld = headerValue(headers, 'mixan-client-id');
+ const clientSecretNew = headerValue(headers, 'openpanel-client-secret');
+ const clientSecretOld = headerValue(headers, 'mixan-client-secret');
+ const clientIdFromBody = path<string | undefined>(['clientId'], body);
+ const clientSecretFromBody = path<string | undefined>(['clientSecret'], body);
+ const clientId = clientIdNew || clientIdOld || clientIdFromBody;
+ const clientSecret =
+ clientSecretNew || clientSecretOld || clientSecretFromBody;
+ const origin = headerValue(headers, 'origin');
+ const secretPresented = !!clientSecret;
+
+ const refuse = (message: string): IngestAuthOutcome => ({
+ ok: false,
+ message,
+ payload: {
+ clientId,
+ clientSecret:
+ typeof clientSecret === 'string'
+ ? `${clientSecret.slice(0, REDACTED_SECRET_EDGE_LENGTH)}...${clientSecret.slice(-REDACTED_SECRET_EDGE_LENGTH)}`
+ : 'none',
+ origin,
+ },
+ secretPresented,
+ });
+
+ if (!clientId) {
+ return refuse('Ingestion: Missing client id');
+ }
+
+ if (!CLIENT_ID_UUID_REGEX.test(clientId)) {
+ return refuse('Ingestion: Client ID must be a valid UUIDv4');
+ }
+
+ const client = await getClientByIdCached(clientId);
+
+ if (!client) {
+ return refuse('Ingestion: Invalid client id');
+ }
+
+ if (!client.project) {
+ return refuse('Ingestion: Client has no project');
+ }
+
+ // Filter out blocked IPs
+ const ipFilter = client.project.filters.filter(
+ (filter): filter is IProjectFilterIp => filter.type === 'ip'
+ );
+ if (ipFilter.some((filter) => filter.ip === clientIp)) {
+ return refuse('Ingestion: IP address is blocked by project filter');
+ }
+
+ // Filter out blocked profile ids
+ const profileFilter = client.project.filters.filter(
+ (filter): filter is IProjectFilterProfileId => filter.type === 'profile_id'
+ );
+ const profileId =
+ path<string | undefined>(['payload', 'profileId'], body) || // Track handler
+ path<string | undefined>(['profileId'], body); // Event handler
+
+ if (profileFilter.some((filter) => filter.profileId === profileId)) {
+ return refuse('Ingestion: Profile id is blocked by project filter');
+ }
+
+ const revenue =
+ path(['payload', 'properties', '__revenue'], body) ??
+ path(['properties', '__revenue'], body);
+
+ // Only allow revenue tracking if it was sent with a client secret
+ // or if the project has allowUnsafeRevenueTracking enabled
+ if (
+ !(client.project.allowUnsafeRevenueTracking || clientSecret) &&
+ typeof revenue !== 'undefined'
+ ) {
+ return refuse(
+ 'Ingestion: Revenue tracking is not allowed without a client secret'
+ );
+ }
+
+ if (client.ignoreCorsAndSecret) {
+ return { ok: true, client, secretPresented };
+ }
+
+ if (client.project.cors && isOriginAllowed(client.project.cors, origin)) {
+ return { ok: true, client, secretPresented };
+ }
+
+ const secret = client.secret;
+ if (secret && clientSecret) {
+ const isVerified = await getCache(
+ `client:auth:${clientId}:${Buffer.from(clientSecret).toString('base64')}`,
+ VERIFY_CACHE_SECONDS,
+ async () => await verifyPassword(clientSecret, secret),
+ true
+ );
+ if (isVerified) {
+ return { ok: true, client, secretPresented };
+ }
+ }
+
+ return refuse('Ingestion: Invalid cors or secret');
+}
diff --git a/packages/core/src/modules/ingest/src/deduplicate.ts b/packages/core/src/modules/ingest/src/deduplicate.ts
new file mode 100644
index 000000000..e6df816e1
--- /dev/null
+++ b/packages/core/src/modules/ingest/src/deduplicate.ts
@@ -0,0 +1,34 @@
+// Ported from apps/api/src/utils/deduplicate.ts (M8-002). The web SDK can fire
+// the same event twice (pagehide + visibilitychange); a short-lived Redis lock
+// keyed on the whole payload collapses the pair.
+
+import { getLock } from '@openpanel/redis';
+import fastJsonStableHash from 'fast-json-stable-hash';
+
+// Live Redis key prefix — byte-identical to V1's, so a V1 and a V2 process
+// deduplicate against the same locks during the rollout.
+const DEDUPLICATE_KEY_PREFIX = 'fastify:deduplicate:';
+const DEDUPLICATE_TTL_MS = 100;
+
+export async function isDuplicatedEvent({
+ ip,
+ origin,
+ payload,
+ projectId,
+}: {
+ ip: string;
+ origin: string;
+ payload: Record<string, unknown>;
+ projectId: string;
+}): Promise<boolean> {
+ const locked = await getLock(
+ `${DEDUPLICATE_KEY_PREFIX}${fastJsonStableHash.hash(
+ { ...payload, ip, origin, projectId },
+ 'md5'
+ )}`,
+ '1',
+ DEDUPLICATE_TTL_MS
+ );
+
+ return !locked;
+}
diff --git a/packages/core/src/modules/ingest/src/device-id.test.ts b/packages/core/src/modules/ingest/src/device-id.test.ts
new file mode 100644
index 000000000..26d57db95
--- /dev/null
+++ b/packages/core/src/modules/ingest/src/device-id.test.ts
@@ -0,0 +1,125 @@
+/**
+ * Tests for getDeviceId's session resolution against the device-keyed session
+ * store. The id is reused from the live blob only while it's within the idle
+ * window; a lingering (past-window) blob must NOT be reused — see the guard in
+ * device-id.ts. Override device ids resolve through the same path with a
+ * single read.
+ *
+ * Moved from apps/api/src/utils/ids.test.ts with M8-002. The buffer is now an
+ * argument, so the stub replaces V1's spy on the module singleton — same
+ * assertions, and no Redis either way.
+ */
+
+import { afterEach, describe, expect, it, mock } from 'bun:test';
+import type { IClickhouseSession } from '../../session/session.service';
+import { formatClickhouseDate } from '../../session/src/dates';
+import { getDeviceId, type SessionBufferReader } from './device-id';
+
+const NOW = new Date('2026-06-08T12:00:00.000Z').getTime();
+const MINUTE = 60 * 1000;
+const SALTS = { current: 'salt-current', previous: 'salt-previous' };
+const BASE = {
+ projectId: 'proj-1',
+ ip: '1.2.3.4',
+ ua: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/148.0.0.0',
+ salts: SALTS,
+ eventTimeMs: NOW,
+};
+
+// withinIdleWindow only reads `id` + `ended_at`; the rest is irrelevant here.
+const fakeSession = (id: string, endedAtMs: number): IClickhouseSession =>
+ ({
+ id,
+ ended_at: formatClickhouseDate(new Date(endedAtMs)),
+ }) as unknown as IClickhouseSession;
+
+function stubBuffer(session: IClickhouseSession | null) {
+ const getExistingSession = mock(async () => session);
+ return {
+ getExistingSession,
+ buffer: { getExistingSession } as unknown as SessionBufferReader,
+ };
+}
+
+afterEach(() => {
+ mock.restore();
+});
+
+describe('getDeviceId — session resolution', () => {
+ it('mints a deterministic, stable id when no session exists', async () => {
+ const { getExistingSession, buffer } = stubBuffer(null);
+
+ const a = await getDeviceId({
+ ...BASE,
+ overrideDeviceId: 'cookie-abc',
+ sessionBuffer: buffer,
+ });
+ const b = await getDeviceId({
+ ...BASE,
+ overrideDeviceId: 'cookie-abc',
+ sessionBuffer: buffer,
+ });
+
+ expect(a.sessionId).toBeTruthy();
+ expect(b.sessionId).toBe(a.sessionId); // same window → same id
+ expect(a.deviceId).toBe('cookie-abc');
+ expect(getExistingSession).toHaveBeenCalled();
+ });
+
+ it('reuses the live session id when within the idle window', async () => {
+ const { buffer } = stubBuffer(fakeSession('sess-live', NOW - MINUTE));
+
+ const result = await getDeviceId({
+ ...BASE,
+ overrideDeviceId: 'cookie-abc',
+ sessionBuffer: buffer,
+ });
+
+ expect(result).toEqual({ deviceId: 'cookie-abc', sessionId: 'sess-live' });
+ });
+
+ it('does NOT reuse a session that has lingered past the idle window', async () => {
+ // ended 31 min ago — past the 30 min timeout; the reaper just hasn't
+ // closed it yet (blobs have no TTL).
+ const { buffer } = stubBuffer(fakeSession('sess-stale', NOW - 31 * MINUTE));
+
+ const result = await getDeviceId({
+ ...BASE,
+ overrideDeviceId: 'cookie-abc',
+ sessionBuffer: buffer,
+ });
+
+ expect(result.deviceId).toBe('cookie-abc');
+ expect(result.sessionId).toBeTruthy();
+ expect(result.sessionId).not.toBe('sess-stale'); // a fresh id, not the stale one
+ });
+
+ it('reads the store once for an override (no redundant previous lookup)', async () => {
+ const { getExistingSession, buffer } = stubBuffer(null);
+
+ await getDeviceId({
+ ...BASE,
+ overrideDeviceId: 'cookie-abc',
+ sessionBuffer: buffer,
+ });
+
+ expect(getExistingSession).toHaveBeenCalledTimes(1);
+ expect(getExistingSession).toHaveBeenCalledWith({
+ projectId: 'proj-1',
+ deviceId: 'cookie-abc',
+ });
+ });
+
+ it('checks both current and previous salt windows for internal ids', async () => {
+ const { getExistingSession, buffer } = stubBuffer(null);
+
+ await getDeviceId({ ...BASE, sessionBuffer: buffer }); // no override → IP+UA hashing
+
+ expect(getExistingSession).toHaveBeenCalledTimes(2);
+ const deviceIds = getExistingSession.mock.calls.map((call) => {
+ const [args] = call as unknown as [{ deviceId?: string }];
+ return args.deviceId ?? '';
+ });
+ expect(new Set(deviceIds).size).toBe(2); // distinct current/previous hashes
+ });
+});
diff --git a/apps/api/src/utils/ids.ts b/packages/core/src/modules/ingest/src/device-id.ts
similarity index 74%
rename from apps/api/src/utils/ids.ts
rename to packages/core/src/modules/ingest/src/device-id.ts
index fe7ab12ff..2ce76937f 100644
--- a/apps/api/src/utils/ids.ts
+++ b/packages/core/src/modules/ingest/src/device-id.ts
@@ -1,11 +1,32 @@
+// Ported from apps/api/src/utils/ids.ts (M8-002). Device and session identity
+// for an incoming event: the salted device id, and the session id the API
+// answers with synchronously while the worker persists the blob async.
+
import crypto from 'node:crypto';
-import { generateDeviceId } from '@openpanel/core';
-import type { IClickhouseSession } from '@openpanel/db';
-import {
- convertClickhouseDateToJs,
- SESSION_TIMEOUT_MS,
- sessionBuffer,
-} from '@openpanel/db';
+import type { SessionBuffer } from '../../../buffers/session-buffer';
+import { SESSION_TIMEOUT_MS } from '../../../buffers/session-buffer';
+import { generateDeviceId } from '../../../shared/profileId';
+import type { IClickhouseSession } from '../../session/session.service';
+import { convertClickhouseDateToJs } from '../../session/src/dates';
+
+/** Only the read this module needs, so a caller may pass a narrower stub. */
+export type SessionBufferReader = Pick<SessionBuffer, 'getExistingSession'>;
+
+export interface DeviceIdResult {
+ deviceId: string;
+ sessionId: string;
+}
+
+const SESSION_ID_BYTES = 16;
+const SESSION_ID_MIN_BYTES = 8;
+const SESSION_ID_MAX_BYTES = 32;
+const DEFAULT_SESSION_WINDOW_MS = 5 * 60 * 1000;
+const DEFAULT_SESSION_GRACE_MS = 60 * 1000;
+const SESSION_GRACE_CEILING_MS = 5000;
+const SESSION_GRACE_WINDOW_DIVISOR = 6;
+const BASE64_PLUS = /\+/g;
+const BASE64_SLASH = /\//g;
+const BASE64_PADDING = /=+$/g;
export async function getDeviceId({
projectId,
@@ -14,6 +35,7 @@ export async function getDeviceId({
salts,
overrideDeviceId,
eventTimeMs,
+ sessionBuffer,
}: {
projectId: string;
ip: string;
@@ -23,7 +45,8 @@ export async function getDeviceId({
/** Event timestamp (ms). Used to decide whether an existing session is
* still within its idle window. Defaults to `Date.now()`. */
eventTimeMs?: number;
-}) {
+ sessionBuffer: SessionBufferReader;
+}): Promise<DeviceIdResult> {
if (overrideDeviceId) {
// A caller-supplied device id is stable (no salt rotation), so it's the only
// candidate — resolve it through the same path as internal ids.
@@ -31,6 +54,7 @@ export async function getDeviceId({
projectId,
deviceIds: [overrideDeviceId],
eventTimeMs: eventTimeMs ?? Date.now(),
+ sessionBuffer,
});
}
@@ -55,14 +79,10 @@ export async function getDeviceId({
projectId,
deviceIds: [currentDeviceId, previousDeviceId],
eventTimeMs: eventTimeMs ?? Date.now(),
+ sessionBuffer,
});
}
-interface DeviceIdResult {
- deviceId: string;
- sessionId: string;
-}
-
/**
* Returns true when an existing session is recent enough that the incoming
* event should EXTEND it rather than start a new session.
@@ -85,12 +105,14 @@ async function getInfoFromSession({
projectId,
deviceIds,
eventTimeMs,
+ sessionBuffer,
}: {
projectId: string;
/** Candidate device ids in priority order (e.g. [current, previous] salt
* windows, or just [override]). Deduped; the first is canonical. */
deviceIds: string[];
eventTimeMs: number;
+ sessionBuffer: SessionBufferReader;
}): Promise<DeviceIdResult> {
const candidates = [...new Set(deviceIds.filter(Boolean))];
const primary = candidates[0] ?? '';
@@ -105,9 +127,9 @@ async function getInfoFromSession({
)
);
- for (const [i, session] of sessions.entries()) {
+ for (const [index, session] of sessions.entries()) {
if (session && withinIdleWindow(session, eventTimeMs)) {
- return { deviceId: candidates[i]!, sessionId: session.id };
+ return { deviceId: candidates[index]!, sessionId: session.id };
}
}
} catch (error) {
@@ -129,7 +151,10 @@ async function getInfoFromSession({
projectId,
deviceId: primary,
eventMs: eventTimeMs,
- graceMs: Math.min(5000, Math.floor(SESSION_TIMEOUT_MS / 6)),
+ graceMs: Math.min(
+ SESSION_GRACE_CEILING_MS,
+ Math.floor(SESSION_TIMEOUT_MS / SESSION_GRACE_WINDOW_DIVISOR)
+ ),
windowMs: SESSION_TIMEOUT_MS,
}),
};
@@ -143,7 +168,7 @@ async function getInfoFromSession({
* - graceMs: 1 minute by default (events in first minute of a bucket map to previous bucket)
* - Output: base64url, 128-bit (16 bytes) truncated from SHA-256
*/
-function getSessionId(params: {
+export function getSessionId(params: {
projectId: string;
deviceId: string;
eventMs?: number; // use event timestamp; defaults to Date.now()
@@ -155,9 +180,9 @@ function getSessionId(params: {
projectId,
deviceId,
eventMs = Date.now(),
- windowMs = 5 * 60 * 1000,
- graceMs = 60 * 1000,
- bytes = 16,
+ windowMs = DEFAULT_SESSION_WINDOW_MS,
+ graceMs = DEFAULT_SESSION_GRACE_MS,
+ bytes = SESSION_ID_BYTES,
} = params;
if (!projectId) {
@@ -172,7 +197,7 @@ function getSessionId(params: {
if (graceMs < 0 || graceMs >= windowMs) {
throw new Error('graceMs must be >= 0 and < windowMs');
}
- if (bytes < 8 || bytes > 32) {
+ if (bytes < SESSION_ID_MIN_BYTES || bytes > SESSION_ID_MAX_BYTES) {
throw new Error('bytes must be between 8 and 32');
}
@@ -190,7 +215,7 @@ function getSessionId(params: {
// base64url
return truncated
.toString('base64')
- .replace(/\+/g, '-')
- .replace(/\//g, '_')
- .replace(/=+$/g, '');
+ .replace(BASE64_PLUS, '-')
+ .replace(BASE64_SLASH, '_')
+ .replace(BASE64_PADDING, '');
}
diff --git a/packages/core/src/modules/ingest/src/headers.ts b/packages/core/src/modules/ingest/src/headers.ts
new file mode 100644
index 000000000..6a5231b46
--- /dev/null
+++ b/packages/core/src/modules/ingest/src/headers.ts
@@ -0,0 +1,24 @@
+// A lowercase header record — Fastify's `IncomingHttpHeaders`, or a `Headers`
+// flattened with `Object.fromEntries`. The ingest path reads headers exactly
+// as V1 read them, so no transport has to build a `Headers` from a raw record
+// (which throws on a malformed name V1 simply ignored).
+
+export type IngestHeaders = Record<string, string | string[] | undefined>;
+
+export function headerValue(
+ headers: IngestHeaders,
+ name: string
+): string | undefined {
+ const value = headers[name];
+ return Array.isArray(value) ? value[0] : value;
+}
+
+/** `Headers` -> the record shape the ingest path reads. Duplicated values are
+ * joined by `Headers` itself, exactly as a Fastify record would hold them. */
+export function toIngestHeaders(headers: Headers): IngestHeaders {
+ const record: IngestHeaders = {};
+ for (const [name, value] of headers) {
+ record[name] = value;
+ }
+ return record;
+}
diff --git a/packages/core/src/modules/ingest/src/incoming-event.ts b/packages/core/src/modules/ingest/src/incoming-event.ts
new file mode 100644
index 000000000..5a80de335
--- /dev/null
+++ b/packages/core/src/modules/ingest/src/incoming-event.ts
@@ -0,0 +1,33 @@
+// The Kafka topic payload and the producer signature over it.
+//
+// This is a WIRE FORMAT, not an internal type: a rolling deploy overlaps a V1
+// producer with a V2 consumer on the same topic, so it stays byte-compatible
+// with @openpanel/queue's `EventsQueuePayloadIncomingEvent['payload']`
+// (ADR-004). A leaf file so `context.ts` can name the producer without
+// reaching into the ingest service.
+
+import type { GeoLocation } from '../../../clients/geo';
+import type { parseUserAgent } from '../../../shared/parser-user-agent';
+import type { ITrackPayload } from '../ingest.constants';
+
+export interface IncomingEventPayload {
+ // Minted by the producer (/track, /event) so a Kafka redelivery becomes an
+ // identical-id row instead of a new one. Optional on the wire: a V1
+ // producer omits it and a V1 consumer ignores it.
+ id?: string;
+ projectId: string;
+ event: ITrackPayload & {
+ timestamp: string | number;
+ isTimestampFromThePast: boolean;
+ };
+ uaInfo: ReturnType<typeof parseUserAgent>;
+ geo: GeoLocation;
+ headers: Record<string, string | undefined>;
+ deviceId: string;
+ sessionId: string;
+}
+
+export type IncomingEventProducer = (
+ payload: IncomingEventPayload,
+ partitionKey: string
+) => Promise<unknown>;
diff --git a/packages/core/src/modules/insight/insight.jobs.test.ts b/packages/core/src/modules/insight/insight.jobs.test.ts
index cb5793fdf..7d7ecad70 100644
--- a/packages/core/src/modules/insight/insight.jobs.test.ts
+++ b/packages/core/src/modules/insight/insight.jobs.test.ts
@@ -43,6 +43,7 @@ function stubJobCtx(
gsc: {} as Services['gsc'],
cohort: {} as Services['cohort'],
import: {} as Services['import'],
+ ingest: {} as Services['ingest'],
organization: {} as Services['organization'],
onboarding: {} as Services['onboarding'],
session: {} as Services['session'],
@@ -132,6 +133,7 @@ test('insightsDaily fans out one insightsProject enqueue per candidate, jobId de
gsc: {} as Services['gsc'],
cohort: {} as Services['cohort'],
import: {} as Services['import'],
+ ingest: {} as Services['ingest'],
organization: {} as Services['organization'],
onboarding: {} as Services['onboarding'],
session: {} as Services['session'],
diff --git a/packages/core/src/modules/notification/notification.jobs.test.ts b/packages/core/src/modules/notification/notification.jobs.test.ts
index 5c328f3c7..0f821bcc4 100644
--- a/packages/core/src/modules/notification/notification.jobs.test.ts
+++ b/packages/core/src/modules/notification/notification.jobs.test.ts
@@ -32,6 +32,7 @@ function stubJobCtx(notification: Partial<NotificationService>): JobCtx {
cohort: {} as Services['cohort'],
gsc: {} as Services['gsc'],
import: {} as Services['import'],
+ ingest: {} as Services['ingest'],
organization: {} as Services['organization'],
onboarding: {} as Services['onboarding'],
session: {} as Services['session'],
diff --git a/packages/core/src/modules/onboarding/onboarding.jobs.test.ts b/packages/core/src/modules/onboarding/onboarding.jobs.test.ts
index 355dec217..23ecb07cc 100644
--- a/packages/core/src/modules/onboarding/onboarding.jobs.test.ts
+++ b/packages/core/src/modules/onboarding/onboarding.jobs.test.ts
@@ -34,6 +34,7 @@ function stubJobCtx(
insight: {} as Services['insight'],
gsc: {} as Services['gsc'],
import: {} as Services['import'],
+ ingest: {} as Services['ingest'],
cohort: {} as Services['cohort'],
organization: {} as Services['organization'],
onboarding: onboarding as OnboardingService,
diff --git a/packages/core/src/modules/organization/organization.jobs.test.ts b/packages/core/src/modules/organization/organization.jobs.test.ts
index 0b22dfa20..c92495b5e 100644
--- a/packages/core/src/modules/organization/organization.jobs.test.ts
+++ b/packages/core/src/modules/organization/organization.jobs.test.ts
@@ -37,6 +37,7 @@ function stubJobCtx(
insight: {} as Services['insight'],
gsc: {} as Services['gsc'],
import: {} as Services['import'],
+ ingest: {} as Services['ingest'],
cohort: {} as Services['cohort'],
organization: organization as OrganizationService,
onboarding: {} as Services['onboarding'],
diff --git a/packages/core/src/modules/profile/profile.service.ts b/packages/core/src/modules/profile/profile.service.ts
index d85c590db..762d432b6 100644
--- a/packages/core/src/modules/profile/profile.service.ts
+++ b/packages/core/src/modules/profile/profile.service.ts
@@ -569,7 +569,9 @@ export async function getProfileValues(input: {
// ---- apps/api's /profile controller bodies
export interface IdentifyProfileInput {
- profileId: string;
+ /** `string | number` because /track's identify payload allows both
+ * (ingest.constants' `IProfileId`) and `upsertProfile` stores either. */
+ profileId: string | number;
firstName?: string;
lastName?: string;
email?: string;
diff --git a/packages/core/src/rest.routes.ts b/packages/core/src/rest.routes.ts
index bd5ea742c..80563b023 100644
--- a/packages/core/src/rest.routes.ts
+++ b/packages/core/src/rest.routes.ts
@@ -15,6 +15,7 @@ import { exportRoutes, insightsRoutes } from './modules/export/export.routes';
import { gscRoutes } from './modules/gsc/gsc.routes';
import { healthRoutes } from './modules/health/health.routes';
import { importRoutes } from './modules/import/import.routes';
+import { ingestRoutes } from './modules/ingest/ingest.routes';
import { integrationRoutes } from './modules/integration/integration.routes';
import { mcpRoutes } from './modules/mcp/mcp.routes';
import { miscRoutes } from './modules/misc/misc.routes';
@@ -47,9 +48,12 @@ const OPENAPI_EXCLUDED_PATHS = ['/metrics'];
// stub that always returns null (import.routes.ts's header), and main.ts
// does not mount `publicApiRoutes` until a real `AppDeps` exists (P3/P4/P8).
// tools' `/site-checker` + `/ip-lookup` (M7-008) joined it too — unauthenticated
-// like V1's, per the module map.
+// like V1's, per the module map. ingest's `/track` + `/track/device-id`
+// (M8-002) lead the list: the hot path, `clientAuth: { ingest: true }`, and
+// the one route surface whose hook ORDER is part of the contract.
export const publicApiRoutes = (deps: AppDeps) =>
new Elysia({ name: 'core/public-api-routes' })
+ .use(ingestRoutes(deps))
.use(importRoutes(deps))
.use(projectRoutes(deps))
.use(clientRoutes(deps))
diff --git a/packages/core/src/services.ts b/packages/core/src/services.ts
index 6ffaf85fa..4e9fcc43b 100644
--- a/packages/core/src/services.ts
+++ b/packages/core/src/services.ts
@@ -28,6 +28,10 @@ import {
createImportService,
type ImportService,
} from './modules/import/import.service';
+import {
+ createIngestService,
+ type IngestService,
+} from './modules/ingest/ingest.service';
import {
createInsightService,
type InsightService,
@@ -78,6 +82,7 @@ export interface Services {
gsc: GscService;
cohort: CohortService;
import: ImportService;
+ ingest: IngestService;
organization: OrganizationService;
onboarding: OnboardingService;
notification: NotificationService;
@@ -99,6 +104,7 @@ export function createServices(deps: ServiceDeps): Services {
gsc: createGscService(deps),
cohort: createCohortService(deps),
import: createImportService(deps),
+ ingest: createIngestService(deps),
organization: createOrganizationService(deps),
onboarding: createOnboardingService(deps),
notification: createNotificationService(deps),
diff --git a/packages/core/test/http-fixtures.ts b/packages/core/test/http-fixtures.ts
index b7c8b510b..f5ed3407a 100644
--- a/packages/core/test/http-fixtures.ts
+++ b/packages/core/test/http-fixtures.ts
@@ -44,6 +44,7 @@ export function stubAppDeps(): AppDepsStub {
clients: {},
buffers: {} as Buffers,
producers,
+ produceIncomingEvent: () => Promise.resolve(),
logger,
config: { selfHosted: false },
},
diff --git a/packages/core/test/request-id-end-to-end.test.ts b/packages/core/test/request-id-end-to-end.test.ts
index 477ae8417..1a8712e6d 100644
--- a/packages/core/test/request-id-end-to-end.test.ts
+++ b/packages/core/test/request-id-end-to-end.test.ts
@@ -146,6 +146,7 @@ function stubDeps() {
clients: {},
buffers: {} as Buffers,
producers: producers as unknown as QueueProducerHandle,
+ produceIncomingEvent: () => Promise.resolve(),
logger: bindingLogger(lines),
config: { selfHosted: false },
};
diff --git a/packages/core/test/rpc-fixtures.ts b/packages/core/test/rpc-fixtures.ts
index 619092877..4dd67c904 100644
--- a/packages/core/test/rpc-fixtures.ts
+++ b/packages/core/test/rpc-fixtures.ts
@@ -92,6 +92,7 @@ export function stubHttpCtx(
clients: {},
buffers: {} as Buffers,
producers: createRecordingProducers(queues),
+ produceIncomingEvent: () => Promise.resolve(),
logger,
config: { selfHosted: false },
};
diff --git a/packages/validation/src/event-blocklist.ts b/packages/validation/src/event-blocklist.ts
deleted file mode 100644
index e95540659..000000000
--- a/packages/validation/src/event-blocklist.ts
+++ /dev/null
@@ -1,120 +0,0 @@
-const MAX_EVENT_LENGTH = 80;
-
-// Substrings that indicate attack/spam payloads
-const BLOCKED_SUBSTRINGS = [
- // === Security Scanner Domains ===
- 'oastify.com',
- 'burpcollaborator',
- 'interact.sh',
- 'oast.me',
-
- // === SQL Injection ===
- 'pg_sleep',
- 'waitfor delay',
- 'xp_dirtree',
- 'load_file(',
- 'extractvalue(',
- 'dbms_pipe.receive_message',
- 'union select',
-
- // === Command Injection ===
- 'nslookup ',
- '/bin/sleep',
- '/bin/bash',
- 'cmd.exe',
- 'wget+http',
- 'wget http',
- 'chmod+777',
- 'chmod 777',
-
- // === Java/Code Execution ===
- 'processbuilder',
- 'runtime.getruntime',
- 'java.lang.processbuilder',
- 'eval-stdin.php',
-
- // === Path Traversal ===
- '../',
- '..\\',
- '%2e%2e',
- '%u002e%u002e',
- '/etc/passwd',
- '/etc/shadow',
- 'win.ini',
- 'system.ini',
-
- // === Template/SSTI Injection ===
- '${',
- '%{',
-
- // === XXE / XML Attacks ===
- '<!doctype',
- '<!entity',
- '<xi:include',
- 'xsi:schemalocation',
-
- // === SMTP Header Injection ===
- '\r\n',
- 'bcc:',
-
- // === Common File Scanning (paths as events) ===
- 'phpinfo.php',
- 'wp-config.php',
- '.git/config',
- '.env.backup',
- '.env.bak',
- '/vendor/phpunit/',
-
- // === Malware/Botnet Indicators ===
- 'mozi.m',
- '/setup.cgi?',
- '/cgi-bin/',
-
- // === Ruby Object Inspection Leaks ===
- '#<article:0x',
- '#<video:0x',
- '#<brand:0x',
-
- // === SQL Injection Patterns ===
- 'exec master.dbo',
- 'declare @',
- "' and '",
- "' or '",
- "') or ",
- "')and ",
-];
-
-// Patterns that indicate the "event" is actually a URL path being scanned
-const PATH_SCAN_PATTERNS = [
- /^\/[a-z_-]+\.(php|env|yml|yaml|json|xml|config|ini|bak|sql|log)/i,
- /^\/\.[a-z]/i, // Hidden files like /.env, /.git
- /^\/(wp-|wordpress)/i, // WordPress scanning
- /^\/phpmyadmin/i,
- /^\/.+\.php$/i, // Any .php path
-];
-
-/**
- * Check if an event name should be blocked
- * @param name - The event name to check
- * @returns true if the event name should be blocked, false otherwise
- */
-export function isBlockedEventName(name: string): boolean {
- // Length check - attack payloads are often very long
- if (name.length > MAX_EVENT_LENGTH) return true;
-
- // Contains newlines (always suspicious for event names)
- if (name.includes('\n') || name.includes('\r')) return true;
-
- // Substring blocklist (case-insensitive)
- const lower = name.toLowerCase();
- if (BLOCKED_SUBSTRINGS.some((blocked) => lower.includes(blocked))) {
- return true;
- }
-
- // Path scanning patterns
- if (PATH_SCAN_PATTERNS.some((pattern) => pattern.test(name))) {
- return true;
- }
-
- return false;
-}
diff --git a/packages/validation/src/index.ts b/packages/validation/src/index.ts
index a5a440f96..2296666cd 100644
--- a/packages/validation/src/index.ts
+++ b/packages/validation/src/index.ts
@@ -146,7 +146,6 @@ export const zEditOrganization = z.object({
export * from './chat';
export * from './cohort.validation';
-export * from './event-blocklist';
export * from './import.validation';
export * from './track.validation';
export * from './types.insights';
diff --git a/packages/validation/src/track.validation.ts b/packages/validation/src/track.validation.ts
index 24fd95771..4e1988679 100644
--- a/packages/validation/src/track.validation.ts
+++ b/packages/validation/src/track.validation.ts
@@ -1,259 +1,7 @@
-import { RESERVED_EVENT_NAMES } from '@openpanel/constants';
-import { z } from 'zod';
-import { isBlockedEventName } from './event-blocklist';
-
-// ----- Hand-written types (source of truth) -----
-//
-// These interfaces are duplicated in code that ships in our SDK type
-// declarations. We hand-write them (instead of using `z.infer<…>`) so:
-// 1. Generated `.d.ts` files for the SDKs are readable plain TypeScript
-// with no `import type { ITrackPayload } from '@openpanel/validation'`
-// lines (the package isn't published).
-// 2. The interfaces stay clean — no zod internals leaking through.
-// Each schema below has `satisfies z.ZodType<…>` attached so a drift
-// between the interface and the schema fails to compile.
-
-export type IProfileId = string | number;
-
-export interface IGroupPayload {
- id: string;
- type: string;
- name: string;
- properties?: Record<string, unknown>;
-}
-
-export interface IAssignGroupPayload {
- groupIds: string[];
- profileId?: IProfileId;
-}
-
-export interface ITrackPayload {
- name: string;
- properties?: Record<string, unknown>;
- profileId?: IProfileId;
- groups?: string[] | null;
-}
-
-export interface IIdentifyPayload {
- profileId: IProfileId;
- firstName?: string;
- lastName?: string;
- email?: string;
- avatar?: string;
- properties?: Record<string, unknown>;
-}
-
-export interface IIncrementPayload {
- profileId: IProfileId;
- property: string;
- value?: number;
-}
-
-export interface IDecrementPayload {
- profileId: IProfileId;
- property: string;
- value?: number;
-}
-
-export interface IAliasPayload {
- profileId: IProfileId;
- alias: string;
-}
-
-export interface IReplayPayload {
- chunk_index: number;
- events_count: number;
- is_full_snapshot: boolean;
- started_at: string;
- ended_at: string;
- payload: string;
- // Server-issued session id (from a prior /track response) the SDK echoes back,
- // so the chunk is filed under the right session without device resolution.
- sessionId?: string;
-}
-
-export type ITrackHandlerPayload =
- | { type: 'track'; payload: ITrackPayload }
- | { type: 'identify'; payload: IIdentifyPayload }
- | { type: 'increment'; payload: IIncrementPayload }
- | { type: 'decrement'; payload: IDecrementPayload }
- | { type: 'alias'; payload: IAliasPayload }
- | { type: 'replay'; payload: IReplayPayload }
- | { type: 'group'; payload: IGroupPayload }
- | { type: 'assign_group'; payload: IAssignGroupPayload };
-
-// ----- Schemas (each `satisfies` its hand-written interface) -----
-
-export const zProfileId = z.union([
- z.string().min(1),
- z.number(),
-]) satisfies z.ZodType<IProfileId>;
-
-export const zGroupPayload = z.object({
- id: z.string().min(1),
- type: z.string().min(1),
- name: z.string().min(1),
- properties: z.record(z.string(), z.unknown()).optional(),
-}) satisfies z.ZodType<IGroupPayload>;
-
-export const zAssignGroupPayload = z.object({
- groupIds: z.array(z.string().min(1)),
- profileId: zProfileId.optional(),
-}) satisfies z.ZodType<IAssignGroupPayload>;
-
-export const zTrackPayload = z
- .object({
- name: z.string().min(1),
- properties: z.record(z.string(), z.unknown()).optional(),
- profileId: zProfileId.optional(),
- groups: z.array(z.string().min(1)).nullish(),
- })
- .refine((data) => !RESERVED_EVENT_NAMES.includes(data.name as any), {
- message: `Event name cannot be one of the reserved names: ${RESERVED_EVENT_NAMES.join(', ')}`,
- path: ['name'],
- })
- .refine((data) => !isBlockedEventName(data.name), {
- message: 'Event name contains blocked content',
- path: ['name'],
- })
- .refine(
- (data) => {
- if (data.name !== 'revenue') {
- return true;
- }
- const revenue = data.properties?.__revenue;
- if (revenue === undefined || revenue === null) {
- return true;
- }
- const isInt = Number.isInteger(revenue);
- if (isInt && Number(revenue) < 0) {
- return false;
- }
- return isInt;
- },
- {
- message: '__revenue must be an integer (no floats or strings)',
- path: ['properties', '__revenue'],
- }
- ) satisfies z.ZodType<ITrackPayload>;
-
-export const zIdentifyPayload = z.object({
- profileId: zProfileId,
- firstName: z.string().optional(),
- lastName: z.string().optional(),
- email: z.string().email().optional(),
- avatar: z.string().url().optional(),
- properties: z.record(z.string(), z.unknown()).optional(),
-}) satisfies z.ZodType<IIdentifyPayload>;
-
-export const zIncrementPayload = z.object({
- profileId: zProfileId,
- property: z.string().min(1),
- value: z.number().positive().optional(),
-}) satisfies z.ZodType<IIncrementPayload>;
-
-export const zDecrementPayload = z.object({
- profileId: zProfileId,
- property: z.string().min(1),
- value: z.number().positive().optional(),
-}) satisfies z.ZodType<IDecrementPayload>;
-
-export const zAliasPayload = z.object({
- profileId: zProfileId,
- alias: z.string().min(1),
-}) satisfies z.ZodType<IAliasPayload>;
-
-export const zReplayPayload = z.object({
- chunk_index: z.number().int().min(0).max(65_535),
- events_count: z.number().int().min(1),
- is_full_snapshot: z.boolean(),
- started_at: z.string().datetime(),
- ended_at: z.string().datetime(),
- payload: z.string().max(1_048_576 * 2), // 2MB max
- sessionId: z.string().max(64).optional(),
-}) satisfies z.ZodType<IReplayPayload>;
-
-export const zTrackHandlerPayload = z.discriminatedUnion('type', [
- z
- .object({
- type: z.enum(['track']),
- payload: zTrackPayload,
- })
- .meta({ title: 'Track' }),
- z
- .object({
- type: z.enum(['identify']),
- payload: zIdentifyPayload,
- })
- .meta({ title: 'Identify' }),
- z
- .object({
- type: z.enum(['increment']),
- payload: zIncrementPayload,
- })
- .meta({ title: 'Increment' }),
- z
- .object({
- type: z.enum(['decrement']),
- payload: zDecrementPayload,
- })
- .meta({ title: 'Decrement' }),
- z
- .object({
- type: z.enum(['alias']),
- payload: zAliasPayload,
- })
- .meta({ title: 'Alias' }),
- z
- .object({
- type: z.enum(['replay']),
- payload: zReplayPayload,
- })
- .meta({ title: 'Replay' }),
- z
- .object({
- type: z.enum(['group']),
- payload: zGroupPayload,
- })
- .meta({ title: 'Group' }),
- z
- .object({
- type: z.enum(['assign_group']),
- payload: zAssignGroupPayload,
- })
- .meta({ title: 'Assign Group' }),
-]) satisfies z.ZodType<ITrackHandlerPayload>;
-
-// Deprecated types for beta version of the SDKs
-
-export interface DeprecatedOpenpanelEventOptions {
- profileId?: string;
-}
-
-export interface DeprecatedPostEventPayload {
- name: string;
- timestamp: string;
- profileId?: string;
- properties?: Record<string, unknown> & DeprecatedOpenpanelEventOptions;
-}
-
-export interface DeprecatedUpdateProfilePayload {
- profileId: string;
- firstName?: string;
- lastName?: string;
- email?: string;
- avatar?: string;
- properties?: Record<string, unknown>;
-}
-
-export interface DeprecatedIncrementProfilePayload {
- profileId: string;
- property: string;
- value: number;
-}
-
-export interface DeprecatedDecrementProfilePayload {
- profileId?: string;
- property: string;
- value: number;
-}
+// Moved into @openpanel/core's ingest module (M8-002, ADR-008's module map:
+// ingest owns "C" — track.validation + event-blocklist + RESERVED_EVENT_NAMES
+// are one file there). Re-exported here for existing @openpanel/validation
+// importers (apps/api's ingest routes and hooks, the SDKs' shared types,
+// apps/start) — same shape as ./import.validation.ts since M5-004. The SDK
+// deep-import retarget to the constants subpath is P11.
+export * from '@openpanel/core/modules/ingest/ingest.constants';
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index fbc2a05c1..33abe4199 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -206,9 +206,6 @@ importers:
'@openpanel/tsconfig':
specifier: workspace:*
version: link:../../tooling/typescript
- '@types/js-yaml':
- specifier: ^4.0.9
- version: 4.0.9
'@types/ramda':
specifier: ^0.30.2
version: 0.30.2
@@ -221,9 +218,6 @@ importers:
'@types/ws':
specifier: ^8.5.14
version: 8.5.14
- js-yaml:
- specifier: ^4.1.0
- version: 4.1.0
tsdown:
specifier: 0.14.2
@@ -1166,9 +1160,15 @@ importers:
fast-deep-equal:
specifier: ^3.1.3
version: 3.1.3
+ fast-json-stable-hash:
+ specifier: ^1.0.3
+ version: 1.0.3
ipaddr.js:
specifier: ^2.5.0
version: 2.5.0
+ js-yaml:
+ specifier: ^4.1.0
+ version: 4.1.1
lru-cache:
specifier: ^11.2.4
version: 11.5.1
@@ -1220,6 +1220,9 @@ importers:
url-metadata:
specifier: ^5.4.1
version: 5.4.1
+ uuid:
+ specifier: ^9.0.1
+ version: 9.0.1
zod:
specifier: 'catalog:'
version: 4.3.6
@@ -1230,6 +1233,9 @@ importers:
'@openpanel/tsconfig':
specifier: workspace:*
version: link:../../tooling/typescript
+ '@types/js-yaml':
+ specifier: ^4.0.9
+ version: 4.0.9
'@types/qrcode':
specifier: ^1.5.6
version: 1.5.6
@@ -1242,6 +1248,9 @@ importers:
'@types/unzipper':
specifier: ^0.10.10
version: 0.10.11
+ '@types/uuid':
+ specifier: ^10.0.0
+ version: 10.0.0
bun-types:
specifier: 1.4.0
version: 1.4.0
@@ -13829,10 +13838,6 @@ packages:
resolution: {integrity: sha512-PMSmkqxr106Xa156c2M265Z+FTrPl+oxd/rgOQy2tijQeK5TxQ43psO1ZCwhVOSdnn+RzkzlRz/eY4BgJBYVpg==}
hasBin: true
- resolution: {integrity: sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==}
- hasBin: true
-
resolution: {integrity: sha512-qQKT4zQxXl8lLwBtHMWwaTcGfFOZviOJet3Oy/xmGk2gZH677CJM9EvtfdSkgWcATZhj/55JZ0rmy3myCT5lsA==}
hasBin: true
@@ -34815,10 +34820,6 @@ snapshots:
argparse: 1.0.10
esprima: 4.0.1
- dependencies:
- argparse: 2.0.1
-
dependencies:
argparse: 2.0.1