OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.
ralph(M8-002): ingest module: /track + hooks (client-auth, duplicate, isBot) + bots into core
ralphd5cf585418cc
diff --git a/apps/api/package.json b/apps/api/package.jsonindex 46becd36f..3af22cc73 100644--- a/apps/api/package.json+++ b/apps/api/package.json@@ -7,7 +7,6 @@"testing": "API_PORT=3333 pnpm dev","start": "dotenv -e ../../.env node dist/index.js","build": "rm -rf dist && tsdown",- "gen:bots": "jiti scripts/get-bots.ts && biome format --write src/bots/bots.ts","test:manage": "jiti scripts/test-manage-api.ts","e2e:sessions": "LOG_LEVEL=warn dotenv -e ../../.env -- jiti e2e/session-e2e.ts","e2e:sessions:stress": "LOG_LEVEL=warn dotenv -e ../../.env -- jiti e2e/session-stress.ts",@@ -55,12 +54,10 @@"devDependencies": {"@faker-js/faker": "^9.0.1","@openpanel/tsconfig": "workspace:*",- "@types/js-yaml": "^4.0.9","@types/ramda": "^0.30.2","@types/source-map-support": "^0.5.10","@types/uuid": "^10.0.0","@types/ws": "^8.5.14",- "js-yaml": "^4.1.0","tsdown": "0.14.2","typescript": "catalog:","vitest": "^1.0.0"diff --git a/apps/api/src/controllers/event.controller.test.ts b/apps/api/src/controllers/event.controller.test.tsindex 6e23036f2..d7aee7a58 100644--- a/apps/api/src/controllers/event.controller.test.ts+++ b/apps/api/src/controllers/event.controller.test.ts@@ -9,8 +9,9 @@* - legacyEventUsageHook increments openpanel_legacy_event_requests_total,* labelled by client id, and never mints a label for an unauthenticated* request.- * - validateSdkRequest still accepts the `mixan-*` headers (ADR-015 entry 5 is- * pending the same usage metric, so the fallback stays for now).+ * The `mixan-*` header fallback moved with the validator itself into+ * @openpanel/core's ingest module (M8-002) — see+ * packages/core/src/modules/ingest/src/client-auth.test.ts.** `@openpanel/db` is partially mocked (importActual + overrides) because it* imports `@openpanel/queue`, which the produce assertions replace.@@ -25,7 +26,6 @@ import { beforeEach, describe, expect, it, vi } from 'vitest';import { postEvent } from './event.controller';import { legacyEventUsageHook } from '@/hooks/legacy-event.hook';import { legacyEventRequestsTotal } from '@/metrics';-import { validateSdkRequest } from '@/utils/auth';vi.mock('@openpanel/queue', async () => {const actual =@@ -44,7 +44,7 @@ vi.mock('@openpanel/core', async () => {vi.mock('@openpanel/db', async () => {const actual =await vi.importActual<typeof import('@openpanel/db')>('@openpanel/db');- return { ...actual, getSalts: vi.fn(), getClientByIdCached: vi.fn() };+ return { ...actual, getSalts: vi.fn() };});const PROJECT_ID = 'legacy-event-project';@@ -214,32 +214,3 @@ describe('legacyEventUsageHook', () => {});});});--describe('mixan-* header fallback', () => {- it('authenticates a client sent as mixan-client-id (ADR-015 entry 5 deferred)', async () => {- const { getClientByIdCached } = await import('@openpanel/db');- vi.mocked(getClientByIdCached).mockResolvedValue({- id: CLIENT_ID,- projectId: PROJECT_ID,- // Keeps this test about the header fallback rather than re-proving- // secret verification, which the auth contracts already cover.- ignoreCorsAndSecret: true,- project: {- id: PROJECT_ID,- filters: [],- allowUnsafeRevenueTracking: false,- },- } as unknown as Awaited<ReturnType<typeof getClientByIdCached>>);-- const request = makeRequest(legacyBody(), {- 'mixan-client-id': CLIENT_ID,- 'mixan-client-secret': 'legacy-secret',- });- delete (request as { client?: unknown }).client;-- const client = await validateSdkRequest(request);-- expect(client.id).toBe(CLIENT_ID);- expect(request.clientSecretAuth).toBe(true);- });-});diff --git a/apps/api/src/controllers/event.controller.ts b/apps/api/src/controllers/event.controller.tsindex 914983de7..77646cca2 100644--- a/apps/api/src/controllers/event.controller.ts+++ b/apps/api/src/controllers/event.controller.ts@@ -1,6 +1,14 @@import { generateId } from '@openpanel/common';-import { getAsnInfo, getGeoLocation, parseUserAgent } from '@openpanel/core';-import { getSalts } from '@openpanel/db';+import {+ applyBotSuspicion,+ getAsnInfo,+ getDeviceId,+ getGeoLocation,+ getStringHeaders,+ getTimestamp,+ parseUserAgent,+} from '@openpanel/core';+import { getSalts, sessionBuffer } from '@openpanel/db';import {type EventsQueuePayloadIncomingEvent,produceIncomingEvent,@@ -8,9 +16,6 @@ import {import type { DeprecatedPostEventPayload } from '@openpanel/validation';import type { FastifyReply, FastifyRequest } from 'fastify';import { v4 as uuid } from 'uuid';-import { getStringHeaders, getTimestamp } from './track.controller';-import { applyBotSuspicion } from '@/bots/suspicion';-import { getDeviceId } from '@/utils/ids';export async function postEvent(request: FastifyRequest<{@@ -42,6 +47,7 @@ export async function postEvent(ip,ua,salts,+ sessionBuffer,});const uaInfo = parseUserAgent(ua, request.body?.properties);diff --git a/apps/api/src/controllers/track.controller.test.ts b/apps/api/src/controllers/track.controller.test.tsdeleted file mode 100644index 0fd3850e2..000000000--- a/apps/api/src/controllers/track.controller.test.ts+++ /dev/null@@ -1,250 +0,0 @@-/**- * Tests for:- * - getOverrideDeviceId — the guard around a track event's caller-supplied- * `__deviceId` (it becomes a Redis key segment, a BullMQ jobId and a ClickHouse- * device_id, so it must be trimmed, non-empty, and length-bounded).- * - handleReplay — replay files a chunk under the session id the SDK echoes back;- * it needs no device resolution and trusts the client-sent session id.- * - handler — the single events transport (ADR-004): every tracked event goes to- * Kafka via produceIncomingEvent, with no branch and no fallback queue.- *- * `@openpanel/db` and `@openpanel/queue` are partially mocked (importActual +- * overrides) rather than replaced: they import each other, and the replay test- * below spies on the real `replayBuffer`.- */--import { getAsnInfo, getGeoLocation } from '@openpanel/core';-import {- getSalts,- replayBuffer,- sessionBuffer,- upsertProfile,-} from '@openpanel/db';-import { produceIncomingEvent } from '@openpanel/queue';-import type {- IReplayPayload,- ITrackHandlerPayload,-} from '@openpanel/validation';-import type { FastifyReply, FastifyRequest } from 'fastify';-import { beforeEach, describe, expect, it, vi } from 'vitest';-import { getOverrideDeviceId, handleReplay, handler } from './track.controller';--vi.mock('@openpanel/queue', async () => {- const actual =- await vi.importActual<typeof import('@openpanel/queue')>(- '@openpanel/queue'- );- return { ...actual, produceIncomingEvent: vi.fn() };-});--vi.mock('@openpanel/core', async () => {- const actual =- await vi.importActual<typeof import('@openpanel/core')>('@openpanel/core');- return { ...actual, getGeoLocation: vi.fn(), getAsnInfo: vi.fn() };-});--vi.mock('@openpanel/db', async () => {- const actual =- await vi.importActual<typeof import('@openpanel/db')>('@openpanel/db');- return { ...actual, getSalts: vi.fn(), upsertProfile: vi.fn() };-});--const track = (properties?: Record<string, unknown>): ITrackHandlerPayload =>- ({- type: 'track',- payload: { name: 'page_view', properties },- }) as ITrackHandlerPayload;--describe('getOverrideDeviceId', () => {- it('returns a trimmed device id', () => {- expect(getOverrideDeviceId(track({ __deviceId: ' cookie-abc ' }))).toBe(- 'cookie-abc'- );- });-- it('returns the device id as-is when already clean', () => {- expect(getOverrideDeviceId(track({ __deviceId: 'cookie-abc' }))).toBe(- 'cookie-abc'- );- });-- it('ignores empty / whitespace-only values', () => {- expect(getOverrideDeviceId(track({ __deviceId: '' }))).toBeUndefined();- expect(getOverrideDeviceId(track({ __deviceId: ' ' }))).toBeUndefined();- });-- it('ignores non-string values', () => {- expect(getOverrideDeviceId(track({ __deviceId: 123 }))).toBeUndefined();- expect(getOverrideDeviceId(track({}))).toBeUndefined();- expect(getOverrideDeviceId(track(undefined))).toBeUndefined();- });-- it('ignores pathologically long values (cap is 64)', () => {- expect(- getOverrideDeviceId(track({ __deviceId: 'x'.repeat(65) }))- ).toBeUndefined();- expect(getOverrideDeviceId(track({ __deviceId: 'x'.repeat(64) }))).toBe(- 'x'.repeat(64)- );- });-- it('returns undefined for non-track payload types', () => {- expect(- getOverrideDeviceId({- type: 'identify',- payload: { profileId: 'p1', properties: { __deviceId: 'cookie' } },- } as ITrackHandlerPayload)- ).toBeUndefined();- });-});--const chunk = (): IReplayPayload => ({- chunk_index: 0,- events_count: 1,- is_full_snapshot: true,- started_at: '2026-06-08T12:00:00.000Z',- ended_at: '2026-06-08T12:00:01.000Z',- payload: '[]',-});--describe('handleReplay', () => {- beforeEach(() => {- vi.restoreAllMocks();- });-- it('files the chunk under the session id provided by the caller', async () => {- const add = vi.spyOn(replayBuffer, 'add').mockResolvedValue(undefined);-- await handleReplay(chunk(), {- projectId: 'proj-1',- sessionId: 'sess-issued-123',- });-- expect(add).toHaveBeenCalledTimes(1);- expect(add).toHaveBeenCalledWith(- expect.objectContaining({- project_id: 'proj-1',- session_id: 'sess-issued-123',- })- );- });-- it('throws when the session id is missing', async () => {- const add = vi.spyOn(replayBuffer, 'add').mockResolvedValue(undefined);-- await expect(- handleReplay(chunk(), { projectId: 'proj-1', sessionId: undefined })- ).rejects.toThrow('Session ID is required for replay');- expect(add).not.toHaveBeenCalled();- });-});--const PROJECT_ID = 'proj-kafka';-const DEVICE_ID = 'device-kafka-1';--const makeRequest = (body: ITrackHandlerPayload) =>- ({- body,- client: { projectId: PROJECT_ID },- clientIp: '1.2.3.4',- clientSecretAuth: false,- timestamp: new Date('2026-06-08T12:00:00.000Z').getTime(),- headers: { 'user-agent': 'Mozilla/5.0 (Macintosh) Chrome/148.0.0.0' },- }) as unknown as FastifyRequest<{ Body: ITrackHandlerPayload }>;--const makeReply = () => {- const reply = {- statusCode: 0,- body: undefined as unknown,- status(code: number) {- reply.statusCode = code;- return reply;- },- send(payload: unknown) {- reply.body = payload;- return reply;- },- };- return reply as unknown as FastifyReply & typeof reply;-};--describe('handler — events transport', () => {- beforeEach(() => {- vi.clearAllMocks();- vi.mocked(getSalts).mockResolvedValue({- current: 'salt-current',- previous: 'salt-previous',- });- vi.mocked(getGeoLocation).mockResolvedValue({- country: 'SE',- city: 'Stockholm',- region: 'AB',- longitude: 18,- latitude: 59,- });- vi.mocked(getAsnInfo).mockResolvedValue({- asn: undefined,- org: undefined,- isHosting: false,- });- vi.mocked(produceIncomingEvent).mockResolvedValue(undefined);- vi.spyOn(sessionBuffer, 'getExistingSession').mockResolvedValue(null);- });-- it('produces every tracked event to Kafka — the only transport', async () => {- await handler(- makeRequest({- type: 'track',- payload: {- name: 'page_view',- properties: { __deviceId: DEVICE_ID },- },- } as ITrackHandlerPayload),- makeReply()- );-- expect(produceIncomingEvent).toHaveBeenCalledTimes(1);- const [queueData, partitionKey] =- vi.mocked(produceIncomingEvent).mock.calls[0]!;- expect(queueData).toMatchObject({- projectId: PROJECT_ID,- deviceId: DEVICE_ID,- event: { name: 'page_view' },- });- // Browser events partition on the device id, which is what preserves- // per-device mutual exclusion in the consumer.- expect(partitionKey).toBe(DEVICE_ID);- });-- it('surfaces a produce failure instead of falling back to another queue', async () => {- vi.mocked(produceIncomingEvent).mockRejectedValue(- new Error('kafka unavailable')- );-- await expect(- handler(- makeRequest({- type: 'track',- payload: { name: 'page_view', properties: { __deviceId: DEVICE_ID } },- } as ITrackHandlerPayload),- makeReply()- )- ).rejects.toThrow('kafka unavailable');-- expect(produceIncomingEvent).toHaveBeenCalledTimes(1);- });-- it('does not produce for non-event payload types', async () => {- vi.mocked(upsertProfile).mockResolvedValue(undefined as never);-- await handler(- makeRequest({- type: 'identify',- payload: { profileId: 'p1', properties: {} },- } as ITrackHandlerPayload),- makeReply()- );-- expect(produceIncomingEvent).not.toHaveBeenCalled();- });-});diff --git a/apps/api/src/controllers/track.controller.ts b/apps/api/src/controllers/track.controller.tsindex 2d877ebf1..fe1a1dd2c 100644--- a/apps/api/src/controllers/track.controller.ts+++ b/apps/api/src/controllers/track.controller.ts@@ -1,411 +1,59 @@-import { generateId } from '@openpanel/common';import {- type AsnInfo,- type GeoLocation,- generateDeviceId,- getAsnInfo,- getGeoLocation,- parseUserAgent,+ fetchDeviceIdentity,+ ingestTrack,+ type TrackOutcome,} from '@openpanel/core';-import {- convertClickhouseDateToJs,- getProfileById,- getSalts,- groupBuffer,- replayBuffer,- SESSION_TIMEOUT_MS,- sessionBuffer,- upsertProfile,-} from '@openpanel/db';-import {- type EventsQueuePayloadIncomingEvent,- produceIncomingEvent,-} from '@openpanel/queue';-import type {- IAssignGroupPayload,- IDecrementPayload,- IGroupPayload,- IIdentifyPayload,- IIncrementPayload,- IReplayPayload,- ITrackHandlerPayload,- ITrackPayload,-} from '@openpanel/validation';+import { groupBuffer, replayBuffer, sessionBuffer } from '@openpanel/db';+import { produceIncomingEvent } from '@openpanel/queue';+import type { ITrackHandlerPayload } from '@openpanel/validation';import type { FastifyReply, FastifyRequest } from 'fastify';-import { assocPath, pathOr, pick } from 'ramda';-import { v4 as uuid } from 'uuid';-import { applyBotSuspicion, stripBotProperties } from '@/bots/suspicion';import { HttpError } from '@/utils/errors';-import { getDeviceId } from '@/utils/ids';--export function getStringHeaders(headers: FastifyRequest['headers']) {- return Object.entries(- pick(- [- 'user-agent',- 'openpanel-sdk-name',- 'openpanel-sdk-version',- 'openpanel-client-id',- 'request-id',- ],- headers- )- ).reduce(- (acc, [key, value]) => ({- ...acc,- [key]: value ? String(value) : undefined,- }),- {}- );-}--function getIdentity(body: ITrackHandlerPayload): IIdentifyPayload | undefined {- if (body.type === 'track') {- const identity = body.payload.properties?.__identify as- | IIdentifyPayload- | undefined;-- if (identity) {- return identity;- }-- return body.payload.profileId- ? {- profileId: String(body.payload.profileId),- }- : undefined;- }-- return undefined;-}--const MAX_OVERRIDE_DEVICE_ID_LENGTH = 64;--function sanitizeOverrideDeviceId(raw: unknown): string | undefined {- if (typeof raw !== 'string') {- return undefined;- }- const trimmed = raw.trim();- if (!trimmed || trimmed.length > MAX_OVERRIDE_DEVICE_ID_LENGTH) {- return undefined;- }- return trimmed;-}--// Resolve a caller-supplied device id from a track event's `properties.__deviceId`.-export function getOverrideDeviceId(- body: ITrackHandlerPayload-): string | undefined {- if (body.type !== 'track') {- return undefined;- }- return sanitizeOverrideDeviceId(body.payload?.properties?.__deviceId);-}--export function getTimestamp(- timestamp: FastifyRequest['timestamp'],- payload: ITrackHandlerPayload['payload']-) {- const safeTimestamp = timestamp || Date.now();- const userDefinedTimestamp =- 'properties' in payload- ? (payload?.properties?.__timestamp as string | undefined)- : undefined;-- if (!userDefinedTimestamp) {- return { timestamp: safeTimestamp, isTimestampFromThePast: false };- }-- const clientTimestamp = new Date(userDefinedTimestamp);- const clientTimestampNumber = clientTimestamp.getTime();-- // Constants for time validation- const ONE_MINUTE_MS = 60 * 1000;- const FIFTEEN_MINUTES_MS = 15 * ONE_MINUTE_MS;-- // Use safeTimestamp if invalid or more than 1 minute in the future- if (- Number.isNaN(clientTimestampNumber) ||- clientTimestampNumber > safeTimestamp + ONE_MINUTE_MS- ) {- return { timestamp: safeTimestamp, isTimestampFromThePast: false };- }-- // isTimestampFromThePast is true only if timestamp is older than 15 minutes- const isTimestampFromThePast =- clientTimestampNumber < safeTimestamp - FIFTEEN_MINUTES_MS;-- return {- timestamp: clientTimestampNumber,- isTimestampFromThePast,- };-}--interface TrackContext {- projectId: string;- ip: string;- ua?: string;- // Whitelisted subset (getStringHeaders) that ships with the queue payload.- headers: Record<string, string | undefined>;- // Full raw request headers — bot signals need sec-ch-ua / sec-fetch-* /- // accept-language, which the whitelist above deliberately drops.- requestHeaders: FastifyRequest['headers'];- clientSecretAuth: boolean;- timestamp: { value: number; isFromPast: boolean };- identity?: IIdentifyPayload;- deviceId: string;- sessionId: string;- geo: GeoLocation;- asnInfo: AsnInfo;-}--async function buildContext(- request: FastifyRequest<{- Body: ITrackHandlerPayload;- }>,- validatedBody: ITrackHandlerPayload-): Promise<TrackContext> {- const projectId = request.client?.projectId;- if (!projectId) {- throw new HttpError('Missing projectId', { status: 400 });- }-- const timestamp = getTimestamp(request.timestamp, validatedBody.payload);- const ip =- validatedBody.type === 'track' && validatedBody.payload.properties?.__ip- ? (validatedBody.payload.properties.__ip as string)- : request.clientIp;- const ua = request.headers['user-agent'] ?? 'unknown/1.0';-- const headers = getStringHeaders(request.headers);- const identity = getIdentity(validatedBody);- const profileId = identity?.profileId;-- if (profileId && validatedBody.type === 'track') {- validatedBody.payload.profileId = profileId;- }-- const overrideDeviceId = getOverrideDeviceId(validatedBody);-- // Get geo location (needed for track and identify) + ASN (bot detection).- // Both hit the same MaxMind readers keyed on the same IP and are cached, so- // resolving them together adds no meaningful latency.- const [geo, asnInfo, salts] = await Promise.all([- getGeoLocation(ip),- getAsnInfo(ip),- getSalts(),- ]);-- const deviceIdResult = await getDeviceId({- projectId,- ip,- ua,- salts,- overrideDeviceId,- eventTimeMs: timestamp.timestamp,- });-- return {- projectId,- ip,- ua,- headers,- requestHeaders: request.headers,- clientSecretAuth: request.clientSecretAuth ?? false,- timestamp: {- value: timestamp.timestamp,- isFromPast: timestamp.isTimestampFromThePast,- },- identity,- deviceId: deviceIdResult.deviceId,- sessionId: deviceIdResult.sessionId,- geo,- asnInfo,- };-}--async function handleTrack(- payload: ITrackPayload,- context: TrackContext-): Promise<void> {- const { projectId, deviceId, geo, headers, timestamp, sessionId } = context;-- const uaInfo = parseUserAgent(headers['user-agent'], payload.properties);- // Mark (never block) likely bot traffic with __bot / __bot_reasons props.- payload.properties = applyBotSuspicion(payload.properties, {- asnInfo: context.asnInfo,- headers: context.requestHeaders,- clientSecretAuth: context.clientSecretAuth,- isServer: uaInfo.isServer,- });- const groupId = uaInfo.isServer- ? payload.profileId- ? `${projectId}:${payload.profileId}`- : undefined- : deviceId;- const promises: Promise<unknown>[] = [];-- // If we have more than one property in the identity object, we should identify the user- // Otherwise its only a profileId and we should not identify the user- if (context.identity && Object.keys(context.identity).length > 1) {- promises.push(handleIdentify(context.identity, context));- }-- const queueData: EventsQueuePayloadIncomingEvent['payload'] = {- // The id of the ClickHouse row this event becomes, minted here so a- // redelivered Kafka message is a duplicate we can recognise.- id: uuid(),- projectId,- headers,- event: {- ...payload,- groups: payload.groups ?? [],- timestamp: timestamp.value,- isTimestampFromThePast: timestamp.isFromPast,- },- uaInfo,- geo,- deviceId,- sessionId,- };-- const partitionKey = groupId || generateId();-- promises.push(produceIncomingEvent(queueData, partitionKey));-- await Promise.all(promises);-}--async function handleIdentify(- payload: IIdentifyPayload,- context: TrackContext-): Promise<void> {- const { projectId, geo, ua } = context;- // Profiles must not carry forged bot verdicts either.- stripBotProperties(payload.properties);- const uaInfo = parseUserAgent(ua, payload.properties);- await upsertProfile({- ...payload,- id: payload.profileId,- isExternal: true,- projectId,- properties: {- ...(payload.properties ?? {}),- country: geo.country,- city: geo.city,- region: geo.region,- longitude: geo.longitude,- latitude: geo.latitude,- os: uaInfo.os,- os_version: uaInfo.osVersion,- browser: uaInfo.browser,- browser_version: uaInfo.browserVersion,- device: uaInfo.device,- brand: uaInfo.brand,- model: uaInfo.model,- },- });-}--async function adjustProfileProperty(- payload: IIncrementPayload | IDecrementPayload,- projectId: string,- direction: 1 | -1-): Promise<void> {- const { profileId, property, value } = payload;- const profile = await getProfileById(String(profileId), projectId);- if (!profile) {- throw new HttpError('Profile not found', { status: 404 });- }-- const parsed = Number.parseInt(- pathOr<string>('0', property.split('.'), profile.properties),- 10- );-- if (Number.isNaN(parsed)) {- throw new HttpError('Property value is not a number', { status: 400 });- }-- profile.properties = assocPath(- property.split('.'),- parsed + direction * (value || 1),- profile.properties- );-- await upsertProfile({- id: profile.id,- projectId,- properties: profile.properties,- isExternal: true,- });-}--async function handleIncrement(- payload: IIncrementPayload,- context: TrackContext-): Promise<void> {- await adjustProfileProperty(payload, context.projectId, 1);-}--async function handleDecrement(- payload: IDecrementPayload,- context: TrackContext-): Promise<void> {- await adjustProfileProperty(payload, context.projectId, -1);-}--// Replay only needs the server-issued session id (the SDK echoes it back). Trust-// it — scoped to the authed project, unguessable, same trust level as event data.-export async function handleReplay(- payload: IReplayPayload,- { projectId, sessionId }: { projectId: string; sessionId: string | undefined }-): Promise<void> {- if (!sessionId) {- throw new HttpError('Session ID is required for replay', { status: 400 });- }- const row = {- project_id: projectId,- session_id: sessionId,- chunk_index: payload.chunk_index,- started_at: payload.started_at,- ended_at: payload.ended_at,- events_count: payload.events_count,- is_full_snapshot: payload.is_full_snapshot,- payload: payload.payload,- };- await replayBuffer.add(row);-}--async function handleGroup(- payload: IGroupPayload,- context: TrackContext-): Promise<void> {- const { id, type, name, properties = {} } = payload;- await groupBuffer.add({- id,- projectId: context.projectId,- type,- name,- properties,- });-}--async function handleAssignGroup(- payload: IAssignGroupPayload,- context: TrackContext-): Promise<void> {- const profileId = payload.profileId ?? context.deviceId;- if (!profileId) {- return;+// Dissolved into @openpanel/core's ingest module (M8-002): the whole pipeline+// — context building, the seven payload handlers, device/session resolution+// and the Kafka produce — moved to ingest.service.ts. These controllers stay+// (DELEGATE PATTERN) as the live Fastify handlers: they resolve+// `request.client`, hand the pipeline its transport, and translate the+// outcome back into the exact replies and `HttpError`s V1 produced.+//+// The producer is passed in rather than imported by core: @openpanel/queue+// imports @openpanel/core for its logger, so core cannot import it back+// (M8-003 moves the producer into core).++const ingestBuffers = {+ session: sessionBuffer,+ replay: replayBuffer,+ group: groupBuffer,+};++function replyToOutcome(outcome: TrackOutcome, reply: FastifyReply) {+ switch (outcome.status) {+ case 'ok':+ return reply.status(200).send({+ deviceId: outcome.deviceId,+ sessionId: outcome.sessionId,+ });+ case 'alias-not-supported':+ return reply.status(400).send({+ status: 400,+ error: 'Bad Request',+ message: 'Alias is not supported',+ });+ case 'invalid-type':+ return reply.status(400).send({+ status: 400,+ error: 'Bad Request',+ message: 'Invalid type',+ });+ case 'missing-project-id':+ throw new HttpError('Missing projectId', { status: 400 });+ case 'replay-missing-session-id':+ throw new HttpError('Session ID is required for replay', { status: 400 });+ case 'profile-not-found':+ throw new HttpError('Profile not found', { status: 404 });+ default:+ throw new HttpError('Property value is not a number', { status: 400 });}- await upsertProfile({- id: String(profileId),- projectId: context.projectId,- isExternal: !!payload.profileId,- groups: payload.groupIds,- });}export async function handler(@@ -414,149 +62,47 @@ export async function handler(}>,reply: FastifyReply) {- const validatedBody = request.body;-- // Handle alias (not supported)- if (validatedBody.type === 'alias') {- return reply.status(400).send({- status: 400,- error: 'Bad Request',- message: 'Alias is not supported',- });- }-- // Build request context- const context = await buildContext(request, validatedBody);-- // Dispatch to appropriate handler- switch (validatedBody.type) {- case 'track':- await handleTrack(validatedBody.payload, context);- break;- case 'identify':- await handleIdentify(validatedBody.payload, context);- break;- case 'increment':- await handleIncrement(validatedBody.payload, context);- break;- case 'decrement':- await handleDecrement(validatedBody.payload, context);- break;- case 'replay': {- // BACKCOMPAT(replay-sessionid): TEMPORARY legacy branch, remove when new SDK is fully deployed- if (!validatedBody.payload.sessionId) {- await handleReplay(validatedBody.payload, {- projectId: context.projectId,- sessionId: context.sessionId,- });- break;- }-- await handleReplay(validatedBody.payload, {- projectId: context.projectId,- sessionId: validatedBody.payload.sessionId,- });- break;- }- case 'group':- await handleGroup(validatedBody.payload, context);- break;- case 'assign_group':- await handleAssignGroup(validatedBody.payload, context);- break;- default:- return reply.status(400).send({- status: 400,- error: 'Bad Request',- message: 'Invalid type',- });- }+ const outcome = await ingestTrack(+ {+ projectId: request.client?.projectId,+ clientIp: request.clientIp,+ headers: request.headers,+ clientSecretAuth: request.clientSecretAuth ?? false,+ timestamp: request.timestamp,+ body: request.body,+ },+ { buffers: ingestBuffers, produceIncomingEvent }+ );- reply.status(200).send({- deviceId: context.deviceId,- sessionId: context.sessionId,- });+ return replyToOutcome(outcome, reply);}export async function fetchDeviceId(request: FastifyRequest,reply: FastifyReply) {- const salts = await getSalts();- const projectId = request.client?.projectId;- if (!projectId) {- return reply.status(400).send('No projectId');- }-- const ip = request.clientIp;- if (!ip) {- return reply.status(400).send('Missing ip address');- }-- const ua = request.headers['user-agent'];- if (!ua) {- return reply.status(400).send('Missing header: user-agent');- }-- const currentDeviceId = generateDeviceId({- salt: salts.current,- origin: projectId,- ip,- ua,- });- const previousDeviceId = generateDeviceId({- salt: salts.previous,- origin: projectId,- ip,- ua,- });-- try {- const [current, previous] = await Promise.all([- sessionBuffer.getExistingSession({- projectId,- deviceId: currentDeviceId,- }),- sessionBuffer.getExistingSession({- projectId,- deviceId: previousDeviceId,- }),- ]);-- // Blob has no TTL — only treat the session as "current" if its last- // event is within the idle window. Otherwise the SDK should ask the- // server to start a fresh session id on the next event.- const now = Date.now();- const isLive = (s: typeof current) =>- !!s &&- now - convertClickhouseDateToJs(s.ended_at).getTime() <- SESSION_TIMEOUT_MS;-- if (current && isLive(current)) {- return reply.status(200).send({- deviceId: currentDeviceId,- sessionId: current.id,- message: 'current session exists for this device id',- });- }+ const identity = await fetchDeviceIdentity(+ {+ projectId: request.client?.projectId,+ clientIp: request.clientIp,+ headers: request.headers,+ },+ { session: sessionBuffer },+ request.log+ );- if (previous && isLive(previous)) {+ switch (identity.status) {+ case 'missing-project-id':+ return reply.status(400).send('No projectId');+ case 'missing-ip':+ return reply.status(400).send('Missing ip address');+ case 'missing-user-agent':+ return reply.status(400).send('Missing header: user-agent');+ default:return reply.status(200).send({- deviceId: previousDeviceId,- sessionId: previous.id,- message: 'previous session exists for this device id',+ deviceId: identity.deviceId,+ sessionId: identity.sessionId,+ message: identity.message,});- }- } catch (error) {- request.log.error(- { err: error },- 'Error getting session end GET /track/device-id'- );}-- return reply.status(200).send({- deviceId: currentDeviceId,- sessionId: '',- message: 'No session exists for this device id',- });}diff --git a/apps/api/src/hooks/client.hook.ts b/apps/api/src/hooks/client.hook.tsindex b3a9611c8..8a87d14ef 100644--- a/apps/api/src/hooks/client.hook.ts+++ b/apps/api/src/hooks/client.hook.ts@@ -1,15 +1,15 @@-import { SdkAuthError, validateSdkRequest } from '@/utils/auth';import type {DeprecatedPostEventPayload,ITrackHandlerPayload,} from '@openpanel/validation';import type { FastifyReply, FastifyRequest } from 'fastify';+import { SdkAuthError, validateSdkRequest } from '@/utils/auth';export async function clientHook(req: FastifyRequest<{Body: ITrackHandlerPayload | DeprecatedPostEventPayload;}>,- reply: FastifyReply,+ reply: FastifyReply) {try {const client = await validateSdkRequest(req);diff --git a/apps/api/src/hooks/duplicate.hook.ts b/apps/api/src/hooks/duplicate.hook.tsindex 154211721..6b0d312eb 100644--- a/apps/api/src/hooks/duplicate.hook.ts+++ b/apps/api/src/hooks/duplicate.hook.ts@@ -1,55 +1,25 @@+import { isDuplicateIngestRequest } from '@openpanel/core';import type {DeprecatedPostEventPayload,ITrackHandlerPayload,} from '@openpanel/validation';import type { FastifyReply, FastifyRequest } from 'fastify';-import { isDuplicatedEvent } from '@/utils/deduplicate';+// Delegates to @openpanel/core's ingest module (M8-002).export async function duplicateHook(req: FastifyRequest<{Body: ITrackHandlerPayload | DeprecatedPostEventPayload;}>,reply: FastifyReply) {- const ip = req.clientIp;- const origin = req.headers.origin;- const clientId = req.headers['openpanel-client-id'];- const body = req?.body;- const isTrackPayload = getIsTrackPayload(req);- const isReplay = isTrackPayload && req.body.type === 'replay';- const shouldCheck = ip && origin && clientId && !isReplay;- const isDuplicate = shouldCheck- ? await isDuplicatedEvent({- ip,- origin,- payload: body,- projectId: clientId as string,- })- : false;+ const isDuplicate = await isDuplicateIngestRequest({+ method: req.method,+ clientIp: req.clientIp,+ headers: req.headers,+ body: req.body,+ });if (isDuplicate) {return reply.status(200).send('Duplicate event');}}--function getIsTrackPayload(- req: FastifyRequest<{- Body: ITrackHandlerPayload | DeprecatedPostEventPayload;- }>-): req is FastifyRequest<{- Body: ITrackHandlerPayload;-}> {- if (req.method !== 'POST') {- return false;- }-- if (!req.body) {- return false;- }-- if (typeof req.body !== 'object' || Array.isArray(req.body)) {- return false;- }-- return 'type' in req.body;-}diff --git a/apps/api/src/hooks/is-bot.hook.test.ts b/apps/api/src/hooks/is-bot.hook.test.tsdeleted file mode 100644index 4f1bba10f..000000000--- a/apps/api/src/hooks/is-bot.hook.test.ts+++ /dev/null@@ -1,87 +0,0 @@-/**- * Tests for isBotHook — the preHandler that filters bot traffic on ingestion.- *- * The key behaviour guarded here: requests authenticated with a client secret- * (server-side SDKs) are never treated as bots, regardless of user agent. Bot- * detection only applies to public/frontend (origin-authenticated) traffic.- */--import type { FastifyReply, FastifyRequest } from 'fastify';-import { beforeEach, describe, expect, it, vi } from 'vitest';--const isBot = vi.fn();-const createBotEvent = vi.fn();--vi.mock('@/bots', () => ({ isBot: (ua: string) => isBot(ua) }));-vi.mock('@openpanel/db', () => ({- createBotEvent: (...args: unknown[]) => createBotEvent(...args),-}));--const { isBotHook } = await import('./is-bot.hook');--function makeReply() {- const status = vi.fn();- const send = vi.fn();- const reply = { status, send };- status.mockReturnValue(reply);- send.mockReturnValue(reply);- return { reply: reply as unknown as FastifyReply, status, send };-}--function makeReq(overrides: Partial<FastifyRequest> = {}) {- return {- headers: { 'user-agent': 'Googlebot/2.1' },- client: { projectId: 'proj-1' },- body: { type: 'track', payload: { properties: { path: '/home' } } },- ...overrides,- } as unknown as FastifyRequest;-}--beforeEach(() => {- isBot.mockReset();- createBotEvent.mockReset();-});--describe('isBotHook', () => {- it('skips bot detection entirely for client-secret (server-side) traffic', async () => {- const req = makeReq({ clientSecretAuth: true });- const { reply, status } = makeReply();-- await isBotHook(req as never, reply);-- expect(isBot).not.toHaveBeenCalled();- expect(createBotEvent).not.toHaveBeenCalled();- expect(status).not.toHaveBeenCalled();- });-- it('records a bot event and responds 202 for public bot traffic', async () => {- isBot.mockResolvedValue({ name: 'Googlebot', type: 'Search bot' });- const req = makeReq();- const { reply, status } = makeReply();-- await isBotHook(req as never, reply);-- expect(isBot).toHaveBeenCalledWith('Googlebot/2.1');- expect(createBotEvent).toHaveBeenCalledWith(- expect.objectContaining({- name: 'Googlebot',- type: 'Search bot',- projectId: 'proj-1',- path: '/home',- }),- );- expect(status).toHaveBeenCalledWith(202);- });-- it('passes legitimate public traffic through untouched', async () => {- isBot.mockResolvedValue(null);- const req = makeReq({ headers: { 'user-agent': 'node' } as never });- const { reply, status } = makeReply();-- const result = await isBotHook(req as never, reply);-- expect(createBotEvent).not.toHaveBeenCalled();- expect(status).not.toHaveBeenCalled();- expect(result).toBeUndefined();- });-});diff --git a/apps/api/src/hooks/is-bot.hook.ts b/apps/api/src/hooks/is-bot.hook.tsindex a787d71cb..ad1ce7af0 100644--- a/apps/api/src/hooks/is-bot.hook.ts+++ b/apps/api/src/hooks/is-bot.hook.ts@@ -1,57 +1,26 @@-import { createBotEvent } from '@openpanel/db';+import { checkIngestBot } from '@openpanel/core';import type {DeprecatedPostEventPayload,ITrackHandlerPayload,} from '@openpanel/validation';import type { FastifyReply, FastifyRequest } from 'fastify';-import { isBot } from '@/bots';+// Delegates to @openpanel/core's ingest module (M8-002), which owns the+// client-secret exemption, the bot lookup and the bot-event write.export async function isBotHook(req: FastifyRequest<{Body: ITrackHandlerPayload | DeprecatedPostEventPayload;}>,reply: FastifyReply) {- // Requests authenticated with a client secret come from server-side SDKs- // (node, php, go, rust, java, python, …). That auth is a far stronger signal- // of legitimate first-party traffic than the user agent, so never treat them- // as bots — bot detection is for public/frontend (origin-authenticated)- // traffic.- if (req.clientSecretAuth) {- return;- }-- const bot = req.headers['user-agent']- ? await isBot(req.headers['user-agent'])- : null;-- if (bot && req.client?.projectId) {- if ('type' in req.body && req.body.type === 'track') {- const path = (req.body.payload.properties?.__path ||- req.body.payload.properties?.path) as string | undefined;- if (path) {- await createBotEvent({- ...bot,- projectId: req.client.projectId,- path: path ?? '',- createdAt: new Date(),- });- }- // Handle deprecated events (v1)- } else if ('name' in req.body && 'properties' in req.body) {- const path = (req.body.properties?.__path || req.body.properties?.path) as- | string- | undefined;- if (path) {- await createBotEvent({- ...bot,- projectId: req.client.projectId,- path: path ?? '',- createdAt: new Date(),- });- }- }+ const bot = await checkIngestBot({+ headers: req.headers,+ clientSecretAuth: req.clientSecretAuth ?? false,+ projectId: req.client?.projectId,+ body: req.body,+ });+ if (bot) {return reply.status(202).send({ bot });}}diff --git a/apps/api/src/main.ts b/apps/api/src/main.tsindex 181765a9d..bced7c8e8 100644--- a/apps/api/src/main.ts+++ b/apps/api/src/main.ts@@ -21,6 +21,7 @@ import {queues,registerBufferMetrics,} from '@openpanel/core';+import { produceIncomingEvent } from '@openpanel/queue';import pino from 'pino';import { type Config, loadConfig } from './config/env';@@ -111,6 +112,9 @@ function buildDeps(): AppDeps {clients: undefined,buffers: createBuffers(bufferDeps(producers)),producers,+ // @openpanel/queue's Kafka producer, injected because core cannot import+ // it back (M8-002). M8-003 replaces it with core's own.+ produceIncomingEvent,logger,config: { selfHosted: config.SELF_HOSTED },};diff --git a/apps/api/src/utils/auth.ts b/apps/api/src/utils/auth.tsindex 3b1a6a74a..8a5f4f37a 100644--- a/apps/api/src/utils/auth.ts+++ b/apps/api/src/utils/auth.ts@@ -1,37 +1,30 @@-import { verifyPassword } from '@openpanel/core';+import {+ type IngestAuthErrorPayload,+ validateIngestRequest,+ verifyPassword,+} from '@openpanel/core';import type { IServiceClientWithProject } from '@openpanel/db';import { ClientType, getClientByIdCached } from '@openpanel/db';-import { getCache } from '@openpanel/redis';import type {DeprecatedPostEventPayload,- IProjectFilterIp,- IProjectFilterProfileId,ITrackHandlerPayload,} from '@openpanel/validation';import type { FastifyRequest, RawRequestDefaultExpression } from 'fastify';-import { path } from 'ramda';--const cleanDomain = (domain: string) =>- domain- .replace('www.', '')- .replace(/https?:\/\//, '')- .replace(/\/$/, '');+// `validateSdkRequest` dissolved into @openpanel/core's ingest module+// (M8-002): the rules live in modules/ingest/src/client-auth.ts and this is+// the DELEGATE, kept so the Fastify hook keeps its signature and its+// `req.clientSecretAuth` side channel. The other three validators stay here —+// they belong to the client module, not to ingest, and no task has moved them.+//+// The error class stays HERE on purpose. core returns a refusal rather than+// throwing, so the stack trace is captured in this bundle chunk instead of in+// the ~11 MB core chunk whose ~20 MB source map `source-map-support` would+// then parse — ~1.2s, once, on the first failed ingest auth.export class SdkAuthError extends Error {- payload: {- clientId?: string;- clientSecret?: string;- origin?: string;- };+ payload: IngestAuthErrorPayload;- constructor(- message: string,- payload: {- clientId?: string;- clientSecret?: string;- origin?: string;- }- ) {+ constructor(message: string, payload: IngestAuthErrorPayload) {super(message);this.name = 'SdkAuthError';this.message = message;@@ -44,135 +37,23 @@ export async function validateSdkRequest(Body: ITrackHandlerPayload | DeprecatedPostEventPayload;}>): Promise<IServiceClientWithProject> {- const { headers, clientIp } = req;- const clientIdNew = headers['openpanel-client-id'] as string;- const clientIdOld = headers['mixan-client-id'] as string;- const clientSecretNew = headers['openpanel-client-secret'] as string;- const clientSecretOld = headers['mixan-client-secret'] as string;- const clientIdFromBody = path<string | undefined>(['clientId'], req.body);- const clientSecretFromBody = path<string | undefined>(- ['clientSecret'],- req.body- );- const clientId = clientIdNew || clientIdOld || clientIdFromBody;- const clientSecret =- clientSecretNew || clientSecretOld || clientSecretFromBody;- const origin = headers.origin;-- if (clientSecret) {+ const outcome = await validateIngestRequest({+ headers: req.headers,+ clientIp: req.clientIp,+ body: req.body,+ });++ // V1 set this at parse time, before validation could refuse — so it is set+ // on both branches.+ if (outcome.secretPresented) {req.clientSecretAuth = true;}- const createError = (message: string) =>- new SdkAuthError(message, {- clientId,- clientSecret:- typeof clientSecret === 'string'- ? `${clientSecret.slice(0, 5)}...${clientSecret.slice(-5)}`- : 'none',- origin,- });-- if (!clientId) {- throw createError('Ingestion: Missing client id');- }-- if (- !/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/.test(- clientId- )- ) {- throw createError('Ingestion: Client ID must be a valid UUIDv4');- }-- const client = await getClientByIdCached(clientId);-- if (!client) {- throw createError('Ingestion: Invalid client id');- }-- if (!client.project) {- throw createError('Ingestion: Client has no project');- }-- // Filter out blocked IPs- const ipFilter = client.project.filters.filter(- (filter): filter is IProjectFilterIp => filter.type === 'ip'- );- if (ipFilter.some((filter) => filter.ip === clientIp)) {- throw createError('Ingestion: IP address is blocked by project filter');- }-- // Filter out blocked profile ids- const profileFilter = client.project.filters.filter(- (filter): filter is IProjectFilterProfileId => filter.type === 'profile_id'- );- const profileId =- path<string | undefined>(['payload', 'profileId'], req.body) || // Track handler- path<string | undefined>(['profileId'], req.body); // Event handler-- if (profileFilter.some((filter) => filter.profileId === profileId)) {- throw createError('Ingestion: Profile id is blocked by project filter');- }-- const revenue =- path(['payload', 'properties', '__revenue'], req.body) ??- path(['properties', '__revenue'], req.body);-- // Only allow revenue tracking if it was sent with a client secret- // or if the project has allowUnsafeRevenueTracking enabled- if (- !(client.project.allowUnsafeRevenueTracking || clientSecret) &&- typeof revenue !== 'undefined'- ) {- throw createError(- 'Ingestion: Revenue tracking is not allowed without a client secret'- );- }-- if (client.ignoreCorsAndSecret) {- return client;- }-- if (client.project.cors) {- const domainAllowed = client.project.cors.find((domain) => {- const cleanedDomain = cleanDomain(domain);- // support wildcard domains `*.foo.com`- if (cleanedDomain.includes('*')) {- const regex = new RegExp(- `${cleanedDomain.replaceAll('.', '\\.').replaceAll('*', '.+?')}`- );-- return regex.test(origin || '');- }-- if (cleanedDomain === cleanDomain(origin || '')) {- return true;- }- });-- if (domainAllowed) {- return client;- }-- if (client.project.cors.includes('*') && origin) {- return client;- }- }-- if (client.secret && clientSecret) {- const isVerified = await getCache(- `client:auth:${clientId}:${Buffer.from(clientSecret).toString('base64')}`,- 60 * 5,- async () => await verifyPassword(clientSecret, client.secret!),- true- );- if (isVerified) {- return client;- }+ if (!outcome.ok) {+ throw new SdkAuthError(outcome.message, outcome.payload);}- throw createError('Ingestion: Invalid cors or secret');+ return outcome.client;}export async function validateExportRequest(diff --git a/apps/api/src/utils/deduplicate.ts b/apps/api/src/utils/deduplicate.tsdeleted file mode 100644index d45ac9ba5..000000000--- a/apps/api/src/utils/deduplicate.ts+++ /dev/null@@ -1,34 +0,0 @@-import { getLock } from '@openpanel/redis';-import fastJsonStableHash from 'fast-json-stable-hash';--export async function isDuplicatedEvent({- ip,- origin,- payload,- projectId,-}: {- ip: string;- origin: string;- payload: Record<string, any>;- projectId: string;-}) {- const locked = await getLock(- `fastify:deduplicate:${fastJsonStableHash.hash(- {- ...payload,- ip,- origin,- projectId,- },- 'md5',- )}`,- '1',- 100,- );-- if (locked) {- return false;- }-- return true;-}diff --git a/apps/api/src/utils/ids.test.ts b/apps/api/src/utils/ids.test.tsdeleted file mode 100644index e6d8e952c..000000000--- a/apps/api/src/utils/ids.test.ts+++ /dev/null@@ -1,110 +0,0 @@-/**- * Tests for getDeviceId's session resolution against the device-keyed session- * store. The id is reused from the live blob only while it's within the idle- * window; a lingering (past-window) blob must NOT be reused — see the guard in- * ids.ts. Override device ids resolve through the same path with a single read.- *- * We spy on the real `sessionBuffer.getExistingSession` singleton (no Redis),- * and use the real `formatClickhouseDate` so `ended_at` round-trips exactly.- */--import type { IClickhouseSession } from '@openpanel/db';-import { formatClickhouseDate, sessionBuffer } from '@openpanel/db';-import { afterEach, describe, expect, it, vi } from 'vitest';-import { getDeviceId } from './ids';--const NOW = new Date('2026-06-08T12:00:00.000Z').getTime();-const MINUTE = 60 * 1000;-const SALTS = { current: 'salt-current', previous: 'salt-previous' };-const BASE = {- projectId: 'proj-1',- ip: '1.2.3.4',- ua: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/148.0.0.0',- salts: SALTS,- eventTimeMs: NOW,-};--// withinIdleWindow only reads `id` + `ended_at`; the rest is irrelevant here.-const fakeSession = (id: string, endedAtMs: number): IClickhouseSession =>- ({- id,- ended_at: formatClickhouseDate(new Date(endedAtMs)),- }) as unknown as IClickhouseSession;--afterEach(() => {- vi.restoreAllMocks();-});--describe('getDeviceId — session resolution', () => {- it('mints a deterministic, stable id when no session exists', async () => {- const spy = vi- .spyOn(sessionBuffer, 'getExistingSession')- .mockResolvedValue(null);-- const a = await getDeviceId({ ...BASE, overrideDeviceId: 'cookie-abc' });- const b = await getDeviceId({ ...BASE, overrideDeviceId: 'cookie-abc' });-- expect(a.sessionId).toBeTruthy();- expect(b.sessionId).toBe(a.sessionId); // same window → same id- expect(a.deviceId).toBe('cookie-abc');- expect(spy).toHaveBeenCalled();- });-- it('reuses the live session id when within the idle window', async () => {- vi.spyOn(sessionBuffer, 'getExistingSession').mockResolvedValue(- fakeSession('sess-live', NOW - MINUTE)- );-- const result = await getDeviceId({- ...BASE,- overrideDeviceId: 'cookie-abc',- });-- expect(result).toEqual({ deviceId: 'cookie-abc', sessionId: 'sess-live' });- });-- it('does NOT reuse a session that has lingered past the idle window', async () => {- vi.spyOn(sessionBuffer, 'getExistingSession').mockResolvedValue(- // ended 31 min ago — past the 30 min timeout; the reaper just hasn't- // closed it yet (blobs have no TTL).- fakeSession('sess-stale', NOW - 31 * MINUTE)- );-- const result = await getDeviceId({- ...BASE,- overrideDeviceId: 'cookie-abc',- });-- expect(result.deviceId).toBe('cookie-abc');- expect(result.sessionId).toBeTruthy();- expect(result.sessionId).not.toBe('sess-stale'); // a fresh id, not the stale one- });-- it('reads the store once for an override (no redundant previous lookup)', async () => {- const spy = vi- .spyOn(sessionBuffer, 'getExistingSession')- .mockResolvedValue(null);-- await getDeviceId({ ...BASE, overrideDeviceId: 'cookie-abc' });-- expect(spy).toHaveBeenCalledTimes(1);- expect(spy).toHaveBeenCalledWith({- projectId: 'proj-1',- deviceId: 'cookie-abc',- });- });-- it('checks both current and previous salt windows for internal ids', async () => {- const spy = vi- .spyOn(sessionBuffer, 'getExistingSession')- .mockResolvedValue(null);-- await getDeviceId(BASE); // no override → IP+UA hashing-- expect(spy).toHaveBeenCalledTimes(2);- const deviceIds = spy.mock.calls.map((c) =>- 'deviceId' in c[0] ? c[0].deviceId : ''- );- expect(new Set(deviceIds).size).toBe(2); // distinct current/previous hashes- });-});diff --git a/packages/constants/index.ts b/packages/constants/index.tsindex be2dce2bf..f52900566 100644--- a/packages/constants/index.ts+++ b/packages/constants/index.ts@@ -1,4 +1,7 @@-export const RESERVED_EVENT_NAMES = ['session_start', 'session_end'] as const;+// Moved into @openpanel/core's ingest module (M8-002, ADR-008's module map:+// ingest owns "C" — the reserved names are enforced by the same zod refine+// that lives there). Re-exported here for existing @openpanel/constants+// importers — same shape as `ProjectTypeNames` below.export type { EmailCategory } from '@openpanel/core/modules/email/email.constants';// Moved into @openpanel/core's email module (M6-004, ADR-008's module map:@@ -7,6 +10,7 @@ export type { EmailCategory } from '@openpanel/core/modules/email/email.constant// email-preferences pages) — same shape as `ProjectTypeNames`'s re-export// since M6-002. Keys must match the template `category` in @openpanel/email.export { emailCategories } from '@openpanel/core/modules/email/email.constants';+export { RESERVED_EVENT_NAMES } from '@openpanel/core/modules/ingest/ingest.constants';// Moved into @openpanel/core's project module (M6-002, ADR-008's module// map: project owns "C"). Re-exported here for existing// @openpanel/constants importers — same shape asdiff --git a/packages/core/package.json b/packages/core/package.jsonindex 2098fbef4..fa3a1d6a4 100644--- a/packages/core/package.json+++ b/packages/core/package.json@@ -13,6 +13,7 @@"test": "bun test --isolate","codegen": "bun scripts/download-geo.ts && bun scripts/get-datacenter-asns.ts","gen:asns": "bun scripts/get-datacenter-asns.ts",+ "gen:bots": "bun scripts/get-bots.ts && biome format --write ./src/modules/ingest/src/bots/bots.ts","gen:referrers": "bun scripts/get-referrers.ts && biome format --write ./src/shared/referrers/index.ts"},"dependencies": {@@ -48,7 +49,9 @@"date-fns": "^3.3.1","elysia": "1.4.30","fast-deep-equal": "^3.1.3",+ "fast-json-stable-hash": "^1.0.3","ipaddr.js": "^2.5.0",+ "js-yaml": "^4.1.0","lru-cache": "^11.2.4","mathjs": "^12.3.2","nanoid": "^5.1.6",@@ -66,15 +69,18 @@"undici": "^8.10.0","unzipper": "^0.12.3","url-metadata": "^5.4.1",+ "uuid": "^9.0.1","zod": "catalog:"},"devDependencies": {"@clickhouse/client": "^1.18.5","@openpanel/tsconfig": "workspace:*",+ "@types/js-yaml": "^4.0.9","@types/qrcode": "^1.5.6","@types/ramda": "^0.31.1","@types/sqlstring": "^2.3.2","@types/unzipper": "^0.10.10",+ "@types/uuid": "^10.0.0","bun-types": "1.4.0","tar": "^7.4.3","typescript": "catalog:"diff --git a/apps/api/scripts/get-bots.ts b/packages/core/scripts/get-bots.tssimilarity index 93%rename from apps/api/scripts/get-bots.tsrename to packages/core/scripts/get-bots.tsindex 5d8281339..a36eeb4e8 100644--- a/apps/api/scripts/get-bots.ts+++ b/packages/core/scripts/get-bots.ts@@ -1,14 +1,14 @@import fs from 'node:fs';-import path from 'node:path';-import { dirname } from 'node:path';+import path, { dirname } from 'node:path';import { fileURLToPath } from 'node:url';const __filename = fileURLToPath(import.meta.url);const __dirname = dirname(__filename);+import yaml from 'js-yaml';// Regex special characters that indicate we need actual regex-const regexSpecialChars = /[|^$.*+?(){}\[\]\\]/;+const regexSpecialChars = /[|^$.*+?(){}[\]\\]/;// User agents that must NEVER be treated as bots, even though the upstream// device-detector list flags them. Its "Generic Bot" pattern lumps bare@@ -40,7 +40,7 @@ export function stripAllowlistedTokens(regex: string): string {const trailingPipe = match.endsWith('|') ? '|' : '';return `^(?:${kept.join('|')})$${trailingPipe}`;- },+ });}@@ -63,14 +63,14 @@ async function main() {// Get document, or throw exception on errortry {const data = await fetch(- 'https://raw.githubusercontent.com/matomo-org/device-detector/master/regexes/bots.yml',+ 'https://raw.githubusercontent.com/matomo-org/device-detector/master/regexes/bots.yml').then((res) => res.text());const parsedData = yaml.load(data) as any[];const transformedBots = transformBots(parsedData);fs.writeFileSync(- path.resolve(__dirname, '../src/bots/bots.ts'),+ path.resolve(__dirname, '../src/modules/ingest/src/bots/bots.ts'),['// This file is generated by the script get-bots.ts','',@@ -80,11 +80,11 @@ async function main() {'export default bots;','',].join('\n'),- 'utf-8',+ 'utf-8');console.log(- `✅ Generated bots.ts with ${transformedBots.length} bot entries`,+ `✅ Generated bots.ts with ${transformedBots.length} bot entries`);const regexCount = transformedBots.filter((b) => 'regex' in b).length;const includesCount = transformedBots.filter((b) => 'includes' in b).length;diff --git a/packages/core/src/context.test.ts b/packages/core/src/context.test.tsindex 628f390dc..928b7b8db 100644--- a/packages/core/src/context.test.ts+++ b/packages/core/src/context.test.ts@@ -68,6 +68,7 @@ function stubDeps(): {return surface;},},+ produceIncomingEvent: () => Promise.resolve(),logger: stubLogger(),config: { selfHosted: true },},diff --git a/packages/core/src/context.ts b/packages/core/src/context.tsindex c738f9632..1f1f7f4c5 100644--- a/packages/core/src/context.ts+++ b/packages/core/src/context.ts@@ -8,6 +8,7 @@ import type { SessionValidationResult } from '@openpanel/db';import type { Buffers } from './buffers/create-buffers';import type { QueueProducerHandle, QueueProducers } from './jobs.registry';import type { Logger } from './logger';+import type { IncomingEventProducer } from './modules/ingest/src/incoming-event';import { createServices, type Services } from './services';import type { CookieJar, CookieOptions } from './shared/cookie';@@ -48,6 +49,13 @@ export interface AppDeps {clients: ServiceClients;buffers: Buffers;producers: QueueProducerHandle;+ /**+ * The events transport. `produceIncomingEvent` still lives in+ * @openpanel/queue, which imports core for its logger, so core cannot+ * import it back (M8-002); main.ts supplies it and M8-003 replaces it with+ * core's own Kafka producer.+ */+ produceIncomingEvent: IncomingEventProducer;logger: Logger;config: RuntimeFlags;}diff --git a/packages/core/src/http/auth.ts b/packages/core/src/http/auth.tsindex ff1297d85..9eb21cfa8 100644--- a/packages/core/src/http/auth.ts+++ b/packages/core/src/http/auth.ts@@ -40,6 +40,7 @@import { Elysia } from 'elysia';import type { AppDeps } from '../context';+import { toIngestHeaders } from '../modules/ingest/src/headers';import { authenticateClient, type ClientAuthOptions } from './client-auth';import { requestContext } from './context';@@ -64,8 +65,13 @@ export function authMacros(deps: AppDeps) {},},clientAuth: (options: ClientAuthOptions) => ({- async resolve({ ctx, status }) {- const client = await authenticateClient(deps, ctx.headers, options);+ async resolve({ body, ctx, status }) {+ const client = await authenticateClient(+ deps,+ toIngestHeaders(ctx.headers),+ options,+ { ip: ctx.ip, body }+ );if (!client) {return status(401, INVALID_CLIENT_MESSAGE);}diff --git a/packages/core/src/http/client-auth.ts b/packages/core/src/http/client-auth.tsindex dbf346916..60e0a7c73 100644--- a/packages/core/src/http/client-auth.ts+++ b/packages/core/src/http/client-auth.ts@@ -5,14 +5,19 @@// and the MCP copy — that differ only by the accepted `ClientType` set and by// the ingest extension. They collapse into this one function (ADR-011 A-i).//-// P8 fills the body in, moving it onto `services.client.authenticate` and-// porting the ingest branch verbatim: the `ignoreCorsAndSecret` short-circuit,-// the unanchored wildcard origin regex, CORS-OR-secret ordering, the-// ip/profile_id project filters, the `__revenue` gate, the body-field-// credential fallback and the 5-minute verify cache. The signature and the-// macro over it are what P2 fixes.+// M8-002 filled in the INGEST branch — `modules/ingest/src/client-auth.ts`+// holds V1's `validateSdkRequest` verbatim, and this file adapts it onto the+// principal. The `allow`-list tiers (`validateExportRequest`,+// `validateImportRequest`, `validateManageRequest`, MCP's `token: 'basic'`)+// are NOT filled in yet: they belong to the client module, not to ingest, and+// stay the stub that answers 401 — the same NAMED GAP every non-ingest+// `publicApiRoutes` module's header already records.import type { AppDeps } from '../context';+import {+ type IngestHeaders,+ validateIngestRequest,+} from '../modules/ingest/src/client-auth';/** Prisma's `ClientType` enum, by value. Moves to client.constants.ts in P7. */export type ClientType = 'read' | 'write' | 'root';@@ -42,10 +47,39 @@ export interface ClientAuthOptions {token?: 'basic';}-export function authenticateClient(+/** What the ingest tier needs beyond the headers: V1 reads the attribution ip+ * and the body (credential fallback, profile filter, `__revenue` gate). */+export interface ClientAuthRequest {+ ip: string;+ body: unknown;+}++export async function authenticateClient(_deps: AppDeps,- _headers: Headers,- _options: ClientAuthOptions+ headers: IngestHeaders,+ options: ClientAuthOptions,+ request?: ClientAuthRequest): Promise<AuthenticatedClient | null> {- return Promise.resolve(null);+ if (!options.ingest) {+ return null;+ }++ const outcome = await validateIngestRequest({+ headers,+ clientIp: request?.ip,+ body: request?.body,+ });++ if (!outcome.ok) {+ return null;+ }++ const { client } = outcome;+ return {+ id: client.id,+ projectId: client.projectId,+ organizationId: client.organizationId,+ type: client.type,+ secretPresented: outcome.secretPresented,+ };}diff --git a/packages/core/src/index.ts b/packages/core/src/index.tsindex edc3aeed9..79253497d 100644--- a/packages/core/src/index.ts+++ b/packages/core/src/index.ts@@ -1115,6 +1115,42 @@ export {runImportJob,updateImportStatus,} from './modules/import/import.service';+// The ingestion pipeline (M8-002). apps/api's /track controller, its three+// route hooks and the legacy /event controller are thin delegates over these+// — the same functions core's own `ingestRoutes` calls.+export type {+ BotMatch,+ BotSuspicion,+ DeviceIdentity,+ IncomingEventPayload,+ IncomingEventProducer,+ IngestAuthErrorPayload,+ IngestAuthOutcome,+ IngestBuffers,+ IngestHeaders,+ IngestService,+ IngestTransport,+ TrackOutcome,+ TrackRequest,+} from './modules/ingest/ingest.service';+export {+ applyBotSuspicion,+ checkIngestBot,+ detectBot,+ fetchDeviceIdentity,+ getDeviceId,+ getOverrideDeviceId,+ getStringHeaders,+ getTimestamp,+ handleReplay,+ ingestTrack,+ isBot,+ isDuplicatedEvent,+ isDuplicateIngestRequest,+ stripBotProperties,+ summarizeBotSignals,+ validateIngestRequest,+} from './modules/ingest/ingest.service';// Dissolved from @openpanel/db's services/insights* + referrer-spikes.service// (M5-001) — apps/worker's insight job files and packages/trpc's insight// router call these directly, the same way V1 reaches every other dissolveddiff --git a/packages/core/src/jobs/workers.test.ts b/packages/core/src/jobs/workers.test.tsindex 87274d36e..4abe9c4c7 100644--- a/packages/core/src/jobs/workers.test.ts+++ b/packages/core/src/jobs/workers.test.ts@@ -30,6 +30,7 @@ function stubDeps(): AppDeps {clients: {},buffers: {} as Buffers,producers: createRecordingProducers(queues),+ produceIncomingEvent: () => Promise.resolve(),logger: stubLogger(),config: { selfHosted: true },};diff --git a/packages/core/src/modules/cohort/cohort.jobs.test.ts b/packages/core/src/modules/cohort/cohort.jobs.test.tsindex 676c3e2ca..c4aa5ec2d 100644--- a/packages/core/src/modules/cohort/cohort.jobs.test.ts+++ b/packages/core/src/modules/cohort/cohort.jobs.test.ts@@ -35,6 +35,7 @@ function stubJobCtx(cohort: Partial<CohortService>, jobName = 'test'): JobCtx {insight: {} as Services['insight'],gsc: {} as Services['gsc'],import: {} as Services['import'],+ ingest: {} as Services['ingest'],cohort: cohort as CohortService,organization: {} as Services['organization'],onboarding: {} as Services['onboarding'],diff --git a/packages/core/src/modules/gsc/gsc.jobs.test.ts b/packages/core/src/modules/gsc/gsc.jobs.test.tsindex cd33a4488..13f14f933 100644--- a/packages/core/src/modules/gsc/gsc.jobs.test.ts+++ b/packages/core/src/modules/gsc/gsc.jobs.test.ts@@ -31,6 +31,7 @@ function stubJobCtx(gsc: Partial<GscService>, jobName = 'test'): JobCtx {insight: {} as Services['insight'],cohort: {} as Services['cohort'],import: {} as Services['import'],+ ingest: {} as Services['ingest'],gsc: gsc as GscService,organization: {} as Services['organization'],onboarding: {} as Services['onboarding'],@@ -132,6 +133,7 @@ test('gscSync fans out one gscProjectSync enqueue per connected project', asyncinsight: {} as Services['insight'],cohort: {} as Services['cohort'],import: {} as Services['import'],+ ingest: {} as Services['ingest'],gsc: {listConnectionsForSync: async () => [{ projectId: 'p1' },diff --git a/packages/core/src/modules/import/import.jobs.test.ts b/packages/core/src/modules/import/import.jobs.test.tsindex da8feca35..2fa70b13b 100644--- a/packages/core/src/modules/import/import.jobs.test.ts+++ b/packages/core/src/modules/import/import.jobs.test.ts@@ -30,6 +30,7 @@ function stubJobCtx(importService: Partial<ImportService>): JobCtx {notification: {} as Services['notification'],insight: {} as Services['insight'],cohort: {} as Services['cohort'],+ ingest: {} as Services['ingest'],gsc: {} as Services['gsc'],import: importService as ImportService,organization: {} as Services['organization'],diff --git a/packages/core/src/modules/ingest/ingest.constants.ts b/packages/core/src/modules/ingest/ingest.constants.tsnew file mode 100644index 000000000..17b5293e4--- /dev/null+++ b/packages/core/src/modules/ingest/ingest.constants.ts@@ -0,0 +1,398 @@+// The ingestion wire contract (M8-002, ADR-008's module map: ingest owns "C").+// Moved from packages/validation/src/track.validation.ts,+// packages/validation/src/event-blocklist.ts and @openpanel/constants'+// RESERVED_EVENT_NAMES — all three enforce the same thing (what a caller may+// name an event and what a track body may contain), so they are one file here.+// packages/validation and packages/constants re-export this; the SDK deep-import+// retarget is P11.+//+// Isomorphic by rule: zod, another *.constants.ts, or nothing.++import { z } from 'zod';++/** Event names the ingestion pipeline mints itself; a caller may not send them. */+export const RESERVED_EVENT_NAMES = ['session_start', 'session_end'] as const;++const MAX_EVENT_LENGTH = 80;++// Substrings that indicate attack/spam payloads+const BLOCKED_SUBSTRINGS = [+ // === Security Scanner Domains ===+ 'oastify.com',+ 'burpcollaborator',+ 'interact.sh',+ 'oast.me',++ // === SQL Injection ===+ 'pg_sleep',+ 'waitfor delay',+ 'xp_dirtree',+ 'load_file(',+ 'extractvalue(',+ 'dbms_pipe.receive_message',+ 'union select',++ // === Command Injection ===+ 'nslookup ',+ '/bin/sleep',+ '/bin/bash',+ 'cmd.exe',+ 'wget+http',+ 'wget http',+ 'chmod+777',+ 'chmod 777',++ // === Java/Code Execution ===+ 'processbuilder',+ 'runtime.getruntime',+ 'java.lang.processbuilder',+ 'eval-stdin.php',++ // === Path Traversal ===+ '../',+ '..\\',+ '%2e%2e',+ '%u002e%u002e',+ '/etc/passwd',+ '/etc/shadow',+ 'win.ini',+ 'system.ini',++ // === Template/SSTI Injection ===+ '${',+ '%{',++ // === XXE / XML Attacks ===+ '<!doctype',+ '<!entity',+ '<xi:include',+ 'xsi:schemalocation',++ // === SMTP Header Injection ===+ '\r\n',+ 'bcc:',++ // === Common File Scanning (paths as events) ===+ 'phpinfo.php',+ 'wp-config.php',+ '.git/config',+ '.env.backup',+ '.env.bak',+ '/vendor/phpunit/',++ // === Malware/Botnet Indicators ===+ 'mozi.m',+ '/setup.cgi?',+ '/cgi-bin/',++ // === Ruby Object Inspection Leaks ===+ '#<article:0x',+ '#<video:0x',+ '#<brand:0x',++ // === SQL Injection Patterns ===+ 'exec master.dbo',+ 'declare @',+ "' and '",+ "' or '",+ "') or ",+ "')and ",+];++// Patterns that indicate the "event" is actually a URL path being scanned+const PATH_SCAN_PATTERNS = [+ /^\/[a-z_-]+\.(php|env|yml|yaml|json|xml|config|ini|bak|sql|log)/i,+ /^\/\.[a-z]/i, // Hidden files like /.env, /.git+ /^\/(wp-|wordpress)/i, // WordPress scanning+ /^\/phpmyadmin/i,+ /^\/.+\.php$/i, // Any .php path+];++/**+ * Check if an event name should be blocked+ * @param name - The event name to check+ * @returns true if the event name should be blocked, false otherwise+ */+export function isBlockedEventName(name: string): boolean {+ // Length check - attack payloads are often very long+ if (name.length > MAX_EVENT_LENGTH) {+ return true;+ }++ // Contains newlines (always suspicious for event names)+ if (name.includes('\n') || name.includes('\r')) {+ return true;+ }++ // Substring blocklist (case-insensitive)+ const lower = name.toLowerCase();+ if (BLOCKED_SUBSTRINGS.some((blocked) => lower.includes(blocked))) {+ return true;+ }++ // Path scanning patterns+ if (PATH_SCAN_PATTERNS.some((pattern) => pattern.test(name))) {+ return true;+ }++ return false;+}++// ----- Hand-written types (source of truth) -----+//+// These interfaces are duplicated in code that ships in our SDK type+// declarations. We hand-write them (instead of using `z.infer<…>`) so:+// 1. Generated `.d.ts` files for the SDKs are readable plain TypeScript+// with no `import type { ITrackPayload } from '@openpanel/validation'`+// lines (the package isn't published).+// 2. The interfaces stay clean — no zod internals leaking through.+// Each schema below has `satisfies z.ZodType<…>` attached so a drift+// between the interface and the schema fails to compile.++export type IProfileId = string | number;++export interface IGroupPayload {+ id: string;+ type: string;+ name: string;+ properties?: Record<string, unknown>;+}++export interface IAssignGroupPayload {+ groupIds: string[];+ profileId?: IProfileId;+}++export interface ITrackPayload {+ name: string;+ properties?: Record<string, unknown>;+ profileId?: IProfileId;+ groups?: string[] | null;+}++export interface IIdentifyPayload {+ profileId: IProfileId;+ firstName?: string;+ lastName?: string;+ email?: string;+ avatar?: string;+ properties?: Record<string, unknown>;+}++export interface IIncrementPayload {+ profileId: IProfileId;+ property: string;+ value?: number;+}++export interface IDecrementPayload {+ profileId: IProfileId;+ property: string;+ value?: number;+}++export interface IAliasPayload {+ profileId: IProfileId;+ alias: string;+}++export interface IReplayPayload {+ chunk_index: number;+ events_count: number;+ is_full_snapshot: boolean;+ started_at: string;+ ended_at: string;+ payload: string;+ // Server-issued session id (from a prior /track response) the SDK echoes back,+ // so the chunk is filed under the right session without device resolution.+ sessionId?: string;+}++export type ITrackHandlerPayload =+ | { type: 'track'; payload: ITrackPayload }+ | { type: 'identify'; payload: IIdentifyPayload }+ | { type: 'increment'; payload: IIncrementPayload }+ | { type: 'decrement'; payload: IDecrementPayload }+ | { type: 'alias'; payload: IAliasPayload }+ | { type: 'replay'; payload: IReplayPayload }+ | { type: 'group'; payload: IGroupPayload }+ | { type: 'assign_group'; payload: IAssignGroupPayload };++// ----- Schemas (each `satisfies` its hand-written interface) -----++export const zProfileId = z.union([+ z.string().min(1),+ z.number(),+]) satisfies z.ZodType<IProfileId>;++export const zGroupPayload = z.object({+ id: z.string().min(1),+ type: z.string().min(1),+ name: z.string().min(1),+ properties: z.record(z.string(), z.unknown()).optional(),+}) satisfies z.ZodType<IGroupPayload>;++export const zAssignGroupPayload = z.object({+ groupIds: z.array(z.string().min(1)),+ profileId: zProfileId.optional(),+}) satisfies z.ZodType<IAssignGroupPayload>;++export const zTrackPayload = z+ .object({+ name: z.string().min(1),+ properties: z.record(z.string(), z.unknown()).optional(),+ profileId: zProfileId.optional(),+ groups: z.array(z.string().min(1)).nullish(),+ })+ .refine(+ (data) => !(RESERVED_EVENT_NAMES as readonly string[]).includes(data.name),+ {+ message: `Event name cannot be one of the reserved names: ${RESERVED_EVENT_NAMES.join(', ')}`,+ path: ['name'],+ }+ )+ .refine((data) => !isBlockedEventName(data.name), {+ message: 'Event name contains blocked content',+ path: ['name'],+ })+ .refine(+ (data) => {+ if (data.name !== 'revenue') {+ return true;+ }+ const revenue = data.properties?.__revenue;+ if (revenue === undefined || revenue === null) {+ return true;+ }+ const isInt = Number.isInteger(revenue);+ if (isInt && Number(revenue) < 0) {+ return false;+ }+ return isInt;+ },+ {+ message: '__revenue must be an integer (no floats or strings)',+ path: ['properties', '__revenue'],+ }+ ) satisfies z.ZodType<ITrackPayload>;++export const zIdentifyPayload = z.object({+ profileId: zProfileId,+ firstName: z.string().optional(),+ lastName: z.string().optional(),+ email: z.string().email().optional(),+ avatar: z.string().url().optional(),+ properties: z.record(z.string(), z.unknown()).optional(),+}) satisfies z.ZodType<IIdentifyPayload>;++export const zIncrementPayload = z.object({+ profileId: zProfileId,+ property: z.string().min(1),+ value: z.number().positive().optional(),+}) satisfies z.ZodType<IIncrementPayload>;++export const zDecrementPayload = z.object({+ profileId: zProfileId,+ property: z.string().min(1),+ value: z.number().positive().optional(),+}) satisfies z.ZodType<IDecrementPayload>;++export const zAliasPayload = z.object({+ profileId: zProfileId,+ alias: z.string().min(1),+}) satisfies z.ZodType<IAliasPayload>;++export const zReplayPayload = z.object({+ chunk_index: z.number().int().min(0).max(65_535),+ events_count: z.number().int().min(1),+ is_full_snapshot: z.boolean(),+ started_at: z.string().datetime(),+ ended_at: z.string().datetime(),+ payload: z.string().max(1_048_576 * 2), // 2MB max+ sessionId: z.string().max(64).optional(),+}) satisfies z.ZodType<IReplayPayload>;++export const zTrackHandlerPayload = z.discriminatedUnion('type', [+ z+ .object({+ type: z.enum(['track']),+ payload: zTrackPayload,+ })+ .meta({ title: 'Track' }),+ z+ .object({+ type: z.enum(['identify']),+ payload: zIdentifyPayload,+ })+ .meta({ title: 'Identify' }),+ z+ .object({+ type: z.enum(['increment']),+ payload: zIncrementPayload,+ })+ .meta({ title: 'Increment' }),+ z+ .object({+ type: z.enum(['decrement']),+ payload: zDecrementPayload,+ })+ .meta({ title: 'Decrement' }),+ z+ .object({+ type: z.enum(['alias']),+ payload: zAliasPayload,+ })+ .meta({ title: 'Alias' }),+ z+ .object({+ type: z.enum(['replay']),+ payload: zReplayPayload,+ })+ .meta({ title: 'Replay' }),+ z+ .object({+ type: z.enum(['group']),+ payload: zGroupPayload,+ })+ .meta({ title: 'Group' }),+ z+ .object({+ type: z.enum(['assign_group']),+ payload: zAssignGroupPayload,+ })+ .meta({ title: 'Assign Group' }),+]) satisfies z.ZodType<ITrackHandlerPayload>;++// Deprecated types for beta version of the SDKs++export interface DeprecatedOpenpanelEventOptions {+ profileId?: string;+}++export interface DeprecatedPostEventPayload {+ name: string;+ timestamp: string;+ profileId?: string;+ properties?: Record<string, unknown> & DeprecatedOpenpanelEventOptions;+}++export interface DeprecatedUpdateProfilePayload {+ profileId: string;+ firstName?: string;+ lastName?: string;+ email?: string;+ avatar?: string;+ properties?: Record<string, unknown>;+}++export interface DeprecatedIncrementProfilePayload {+ profileId: string;+ property: string;+ value: number;+}++export interface DeprecatedDecrementProfilePayload {+ profileId?: string;+ property: string;+ value: number;+}diff --git a/packages/core/src/modules/ingest/ingest.routes.ts b/packages/core/src/modules/ingest/ingest.routes.tsnew file mode 100644index 000000000..ef004a509--- /dev/null+++ b/packages/core/src/modules/ingest/ingest.routes.ts@@ -0,0 +1,196 @@+// Ported from apps/api/src/routes/track.router.ts + track.controller.ts+// (M8-002). V1's Fastify router stays the LIVE route (DELEGATE PATTERN) and+// its controller and hooks delegate into ingest.service.ts — the same+// functions this file calls.+//+// THE HOOK ORDER IS THE CONTRACT (ADR-002 "behaviour that must be preserved+// explicitly" 5): duplicate -> clientAuth -> isBot. Here it holds by+// lifecycle phase rather than by registration luck — measured on Elysia+// 1.4.30, a route declaring all three runs+// `derive -> transform -> macro resolve -> beforeHandle`. So the duplicate+// check is a `transform` (V1's `preValidation`), `clientAuth` is the macro,+// and the bot check is a `beforeHandle`, which is what lets it read+// `client.secretPresented` the way V1's `isBotHook` reads the+// `req.clientSecretAuth` side channel `validateSdkRequest` sets.+//+// Both routes carry the whole chain because V1's `fastify.addHook` calls are+// plugin-scoped, so `GET /track/device-id` goes through it too.+//+// NAMED GAPS, same as profile.routes.ts: not yet reachable — main.ts does not+// mount `publicApiRoutes` until a real `AppDeps` exists. V1's+// `subscriptionHook` (the wind-down gate) is NOT ported here: it belongs to+// the subscription/organization modules and reads `process.env.SELF_HOSTED`,+// which core does not do. It stays live on V1's router.++import type { AppDeps, HttpCtx } from '../../context';+import { defineRoutes } from '../../http/define';+import { zTrackHandlerPayload } from './ingest.constants';+import {+ checkIngestBot,+ fetchDeviceIdentity,+ ingestTrack,+ isDuplicateIngestRequest,+ type TrackOutcome,+} from './ingest.service';+import { toIngestHeaders } from './src/headers';++const TAGS = ['Track'];+const DUPLICATE_BODY = 'Duplicate event';++type StatusFn = (code: number, body?: unknown) => unknown;++// V1's error handler turns each thrown HttpError into this body; core has no+// such handler, so the mapping is explicit and the codes and messages are+// V1's.+function respondToOutcome(outcome: TrackOutcome, status: StatusFn) {+ switch (outcome.status) {+ case 'ok':+ return { deviceId: outcome.deviceId, sessionId: outcome.sessionId };+ case 'alias-not-supported':+ return status(400, {+ status: 400,+ error: 'Bad Request',+ message: 'Alias is not supported',+ });+ case 'invalid-type':+ return status(400, {+ status: 400,+ error: 'Bad Request',+ message: 'Invalid type',+ });+ case 'missing-project-id':+ return status(400, { status: 400, message: 'Missing projectId' });+ case 'replay-missing-session-id':+ return status(400, {+ status: 400,+ message: 'Session ID is required for replay',+ });+ case 'profile-not-found':+ return status(404, { status: 404, message: 'Profile not found' });+ default:+ return status(400, {+ status: 400,+ message: 'Property value is not a number',+ });+ }+}++// V1's `isBotHook`: a `preHandler` registered after `clientHook`, so it can+// read the `clientSecretAuth` side channel. Here it is a route-level+// `beforeHandle`, which runs after the macro resolved the principal.+async function botGuard({+ body,+ client,+ ctx,+ status,+}: {+ body: unknown;+ client: { projectId: string | null; secretPresented: boolean };+ ctx: HttpCtx;+ status: StatusFn;+}) {+ const bot = await checkIngestBot({+ headers: toIngestHeaders(ctx.headers),+ clientSecretAuth: client.secretPresented,+ projectId: client.projectId,+ body,+ });+ if (bot) {+ return status(202, { bot });+ }+}++export const ingestRoutes = defineRoutes((app, deps: AppDeps) =>+ app.guard(+ {+ // V1's `duplicateHook`, registered on `preValidation` so it answers+ // before the client is ever authenticated. A guard hook runs ahead of+ // the `clientAuth` macro's resolve and is scoped to the two routes+ // below — measured on Elysia 1.4.30:+ // derive -> guard beforeHandle -> macro resolve -> route beforeHandle.+ async beforeHandle({ body, ctx, request, status }) {+ if (+ await isDuplicateIngestRequest({+ method: request.method,+ clientIp: ctx.ip,+ headers: toIngestHeaders(ctx.headers),+ body,+ })+ ) {+ return status(200, DUPLICATE_BODY);+ }+ },+ },+ (routes) =>+ routes+ .post(+ '/track',+ async ({ body, client, ctx, status, timestamp }) => {+ const outcome = await ingestTrack(+ {+ projectId: client.projectId,+ clientIp: ctx.ip,+ headers: toIngestHeaders(ctx.headers),+ clientSecretAuth: client.secretPresented,+ timestamp,+ body,+ },+ {+ buffers: ctx.buffers,+ produceIncomingEvent: deps.produceIncomingEvent,+ }+ );++ return respondToOutcome(outcome, status);+ },+ {+ clientAuth: { ingest: true },+ body: zTrackHandlerPayload,+ beforeHandle: botGuard,+ detail: {+ tags: TAGS,+ description:+ 'Ingest a tracking event (track, identify, group, increment, decrement, replay).',+ },+ }+ )+ .get(+ '/track/device-id',+ async ({ client, ctx, status }) => {+ const identity = await fetchDeviceIdentity(+ {+ projectId: client.projectId,+ clientIp: ctx.ip,+ headers: toIngestHeaders(ctx.headers),+ },+ ctx.buffers,+ ctx.logger+ );++ switch (identity.status) {+ case 'missing-project-id':+ return status(400, 'No projectId');+ case 'missing-ip':+ return status(400, 'Missing ip address');+ case 'missing-user-agent':+ return status(400, 'Missing header: user-agent');+ default:+ return {+ deviceId: identity.deviceId,+ sessionId: identity.sessionId,+ message: identity.message,+ };+ }+ },+ {+ clientAuth: { ingest: true },+ beforeHandle: botGuard,+ detail: {+ tags: TAGS,+ description:+ 'Get or generate a stable device ID and session ID for the current visitor.',+ },+ }+ )+ )+);diff --git a/packages/core/src/modules/ingest/ingest.service.test.ts b/packages/core/src/modules/ingest/ingest.service.test.tsnew file mode 100644index 000000000..3eb73260d--- /dev/null+++ b/packages/core/src/modules/ingest/ingest.service.test.ts@@ -0,0 +1,249 @@+/**+ * Tests for:+ * - getOverrideDeviceId — the guard around a track event's caller-supplied+ * `__deviceId` (it becomes a Redis key segment, a BullMQ jobId and a ClickHouse+ * device_id, so it must be trimmed, non-empty, and length-bounded).+ * - handleReplay — replay files a chunk under the session id the SDK echoes back;+ * it needs no device resolution and trusts the client-sent session id.+ * - ingestTrack — the single events transport (ADR-004): every tracked event goes+ * to Kafka via the injected producer, with no branch and no fallback queue.+ *+ * Moved from apps/api/src/controllers/track.controller.test.ts with M8-002.+ * The producer and the buffers are arguments now, so V1's `vi.mock` of+ * @openpanel/queue and its `vi.spyOn(replayBuffer)` become plain stubs — same+ * assertions. Geo/ASN and the salts are still module-level, so those keep a+ * `mock.module` (unhoisted, hence the `await import` in `beforeAll`).+ */++import { beforeAll, beforeEach, describe, expect, it, mock } from 'bun:test';+import type { Buffers } from '../../buffers/create-buffers';+import type { GeoLocation } from '../../clients/geo';+import type { IReplayPayload, ITrackHandlerPayload } from './ingest.constants';++const getGeoLocation = mock(+ async (): Promise<GeoLocation> => ({+ country: 'SE',+ city: 'Stockholm',+ region: 'AB',+ longitude: 18,+ latitude: 59,+ })+);+const getAsnInfo = mock(async () => ({+ asn: undefined,+ org: undefined,+ isHosting: false,+}));+const getSalts = mock(async () => ({+ current: 'salt-current',+ previous: 'salt-previous',+}));+const identifyProfile = mock(async () => undefined);+const upsertProfile = mock(async () => undefined);+const getProfileById = mock(async () => null);++mock.module('../../clients/geo', () => ({ getGeoLocation, getAsnInfo }));+mock.module('@openpanel/db/src/services/salt.service', () => ({ getSalts }));++let getOverrideDeviceId: typeof import('./ingest.service').getOverrideDeviceId;+let handleReplay: typeof import('./ingest.service').handleReplay;+let ingestTrack: typeof import('./ingest.service').ingestTrack;++// The profile module is spread and then overridden by name: event.service+// imports more of it than ingest.service does, so a factory returning only+// the three overrides makes the rest vanish under `--isolate`. Overriding+// `identifyProfile` (not just `upsertProfile`) is what keeps the identify+// path off Postgres and the profile buffer — profile.service calls its own+// local `upsertProfile`, not the exported binding.+beforeAll(async () => {+ const profile = await import('../profile/profile.service');+ mock.module('../profile/profile.service', () => ({+ ...profile,+ identifyProfile,+ upsertProfile,+ getProfileById,+ }));+ ({ getOverrideDeviceId, handleReplay, ingestTrack } = await import(+ './ingest.service'+ ));+});++const track = (properties?: Record<string, unknown>): ITrackHandlerPayload =>+ ({+ type: 'track',+ payload: { name: 'page_view', properties },+ }) as ITrackHandlerPayload;++describe('getOverrideDeviceId', () => {+ it('returns a trimmed device id', () => {+ expect(getOverrideDeviceId(track({ __deviceId: ' cookie-abc ' }))).toBe(+ 'cookie-abc'+ );+ });++ it('returns the device id as-is when already clean', () => {+ expect(getOverrideDeviceId(track({ __deviceId: 'cookie-abc' }))).toBe(+ 'cookie-abc'+ );+ });++ it('ignores empty / whitespace-only values', () => {+ expect(getOverrideDeviceId(track({ __deviceId: '' }))).toBeUndefined();+ expect(getOverrideDeviceId(track({ __deviceId: ' ' }))).toBeUndefined();+ });++ it('ignores non-string values', () => {+ expect(getOverrideDeviceId(track({ __deviceId: 123 }))).toBeUndefined();+ expect(getOverrideDeviceId(track({}))).toBeUndefined();+ expect(getOverrideDeviceId(track(undefined))).toBeUndefined();+ });++ it('ignores pathologically long values (cap is 64)', () => {+ expect(+ getOverrideDeviceId(track({ __deviceId: 'x'.repeat(65) }))+ ).toBeUndefined();+ expect(getOverrideDeviceId(track({ __deviceId: 'x'.repeat(64) }))).toBe(+ 'x'.repeat(64)+ );+ });++ it('returns undefined for non-track payload types', () => {+ expect(+ getOverrideDeviceId({+ type: 'identify',+ payload: { profileId: 'p1', properties: { __deviceId: 'cookie' } },+ } as ITrackHandlerPayload)+ ).toBeUndefined();+ });+});++const chunk = (): IReplayPayload => ({+ chunk_index: 0,+ events_count: 1,+ is_full_snapshot: true,+ started_at: '2026-06-08T12:00:00.000Z',+ ended_at: '2026-06-08T12:00:01.000Z',+ payload: '[]',+});++describe('handleReplay', () => {+ it('files the chunk under the session id provided by the caller', async () => {+ const add = mock((_row: Record<string, unknown>) => Promise.resolve());++ const failure = await handleReplay(+ chunk(),+ { projectId: 'proj-1', sessionId: 'sess-issued-123' },+ { add } as unknown as Buffers['replay']+ );++ expect(failure).toBeNull();+ expect(add).toHaveBeenCalledTimes(1);+ expect(add.mock.calls[0]?.[0]).toMatchObject({+ project_id: 'proj-1',+ session_id: 'sess-issued-123',+ });+ });++ it('refuses the chunk when the session id is missing', async () => {+ const add = mock(async () => undefined);++ const failure = await handleReplay(+ chunk(),+ { projectId: 'proj-1', sessionId: undefined },+ { add } as unknown as Buffers['replay']+ );++ expect(failure).toEqual({ status: 'replay-missing-session-id' });+ expect(add).not.toHaveBeenCalled();+ });+});++const PROJECT_ID = 'proj-kafka';+const DEVICE_ID = 'device-kafka-1';++const getExistingSession = mock(async () => null);++const transport = (+ produceIncomingEvent: ReturnType<typeof mock>+): Parameters<typeof ingestTrack>[1] =>+ ({+ buffers: {+ session: { getExistingSession },+ replay: { add: mock(async () => undefined) },+ group: { add: mock(async () => undefined) },+ },+ produceIncomingEvent,+ }) as unknown as Parameters<typeof ingestTrack>[1];++const request = (body: ITrackHandlerPayload) => ({+ projectId: PROJECT_ID,+ clientIp: '1.2.3.4',+ clientSecretAuth: false,+ timestamp: new Date('2026-06-08T12:00:00.000Z').getTime(),+ headers: { 'user-agent': 'Mozilla/5.0 (Macintosh) Chrome/148.0.0.0' },+ body,+});++describe('ingestTrack — events transport', () => {+ beforeEach(() => {+ getExistingSession.mockClear();+ identifyProfile.mockClear();+ });++ it('produces every tracked event to Kafka — the only transport', async () => {+ const produce = mock(async () => undefined);++ await ingestTrack(+ request({+ type: 'track',+ payload: { name: 'page_view', properties: { __deviceId: DEVICE_ID } },+ } as ITrackHandlerPayload),+ transport(produce)+ );++ expect(produce).toHaveBeenCalledTimes(1);+ const [queueData, partitionKey] = produce.mock.calls[0] as unknown as [+ Record<string, unknown>,+ string,+ ];+ expect(queueData).toMatchObject({+ projectId: PROJECT_ID,+ deviceId: DEVICE_ID,+ event: { name: 'page_view' },+ });+ // Browser events partition on the device id, which is what preserves+ // per-device mutual exclusion in the consumer.+ expect(partitionKey).toBe(DEVICE_ID);+ });++ it('surfaces a produce failure instead of falling back to another queue', async () => {+ const produce = mock(() => Promise.reject(new Error('kafka unavailable')));++ await expect(+ ingestTrack(+ request({+ type: 'track',+ payload: { name: 'page_view', properties: { __deviceId: DEVICE_ID } },+ } as ITrackHandlerPayload),+ transport(produce)+ )+ ).rejects.toThrow('kafka unavailable');++ expect(produce).toHaveBeenCalledTimes(1);+ });++ it('does not produce for non-event payload types', async () => {+ const produce = mock(async () => undefined);++ await ingestTrack(+ request({+ type: 'identify',+ payload: { profileId: 'p1', properties: {} },+ } as ITrackHandlerPayload),+ transport(produce)+ );++ expect(produce).not.toHaveBeenCalled();+ expect(identifyProfile).toHaveBeenCalledTimes(1);+ });+});diff --git a/packages/core/src/modules/ingest/ingest.service.ts b/packages/core/src/modules/ingest/ingest.service.tsnew file mode 100644index 000000000..06b8a0651--- /dev/null+++ b/packages/core/src/modules/ingest/ingest.service.ts@@ -0,0 +1,744 @@+// The ingestion pipeline (M8-002). Ported from apps/api's+// track.controller.ts, its three route hooks (duplicate / client / is-bot)+// and utils/ids.ts. V1's Fastify controller and hooks stay the LIVE route+// (DELEGATE PATTERN) and call the functions below; ingest.routes.ts is the+// V2 half over the same functions.+//+// Nothing here throws for a caller error: each transport owns its own status+// codes and bodies, so the pipeline returns a `TrackOutcome` and V1 maps it+// back onto the exact `HttpError`s and replies it produced before.+//+// The Kafka producer is INJECTED. `produceIncomingEvent` still lives in+// @openpanel/queue, which imports @openpanel/core for its logger — core+// cannot import it back without a real package cycle (see+// cohort.service.ts's header). M8-003 moves the producer into core and the+// injection point becomes an AppDeps field.++import { generateId } from '@openpanel/common';+import { assocPath, pathOr, pick } from 'ramda';+import { v4 as uuid } from 'uuid';+import type { Buffers } from '../../buffers/create-buffers';+import { SESSION_TIMEOUT_MS } from '../../buffers/session-buffer';+import {+ type AsnInfo,+ type GeoLocation,+ getAsnInfo,+ getGeoLocation,+} from '../../clients/geo';+import type { Logger } from '../../logger';+import type { ServiceDeps } from '../../services';+import { parseUserAgent } from '../../shared/parser-user-agent';+import { generateDeviceId } from '../../shared/profileId';+import { createBotEvent } from '../event/event.service';+import {+ getProfileById,+ identifyProfile,+ upsertProfile,+} from '../profile/profile.service';+import { convertClickhouseDateToJs } from '../session/src/dates';+import type {+ IAssignGroupPayload,+ IDecrementPayload,+ IGroupPayload,+ IIdentifyPayload,+ IIncrementPayload,+ IReplayPayload,+ ITrackHandlerPayload,+ ITrackPayload,+} from './ingest.constants';+import { isBot } from './src/bots/detect';+import { applyBotSuspicion, stripBotProperties } from './src/bots/suspicion';+import { isDuplicatedEvent } from './src/deduplicate';+import { getDeviceId } from './src/device-id';+import { headerValue, type IngestHeaders } from './src/headers';+import type {+ IncomingEventPayload,+ IncomingEventProducer,+} from './src/incoming-event';++export type { BotMatch } from './src/bots/detect';+export { detectBot, isBot } from './src/bots/detect';+export {+ applyBotSuspicion,+ BOT_CATEGORY_THRESHOLD,+ type BotSuspicion,+ stripBotProperties,+ summarizeBotSignals,+} from './src/bots/suspicion';+export {+ type IngestAuthErrorPayload,+ type IngestAuthOutcome,+ validateIngestRequest,+} from './src/client-auth';+export { isDuplicatedEvent } from './src/deduplicate';+export { getDeviceId } from './src/device-id';+export type { IngestHeaders } from './src/headers';+export type {+ IncomingEventPayload,+ IncomingEventProducer,+} from './src/incoming-event';++/** The buffers this module touches. A subset of `AppDeps.buffers`. */+export type IngestBuffers = Pick<Buffers, 'session' | 'replay' | 'group'>;++export interface IngestTransport {+ buffers: IngestBuffers;+ produceIncomingEvent: IncomingEventProducer;+}++const QUEUE_PAYLOAD_HEADERS = [+ 'user-agent',+ 'openpanel-sdk-name',+ 'openpanel-sdk-version',+ 'openpanel-client-id',+ 'request-id',+];++const MAX_OVERRIDE_DEVICE_ID_LENGTH = 64;+const ONE_MINUTE_MS = 60 * 1000;+const FIFTEEN_MINUTES_MS = 15 * ONE_MINUTE_MS;+const FALLBACK_USER_AGENT = 'unknown/1.0';++function loadSalts() {+ return import('@openpanel/db/src/services/salt.service');+}++/** The whitelisted subset that ships with the queue payload. */+export function getStringHeaders(+ headers: IngestHeaders+): Record<string, string | undefined> {+ return Object.entries(pick(QUEUE_PAYLOAD_HEADERS, headers)).reduce(+ (acc, [key, value]) => ({+ ...acc,+ [key]: value ? String(value) : undefined,+ }),+ {}+ );+}++function getIdentity(body: ITrackHandlerPayload): IIdentifyPayload | undefined {+ if (body.type === 'track') {+ const identity = body.payload.properties?.__identify as+ | IIdentifyPayload+ | undefined;++ if (identity) {+ return identity;+ }++ return body.payload.profileId+ ? { profileId: String(body.payload.profileId) }+ : undefined;+ }++ return undefined;+}++function sanitizeOverrideDeviceId(raw: unknown): string | undefined {+ if (typeof raw !== 'string') {+ return undefined;+ }+ const trimmed = raw.trim();+ if (!trimmed || trimmed.length > MAX_OVERRIDE_DEVICE_ID_LENGTH) {+ return undefined;+ }+ return trimmed;+}++/** Resolve a caller-supplied device id from `properties.__deviceId`. */+export function getOverrideDeviceId(+ body: ITrackHandlerPayload+): string | undefined {+ if (body.type !== 'track') {+ return undefined;+ }+ return sanitizeOverrideDeviceId(body.payload?.properties?.__deviceId);+}++export function getTimestamp(+ timestamp: number | undefined,+ payload: ITrackHandlerPayload['payload']+): { timestamp: number; isTimestampFromThePast: boolean } {+ const safeTimestamp = timestamp || Date.now();+ const userDefinedTimestamp =+ 'properties' in payload+ ? (payload?.properties?.__timestamp as string | undefined)+ : undefined;++ if (!userDefinedTimestamp) {+ return { timestamp: safeTimestamp, isTimestampFromThePast: false };+ }++ const clientTimestamp = new Date(userDefinedTimestamp);+ const clientTimestampNumber = clientTimestamp.getTime();++ // Use safeTimestamp if invalid or more than 1 minute in the future+ if (+ Number.isNaN(clientTimestampNumber) ||+ clientTimestampNumber > safeTimestamp + ONE_MINUTE_MS+ ) {+ return { timestamp: safeTimestamp, isTimestampFromThePast: false };+ }++ // isTimestampFromThePast is true only if timestamp is older than 15 minutes+ const isTimestampFromThePast =+ clientTimestampNumber < safeTimestamp - FIFTEEN_MINUTES_MS;++ return { timestamp: clientTimestampNumber, isTimestampFromThePast };+}++export interface TrackContext {+ projectId: string;+ ip: string;+ ua?: string;+ // Whitelisted subset (getStringHeaders) that ships with the queue payload.+ headers: Record<string, string | undefined>;+ // Full raw request headers — bot signals need sec-ch-ua / sec-fetch-* /+ // accept-language, which the whitelist above deliberately drops.+ requestHeaders: IngestHeaders;+ clientSecretAuth: boolean;+ timestamp: { value: number; isFromPast: boolean };+ identity?: IIdentifyPayload;+ deviceId: string;+ sessionId: string;+ geo: GeoLocation;+ asnInfo: AsnInfo;+}++/** What a transport hands the pipeline, after its own auth ran. */+export interface TrackRequest {+ projectId: string | null | undefined;+ clientIp: string;+ headers: IngestHeaders;+ clientSecretAuth: boolean;+ /** The request-arrival timestamp (V1's `timestampHook`). */+ timestamp: number | undefined;+ body: ITrackHandlerPayload;+}++export type TrackOutcome =+ | { status: 'ok'; deviceId: string; sessionId: string }+ | { status: 'alias-not-supported' }+ | { status: 'invalid-type' }+ | { status: 'missing-project-id' }+ | { status: 'replay-missing-session-id' }+ | { status: 'profile-not-found' }+ | { status: 'profile-property-not-a-number' };++async function buildContext(+ request: TrackRequest,+ buffers: IngestBuffers+): Promise<TrackContext> {+ const projectId = request.projectId as string;+ const timestamp = getTimestamp(request.timestamp, request.body.payload);+ const ip =+ request.body.type === 'track' && request.body.payload.properties?.__ip+ ? (request.body.payload.properties.__ip as string)+ : request.clientIp;+ const ua = headerValue(request.headers, 'user-agent') ?? FALLBACK_USER_AGENT;++ const headers = getStringHeaders(request.headers);+ const identity = getIdentity(request.body);+ const profileId = identity?.profileId;++ if (profileId && request.body.type === 'track') {+ request.body.payload.profileId = profileId;+ }++ const overrideDeviceId = getOverrideDeviceId(request.body);++ // Get geo location (needed for track and identify) + ASN (bot detection).+ // Both hit the same MaxMind readers keyed on the same IP and are cached, so+ // resolving them together adds no meaningful latency.+ const { getSalts } = await loadSalts();+ const [geo, asnInfo, salts] = await Promise.all([+ getGeoLocation(ip),+ getAsnInfo(ip),+ getSalts(),+ ]);++ const deviceIdResult = await getDeviceId({+ projectId,+ ip,+ ua,+ salts,+ overrideDeviceId,+ eventTimeMs: timestamp.timestamp,+ sessionBuffer: buffers.session,+ });++ return {+ projectId,+ ip,+ ua,+ headers,+ requestHeaders: request.headers,+ clientSecretAuth: request.clientSecretAuth,+ timestamp: {+ value: timestamp.timestamp,+ isFromPast: timestamp.isTimestampFromThePast,+ },+ identity,+ deviceId: deviceIdResult.deviceId,+ sessionId: deviceIdResult.sessionId,+ geo,+ asnInfo,+ };+}++async function handleTrack(+ payload: ITrackPayload,+ context: TrackContext,+ transport: IngestTransport+): Promise<void> {+ const { projectId, deviceId, geo, headers, timestamp, sessionId } = context;++ const uaInfo = parseUserAgent(headers['user-agent'], payload.properties);+ // Mark (never block) likely bot traffic with __bot / __bot_reasons props.+ payload.properties = applyBotSuspicion(payload.properties, {+ asnInfo: context.asnInfo,+ headers: context.requestHeaders,+ clientSecretAuth: context.clientSecretAuth,+ isServer: uaInfo.isServer,+ });+ const groupId = uaInfo.isServer+ ? payload.profileId+ ? `${projectId}:${payload.profileId}`+ : undefined+ : deviceId;+ const promises: Promise<unknown>[] = [];++ // If we have more than one property in the identity object, we should identify the user+ // Otherwise its only a profileId and we should not identify the user+ if (context.identity && Object.keys(context.identity).length > 1) {+ promises.push(handleIdentify(context.identity, context));+ }++ const queueData: IncomingEventPayload = {+ // The id of the ClickHouse row this event becomes, minted here so a+ // redelivered Kafka message is a duplicate we can recognise.+ id: uuid(),+ projectId,+ headers,+ event: {+ ...payload,+ groups: payload.groups ?? [],+ timestamp: timestamp.value,+ isTimestampFromThePast: timestamp.isFromPast,+ },+ uaInfo,+ geo,+ deviceId,+ sessionId,+ };++ const partitionKey = groupId || generateId();++ promises.push(transport.produceIncomingEvent(queueData, partitionKey));++ await Promise.all(promises);+}++async function handleIdentify(+ payload: IIdentifyPayload,+ context: TrackContext+): Promise<void> {+ // Profiles must not carry forged bot verdicts either.+ stripBotProperties(payload.properties);+ const userAgent = parseUserAgent(context.ua, payload.properties);+ await identifyProfile(context.projectId, payload, {+ geo: context.geo,+ userAgent,+ });+}++async function adjustProfileProperty(+ payload: IIncrementPayload | IDecrementPayload,+ projectId: string,+ direction: 1 | -1+): Promise<TrackOutcome | null> {+ const { profileId, property, value } = payload;+ const profile = await getProfileById(String(profileId), projectId);+ if (!profile) {+ return { status: 'profile-not-found' };+ }++ const parsed = Number.parseInt(+ pathOr<string>('0', property.split('.'), profile.properties),+ 10+ );++ if (Number.isNaN(parsed)) {+ return { status: 'profile-property-not-a-number' };+ }++ await upsertProfile({+ id: profile.id,+ projectId,+ properties: assocPath(+ property.split('.'),+ parsed + direction * (value || 1),+ profile.properties+ ),+ isExternal: true,+ });++ return null;+}++/** Replay only needs the server-issued session id (the SDK echoes it back).+ * Trust it — scoped to the authed project, unguessable, same trust level as+ * event data. */+export async function handleReplay(+ payload: IReplayPayload,+ {+ projectId,+ sessionId,+ }: { projectId: string; sessionId: string | undefined },+ replayBuffer: IngestBuffers['replay']+): Promise<TrackOutcome | null> {+ if (!sessionId) {+ return { status: 'replay-missing-session-id' };+ }++ await replayBuffer.add({+ project_id: projectId,+ session_id: sessionId,+ chunk_index: payload.chunk_index,+ started_at: payload.started_at,+ ended_at: payload.ended_at,+ events_count: payload.events_count,+ is_full_snapshot: payload.is_full_snapshot,+ payload: payload.payload,+ });+ return null;+}++async function handleGroup(+ payload: IGroupPayload,+ context: TrackContext,+ groupBuffer: IngestBuffers['group']+): Promise<void> {+ const { id, type, name, properties = {} } = payload;+ await groupBuffer.add({+ id,+ projectId: context.projectId,+ type,+ name,+ properties,+ });+}++async function handleAssignGroup(+ payload: IAssignGroupPayload,+ context: TrackContext+): Promise<void> {+ const profileId = payload.profileId ?? context.deviceId;+ if (!profileId) {+ return;+ }+ await upsertProfile({+ id: String(profileId),+ projectId: context.projectId,+ isExternal: !!payload.profileId,+ groups: payload.groupIds,+ });+}++/** `POST /track`. The dispatch order is V1's, including that an unknown type+ * still resolves geo/device before it is refused. */+export async function ingestTrack(+ request: TrackRequest,+ transport: IngestTransport+): Promise<TrackOutcome> {+ const body = request.body;++ if (body.type === 'alias') {+ return { status: 'alias-not-supported' };+ }++ if (!request.projectId) {+ return { status: 'missing-project-id' };+ }++ const context = await buildContext(request, transport.buffers);++ switch (body.type) {+ case 'track':+ await handleTrack(body.payload, context, transport);+ break;+ case 'identify':+ await handleIdentify(body.payload, context);+ break;+ case 'increment': {+ const failure = await adjustProfileProperty(+ body.payload,+ context.projectId,+ 1+ );+ if (failure) {+ return failure;+ }+ break;+ }+ case 'decrement': {+ const failure = await adjustProfileProperty(+ body.payload,+ context.projectId,+ -1+ );+ if (failure) {+ return failure;+ }+ break;+ }+ case 'replay': {+ // BACKCOMPAT(replay-sessionid): TEMPORARY legacy branch, remove when new SDK is fully deployed+ const failure = await handleReplay(+ body.payload,+ {+ projectId: context.projectId,+ sessionId: body.payload.sessionId || context.sessionId,+ },+ transport.buffers.replay+ );+ if (failure) {+ return failure;+ }+ break;+ }+ case 'group':+ await handleGroup(body.payload, context, transport.buffers.group);+ break;+ case 'assign_group':+ await handleAssignGroup(body.payload, context);+ break;+ default:+ return { status: 'invalid-type' };+ }++ return {+ status: 'ok',+ deviceId: context.deviceId,+ sessionId: context.sessionId,+ };+}++export type DeviceIdentity =+ | { status: 'ok'; deviceId: string; sessionId: string; message: string }+ | { status: 'missing-project-id' }+ | { status: 'missing-ip' }+ | { status: 'missing-user-agent' };++/** `GET /track/device-id`. */+export async function fetchDeviceIdentity(+ request: {+ projectId: string | null | undefined;+ clientIp: string | undefined;+ headers: IngestHeaders;+ },+ buffers: Pick<IngestBuffers, 'session'>,+ logger: Logger+): Promise<DeviceIdentity> {+ const { getSalts } = await loadSalts();+ const salts = await getSalts();+ const projectId = request.projectId;+ if (!projectId) {+ return { status: 'missing-project-id' };+ }++ const ip = request.clientIp;+ if (!ip) {+ return { status: 'missing-ip' };+ }++ const ua = headerValue(request.headers, 'user-agent');+ if (!ua) {+ return { status: 'missing-user-agent' };+ }++ const currentDeviceId = generateDeviceId({+ salt: salts.current,+ origin: projectId,+ ip,+ ua,+ });+ const previousDeviceId = generateDeviceId({+ salt: salts.previous,+ origin: projectId,+ ip,+ ua,+ });++ try {+ const [current, previous] = await Promise.all([+ buffers.session.getExistingSession({+ projectId,+ deviceId: currentDeviceId,+ }),+ buffers.session.getExistingSession({+ projectId,+ deviceId: previousDeviceId,+ }),+ ]);++ // Blob has no TTL — only treat the session as "current" if its last+ // event is within the idle window. Otherwise the SDK should ask the+ // server to start a fresh session id on the next event.+ const now = Date.now();++ if (current && isLiveSession(current, now)) {+ return {+ status: 'ok',+ deviceId: currentDeviceId,+ sessionId: current.id,+ message: 'current session exists for this device id',+ };+ }++ if (previous && isLiveSession(previous, now)) {+ return {+ status: 'ok',+ deviceId: previousDeviceId,+ sessionId: previous.id,+ message: 'previous session exists for this device id',+ };+ }+ } catch (error) {+ logger.error(+ { err: error },+ 'Error getting session end GET /track/device-id'+ );+ }++ return {+ status: 'ok',+ deviceId: currentDeviceId,+ sessionId: '',+ message: 'No session exists for this device id',+ };+}++/** The `duplicateHook`: the web SDK can fire the same event twice. */+export async function isDuplicateIngestRequest(request: {+ method: string;+ clientIp: string | undefined;+ headers: IngestHeaders;+ body: unknown;+}): Promise<boolean> {+ const ip = request.clientIp;+ const origin = headerValue(request.headers, 'origin');+ const clientId = headerValue(request.headers, 'openpanel-client-id');+ const isReplay = isTrackBody(request) && request.body.type === 'replay';+ const shouldCheck = ip && origin && clientId && !isReplay;++ if (!shouldCheck) {+ return false;+ }++ return await isDuplicatedEvent({+ ip,+ origin,+ payload: request.body as Record<string, unknown>,+ projectId: clientId,+ });+}++function isTrackBody(request: {+ method: string;+ body: unknown;+}): request is { method: string; body: ITrackHandlerPayload } {+ if (request.method !== 'POST') {+ return false;+ }+ if (!request.body) {+ return false;+ }+ if (typeof request.body !== 'object' || Array.isArray(request.body)) {+ return false;+ }+ return 'type' in request.body;+}++export type BotVerdict = { name: string; type: string } | null;++/**+ * The `isBotHook`. Requests authenticated with a client secret come from+ * server-side SDKs (node, php, go, rust, java, python, …). That auth is a far+ * stronger signal of legitimate first-party traffic than the user agent, so+ * never treat them as bots — bot detection is for public/frontend+ * (origin-authenticated) traffic.+ */+export async function checkIngestBot(request: {+ headers: IngestHeaders;+ clientSecretAuth: boolean;+ projectId: string | null | undefined;+ body: unknown;+}): Promise<BotVerdict> {+ if (request.clientSecretAuth) {+ return null;+ }++ const userAgent = headerValue(request.headers, 'user-agent');+ const bot = userAgent ? await isBot(userAgent) : null;++ if (!(bot && request.projectId)) {+ return null;+ }++ const path = getBotEventPath(request.body);+ if (path) {+ await createBotEvent({+ ...bot,+ projectId: request.projectId,+ path,+ createdAt: new Date(),+ });+ }++ return bot;+}++function getBotEventPath(body: unknown): string | undefined {+ if (!body || typeof body !== 'object') {+ return undefined;+ }+ if ('type' in body && (body as ITrackHandlerPayload).type === 'track') {+ const properties = (+ body as { payload?: { properties?: Record<string, unknown> } }+ ).payload?.properties;+ return (properties?.__path || properties?.path) as string | undefined;+ }+ // Handle deprecated events (v1)+ if ('name' in body && 'properties' in body) {+ const properties = (body as { properties?: Record<string, unknown> })+ .properties;+ return (properties?.__path || properties?.path) as string | undefined;+ }+ return undefined;+}++function isLiveSession(session: { ended_at: string }, now: number): boolean {+ return (+ now - convertClickhouseDateToJs(session.ended_at).getTime() <+ SESSION_TIMEOUT_MS+ );+}++export interface IngestService {+ track(+ request: TrackRequest,+ produceIncomingEvent: IncomingEventProducer+ ): Promise<TrackOutcome>;+ checkBot: typeof checkIngestBot;+ isDuplicate: typeof isDuplicateIngestRequest;+}++export function createIngestService(deps: ServiceDeps): IngestService {+ return {+ track: (request, produceIncomingEvent) =>+ ingestTrack(request, { buffers: deps.buffers, produceIncomingEvent }),+ checkBot: checkIngestBot,+ isDuplicate: isDuplicateIngestRequest,+ };+}diff --git a/packages/core/src/modules/ingest/src/bot-check.test.ts b/packages/core/src/modules/ingest/src/bot-check.test.tsnew file mode 100644index 000000000..52f0e0fd1--- /dev/null+++ b/packages/core/src/modules/ingest/src/bot-check.test.ts@@ -0,0 +1,84 @@+/**+ * Tests for checkIngestBot — the bot filter V1 runs as `isBotHook` on+ * ingestion (apps/api/src/hooks/is-bot.hook.test.ts, moved with M8-002).+ *+ * The key behaviour guarded here: requests authenticated with a client secret+ * (server-side SDKs) are never treated as bots, regardless of user agent. Bot+ * detection only applies to public/frontend (origin-authenticated) traffic.+ */++import { beforeAll, beforeEach, describe, expect, it, mock } from 'bun:test';+import type { BotMatch } from './bots/detect';++const isBot = mock(async (_ua: string): Promise<BotMatch | null> => null);+const createBotEvent = mock(async (_payload: unknown) => undefined);++let checkIngestBot: typeof import('../ingest.service').checkIngestBot;++// Spread the real modules and override by name: ingest.service re-exports+// `detectBot` and reaches more of event.service than `createBotEvent`, and a+// factory returning only the overrides makes those exports vanish under+// `--isolate`.+beforeAll(async () => {+ const detect = await import('./bots/detect');+ const eventService = await import('../../event/event.service');+ mock.module('./bots/detect', () => ({ ...detect, isBot }));+ mock.module('../../event/event.service', () => ({+ ...eventService,+ createBotEvent,+ }));+ ({ checkIngestBot } = await import('../ingest.service'));+});++const baseRequest = {+ headers: { 'user-agent': 'Googlebot/2.1' },+ clientSecretAuth: false,+ projectId: 'proj-1',+ body: { type: 'track', payload: { properties: { path: '/home' } } },+};++beforeEach(() => {+ isBot.mockReset();+ createBotEvent.mockReset();+});++describe('checkIngestBot', () => {+ it('skips bot detection entirely for client-secret (server-side) traffic', async () => {+ const bot = await checkIngestBot({+ ...baseRequest,+ clientSecretAuth: true,+ });++ expect(isBot).not.toHaveBeenCalled();+ expect(createBotEvent).not.toHaveBeenCalled();+ expect(bot).toBeNull();+ });++ it('records a bot event and reports the bot for public bot traffic', async () => {+ isBot.mockResolvedValue({ name: 'Googlebot', type: 'Search bot' });++ const bot = await checkIngestBot(baseRequest);++ expect(isBot).toHaveBeenCalledWith('Googlebot/2.1');+ expect(createBotEvent).toHaveBeenCalledTimes(1);+ expect(createBotEvent.mock.calls[0]?.[0]).toMatchObject({+ name: 'Googlebot',+ type: 'Search bot',+ projectId: 'proj-1',+ path: '/home',+ });+ expect(bot).toEqual({ name: 'Googlebot', type: 'Search bot' });+ });++ it('passes legitimate public traffic through untouched', async () => {+ isBot.mockResolvedValue(null);++ const bot = await checkIngestBot({+ ...baseRequest,+ headers: { 'user-agent': 'node' },+ });++ expect(createBotEvent).not.toHaveBeenCalled();+ expect(bot).toBeNull();+ });+});diff --git a/apps/api/src/bots/bots.readme.md b/packages/core/src/modules/ingest/src/bots/bots.readme.mdsimilarity index 100%rename from apps/api/src/bots/bots.readme.mdrename to packages/core/src/modules/ingest/src/bots/bots.readme.mddiff --git a/apps/api/src/bots/bots.ts b/packages/core/src/modules/ingest/src/bots/bots.tssimilarity index 100%rename from apps/api/src/bots/bots.tsrename to packages/core/src/modules/ingest/src/bots/bots.tsdiff --git a/apps/api/src/bots/index.test.ts b/packages/core/src/modules/ingest/src/bots/detect.test.tssimilarity index 96%rename from apps/api/src/bots/index.test.tsrename to packages/core/src/modules/ingest/src/bots/detect.test.tsindex 624655b2c..f62214c0d 100644--- a/apps/api/src/bots/index.test.ts+++ b/packages/core/src/modules/ingest/src/bots/detect.test.ts@@ -10,8 +10,8 @@* identifiers, not the rest of the bot patterns.*/-import { describe, expect, it } from 'vitest';-import { detectBot } from './index';+import { describe, expect, it } from 'bun:test';+import { detectBot } from './detect';describe('detectBot', () => {describe('does not flag legitimate runtime/client user agents', () => {diff --git a/apps/api/src/bots/index.ts b/packages/core/src/modules/ingest/src/bots/detect.tssimilarity index 100%rename from apps/api/src/bots/index.tsrename to packages/core/src/modules/ingest/src/bots/detect.tsdiff --git a/apps/api/scripts/get-bots.test.ts b/packages/core/src/modules/ingest/src/bots/generator.test.tssimilarity index 91%rename from apps/api/scripts/get-bots.test.tsrename to packages/core/src/modules/ingest/src/bots/generator.test.tsindex 64def0b2f..7b6fb4c56 100644--- a/apps/api/scripts/get-bots.test.ts+++ b/packages/core/src/modules/ingest/src/bots/generator.test.ts@@ -7,8 +7,11 @@* traffic. These run without a network call (main() is guarded on direct run).*/-import { describe, expect, it } from 'vitest';-import { ALLOWLISTED_BOT_UA_TOKENS, stripAllowlistedTokens } from './get-bots';+import { describe, expect, it } from 'bun:test';+import {+ ALLOWLISTED_BOT_UA_TOKENS,+ stripAllowlistedTokens,+} from '../../../../../scripts/get-bots';describe('stripAllowlistedTokens', () => {it('removes allowlisted identifiers from an anchored exact-match group', () => {@@ -18,7 +21,7 @@ describe('stripAllowlistedTokens', () => {it('preserves the genuinely-suspicious branches', () => {const result = stripAllowlistedTokens(- '^(?:chrome|node|Node\\.js|Zeus|ZmEu)$',+ '^(?:chrome|node|Node\\.js|Zeus|ZmEu)$');expect(result).toContain('Zeus');expect(result).toContain('ZmEu');diff --git a/apps/api/src/bots/header-signals.test.ts b/packages/core/src/modules/ingest/src/bots/header-signals.test.tssimilarity index 96%rename from apps/api/src/bots/header-signals.test.tsrename to packages/core/src/modules/ingest/src/bots/header-signals.test.tsindex 1a90b04c6..c7eebabfb 100644--- a/apps/api/src/bots/header-signals.test.ts+++ b/packages/core/src/modules/ingest/src/bots/header-signals.test.ts@@ -1,4 +1,4 @@-import { describe, expect, it } from 'vitest';+import { describe, expect, it } from 'bun:test';import { detectHeaderAnomalies } from './header-signals';// Representative UA strings.@@ -34,7 +34,7 @@ describe('detectHeaderAnomalies', () => {detectHeaderAnomalies({'user-agent': UA.chromeDesktop,...realBrowserHeaders,- }),+ })).toEqual([]);});@@ -45,7 +45,7 @@ describe('detectHeaderAnomalies', () => {'sec-fetch-mode': 'cors','sec-fetch-site': 'cross-site','accept-language': 'en-US,en;q=0.5',- }),+ })).toEqual([]);});@@ -56,7 +56,7 @@ describe('detectHeaderAnomalies', () => {'sec-fetch-mode': 'cors','sec-fetch-site': 'cross-site','accept-language': 'en-US,en;q=0.9',- }),+ })).toEqual([]);});@@ -73,7 +73,10 @@ describe('detectHeaderAnomalies', () => {});describe('does not apply to non-browser clients', () => {- for (const [name, ua] of Object.entries({ curl: UA.curl, go: UA.goClient })) {+ for (const [name, ua] of Object.entries({+ curl: UA.curl,+ go: UA.goClient,+ })) {it(`${name} → no reasons (handled by isServer/auth, not headers)`, () => {expect(detectHeaderAnomalies({ 'user-agent': ua })).toEqual([]);});diff --git a/apps/api/src/bots/header-signals.ts b/packages/core/src/modules/ingest/src/bots/header-signals.tssimilarity index 100%rename from apps/api/src/bots/header-signals.tsrename to packages/core/src/modules/ingest/src/bots/header-signals.tsdiff --git a/apps/api/src/bots/suspicion.test.ts b/packages/core/src/modules/ingest/src/bots/suspicion.test.tssimilarity index 97%rename from apps/api/src/bots/suspicion.test.tsrename to packages/core/src/modules/ingest/src/bots/suspicion.test.tsindex 3214603ea..d7b4020af 100644--- a/apps/api/src/bots/suspicion.test.ts+++ b/packages/core/src/modules/ingest/src/bots/suspicion.test.ts@@ -1,5 +1,5 @@-import type { AsnInfo } from '@openpanel/core';-import { describe, expect, it } from 'vitest';+import { describe, expect, it } from 'bun:test';+import type { AsnInfo } from '../../../../clients/geo';import {applyBotSuspicion,stripBotProperties,diff --git a/apps/api/src/bots/suspicion.ts b/packages/core/src/modules/ingest/src/bots/suspicion.tssimilarity index 98%rename from apps/api/src/bots/suspicion.tsrename to packages/core/src/modules/ingest/src/bots/suspicion.tsindex 4bc13134a..076a53ef2 100644--- a/apps/api/src/bots/suspicion.ts+++ b/packages/core/src/modules/ingest/src/bots/suspicion.ts@@ -9,7 +9,7 @@// Multiple reasons within the same category (two header anomalies) count once,// so a single spoofed request can't inflate itself over the threshold.-import type { AsnInfo } from '@openpanel/core';+import type { AsnInfo } from '../../../../clients/geo';import { detectHeaderAnomalies } from './header-signals';export const BOT_CATEGORY_THRESHOLD = 2;diff --git a/packages/core/src/modules/ingest/src/client-auth.test.ts b/packages/core/src/modules/ingest/src/client-auth.test.tsnew file mode 100644index 000000000..b8c7b4b65--- /dev/null+++ b/packages/core/src/modules/ingest/src/client-auth.test.ts@@ -0,0 +1,167 @@+/**+ * Tests for validateIngestRequest — V1's `validateSdkRequest`, moved with+ * M8-002.+ *+ * The `mixan-*` case comes from apps/api/src/controllers/event.controller.test.ts:+ * ADR-015 entry 5 is pending the `/event` usage metric, so the legacy header+ * fallback stays. The `secretPresented` assertion is what V1 asserted as+ * `request.clientSecretAuth` — the side channel the bot check reads.+ */++import { beforeAll, beforeEach, describe, expect, it, mock } from 'bun:test';++const CLIENT_ID = '11111111-1111-4111-8111-111111111111';+const PROJECT_ID = 'legacy-event-project';++const client = {+ id: CLIENT_ID,+ projectId: PROJECT_ID,+ organizationId: 'org-1',+ type: 'write',+ secret: null,+ // Keeps this test about the header fallback rather than re-proving secret+ // verification, which the auth contracts already cover.+ ignoreCorsAndSecret: true,+ project: {+ id: PROJECT_ID,+ filters: [],+ allowUnsafeRevenueTracking: false,+ cors: [],+ },+};++const getClientByIdCached = mock(async (_id: string) => client);++mock.module('../../client/client.service', () => ({ getClientByIdCached }));++let validateIngestRequest: typeof import('./client-auth').validateIngestRequest;++beforeAll(async () => {+ ({ validateIngestRequest } = await import('./client-auth'));+});++beforeEach(() => {+ getClientByIdCached.mockClear();+});++const body = { name: 'legacy_event', properties: { foo: 'bar' } };++describe('validateIngestRequest', () => {+ it('authenticates a client sent as mixan-client-id (ADR-015 entry 5 deferred)', async () => {+ const result = await validateIngestRequest({+ headers: {+ 'mixan-client-id': CLIENT_ID,+ 'mixan-client-secret': 'legacy-secret',+ },+ clientIp: '203.0.113.10',+ body,+ });++ expect(getClientByIdCached).toHaveBeenCalledWith(CLIENT_ID);+ expect(result.ok && result.client.id).toBe(CLIENT_ID);+ expect(result.secretPresented).toBe(true);+ });++ it('accepts the openpanel-* headers and reports no secret when none was sent', async () => {+ const result = await validateIngestRequest({+ headers: { 'openpanel-client-id': CLIENT_ID },+ clientIp: '203.0.113.10',+ body,+ });++ expect(result.ok && result.client.id).toBe(CLIENT_ID);+ expect(result.secretPresented).toBe(false);+ });++ it('falls back to the credentials carried in the body', async () => {+ const result = await validateIngestRequest({+ headers: {},+ clientIp: '203.0.113.10',+ body: { ...body, clientId: CLIENT_ID, clientSecret: 'from-body' },+ });++ expect(result.ok && result.client.id).toBe(CLIENT_ID);+ expect(result.secretPresented).toBe(true);+ });++ // A refusal is RETURNED, never thrown: the caller builds the error so its+ // stack is captured outside core's bundle chunk (see client-auth.ts).+ it('refuses a missing client id without throwing', async () => {+ const result = await validateIngestRequest({+ headers: {},+ clientIp: '1.2.3.4',+ body,+ });++ expect(result).toMatchObject({+ ok: false,+ message: 'Ingestion: Missing client id',+ });+ expect(getClientByIdCached).not.toHaveBeenCalled();+ });++ it('refuses a client id that is not a UUIDv4', async () => {+ const result = await validateIngestRequest({+ headers: { 'openpanel-client-id': 'not-a-uuid' },+ clientIp: '1.2.3.4',+ body,+ });++ expect(result).toMatchObject({+ ok: false,+ message: 'Ingestion: Client ID must be a valid UUIDv4',+ });+ });++ it('redacts the presented secret in the refusal payload', async () => {+ const result = await validateIngestRequest({+ headers: {+ 'openpanel-client-id': 'not-a-uuid',+ 'openpanel-client-secret': 'abcdefghijklmnop',+ },+ clientIp: '1.2.3.4',+ body,+ });++ expect(result.ok).toBe(false);+ expect(result.ok === false && result.payload.clientSecret).toBe(+ 'abcde...lmnop'+ );+ expect(result.secretPresented).toBe(true);+ });++ it('refuses revenue tracking without a client secret', async () => {+ const result = await validateIngestRequest({+ headers: { 'openpanel-client-id': CLIENT_ID },+ clientIp: '1.2.3.4',+ body: { ...body, properties: { __revenue: 100 } },+ });++ expect(result).toMatchObject({+ ok: false,+ message:+ 'Ingestion: Revenue tracking is not allowed without a client secret',+ });+ });++ it('blocks an ip listed in the project filters', async () => {+ getClientByIdCached.mockResolvedValueOnce({+ ...client,+ project: {+ ...client.project,+ filters: [{ type: 'ip', ip: '203.0.113.10' }],+ },+ } as unknown as typeof client);++ const result = await validateIngestRequest({+ headers: { 'openpanel-client-id': CLIENT_ID },+ clientIp: '203.0.113.10',+ body,+ });++ expect(result).toMatchObject({+ ok: false,+ message: 'Ingestion: IP address is blocked by project filter',+ });+ });+});diff --git a/packages/core/src/modules/ingest/src/client-auth.ts b/packages/core/src/modules/ingest/src/client-auth.tsnew file mode 100644index 000000000..089b43544--- /dev/null+++ b/packages/core/src/modules/ingest/src/client-auth.ts@@ -0,0 +1,214 @@+// The ingest tier of caller authentication (ADR-011 A-i's "ingest extension",+// ported verbatim from apps/api/src/utils/auth.ts's `validateSdkRequest`).+// apps/api's `clientHook` is a thin delegate over this; core's `clientAuth`+// macro reaches it through http/client-auth.ts when a route asks for+// `{ ingest: true }`.+//+// Everything here is behaviour V1 relies on and none of it is decoration: the+// `ignoreCorsAndSecret` short-circuit, the unanchored wildcard origin regex,+// CORS-before-secret ordering, the ip and profile_id project filters, the+// `__revenue` gate, the body-field credential fallback, the `mixan-*` header+// fallback (ADR-015 entry 5 is deferred, not taken) and the 5-minute verify+// cache whose key holds base64(<plaintext secret>) — a recorded finding+// ADR-011 leaves to Carl, not to a port.++import { getCache } from '@openpanel/redis';+import type {+ IProjectFilterIp,+ IProjectFilterProfileId,+} from '@openpanel/validation';+import { path } from 'ramda';+import { verifyPassword } from '../../../shared/crypto';+import { headerValue, type IngestHeaders } from './headers';++export type { IngestHeaders } from './headers';++import {+ getClientByIdCached,+ type IServiceClientWithProject,+} from '../../client/client.service';++const CLIENT_ID_UUID_REGEX =+ /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/;+const VERIFY_CACHE_SECONDS = 60 * 5;+const REDACTED_SECRET_EDGE_LENGTH = 5;+const DOMAIN_PROTOCOL = /https?:\/\//;+const DOMAIN_TRAILING_SLASH = /\/$/;+const DOMAIN_DOT = /\./g;+const DOMAIN_WILDCARD = /\*/g;++export interface IngestAuthErrorPayload {+ clientId?: string;+ clientSecret?: string;+ origin?: string;+}++/**+ * A refusal is RETURNED, never thrown.+ *+ * Two reasons. It keeps the transport in charge of its own status codes, as+ * `TrackOutcome` does. And it keeps the error OBJECT out of this bundle:+ * apps/api's dist splits core into one ~11 MB chunk with a ~20 MB source map,+ * and `source-map-support` parses that map the first time a stack trace+ * crosses it — measured at ~1.2s on the first failed ingest auth, on the hot+ * path, which the SDK wire contract catches as a phantom retry delay.+ * `SdkAuthError` is therefore constructed by the caller.+ */+export type IngestAuthOutcome =+ | {+ ok: true;+ client: IServiceClientWithProject;+ /**+ * Whether a client secret was PRESENTED, set before it is verified. V1+ * sets `req.clientSecretAuth` at parse time and the bot hook reads it+ * afterwards as "this is a server-side SDK, never a bot" — the side+ * channel survives as a field on the outcome (ADR-011 A-i).+ */+ secretPresented: boolean;+ }+ | {+ ok: false;+ message: string;+ payload: IngestAuthErrorPayload;+ secretPresented: boolean;+ };++const cleanDomain = (domain: string) =>+ domain+ .replace('www.', '')+ .replace(DOMAIN_PROTOCOL, '')+ .replace(DOMAIN_TRAILING_SLASH, '');++function isOriginAllowed(cors: string[], origin: string | undefined): boolean {+ const domainAllowed = cors.find((domain) => {+ const cleanedDomain = cleanDomain(domain);+ // support wildcard domains `*.foo.com`+ if (cleanedDomain.includes('*')) {+ const regex = new RegExp(+ `${cleanedDomain.replace(DOMAIN_DOT, '\\.').replace(DOMAIN_WILDCARD, '.+?')}`+ );++ return regex.test(origin || '');+ }++ return cleanedDomain === cleanDomain(origin || '');+ });++ if (domainAllowed) {+ return true;+ }++ return cors.includes('*') && !!origin;+}++export async function validateIngestRequest({+ headers,+ clientIp,+ body,+}: {+ headers: IngestHeaders;+ clientIp: string | undefined;+ body: unknown;+}): Promise<IngestAuthOutcome> {+ const clientIdNew = headerValue(headers, 'openpanel-client-id');+ const clientIdOld = headerValue(headers, 'mixan-client-id');+ const clientSecretNew = headerValue(headers, 'openpanel-client-secret');+ const clientSecretOld = headerValue(headers, 'mixan-client-secret');+ const clientIdFromBody = path<string | undefined>(['clientId'], body);+ const clientSecretFromBody = path<string | undefined>(['clientSecret'], body);+ const clientId = clientIdNew || clientIdOld || clientIdFromBody;+ const clientSecret =+ clientSecretNew || clientSecretOld || clientSecretFromBody;+ const origin = headerValue(headers, 'origin');+ const secretPresented = !!clientSecret;++ const refuse = (message: string): IngestAuthOutcome => ({+ ok: false,+ message,+ payload: {+ clientId,+ clientSecret:+ typeof clientSecret === 'string'+ ? `${clientSecret.slice(0, REDACTED_SECRET_EDGE_LENGTH)}...${clientSecret.slice(-REDACTED_SECRET_EDGE_LENGTH)}`+ : 'none',+ origin,+ },+ secretPresented,+ });++ if (!clientId) {+ return refuse('Ingestion: Missing client id');+ }++ if (!CLIENT_ID_UUID_REGEX.test(clientId)) {+ return refuse('Ingestion: Client ID must be a valid UUIDv4');+ }++ const client = await getClientByIdCached(clientId);++ if (!client) {+ return refuse('Ingestion: Invalid client id');+ }++ if (!client.project) {+ return refuse('Ingestion: Client has no project');+ }++ // Filter out blocked IPs+ const ipFilter = client.project.filters.filter(+ (filter): filter is IProjectFilterIp => filter.type === 'ip'+ );+ if (ipFilter.some((filter) => filter.ip === clientIp)) {+ return refuse('Ingestion: IP address is blocked by project filter');+ }++ // Filter out blocked profile ids+ const profileFilter = client.project.filters.filter(+ (filter): filter is IProjectFilterProfileId => filter.type === 'profile_id'+ );+ const profileId =+ path<string | undefined>(['payload', 'profileId'], body) || // Track handler+ path<string | undefined>(['profileId'], body); // Event handler++ if (profileFilter.some((filter) => filter.profileId === profileId)) {+ return refuse('Ingestion: Profile id is blocked by project filter');+ }++ const revenue =+ path(['payload', 'properties', '__revenue'], body) ??+ path(['properties', '__revenue'], body);++ // Only allow revenue tracking if it was sent with a client secret+ // or if the project has allowUnsafeRevenueTracking enabled+ if (+ !(client.project.allowUnsafeRevenueTracking || clientSecret) &&+ typeof revenue !== 'undefined'+ ) {+ return refuse(+ 'Ingestion: Revenue tracking is not allowed without a client secret'+ );+ }++ if (client.ignoreCorsAndSecret) {+ return { ok: true, client, secretPresented };+ }++ if (client.project.cors && isOriginAllowed(client.project.cors, origin)) {+ return { ok: true, client, secretPresented };+ }++ const secret = client.secret;+ if (secret && clientSecret) {+ const isVerified = await getCache(+ `client:auth:${clientId}:${Buffer.from(clientSecret).toString('base64')}`,+ VERIFY_CACHE_SECONDS,+ async () => await verifyPassword(clientSecret, secret),+ true+ );+ if (isVerified) {+ return { ok: true, client, secretPresented };+ }+ }++ return refuse('Ingestion: Invalid cors or secret');+}diff --git a/packages/core/src/modules/ingest/src/deduplicate.ts b/packages/core/src/modules/ingest/src/deduplicate.tsnew file mode 100644index 000000000..e6df816e1--- /dev/null+++ b/packages/core/src/modules/ingest/src/deduplicate.ts@@ -0,0 +1,34 @@+// Ported from apps/api/src/utils/deduplicate.ts (M8-002). The web SDK can fire+// the same event twice (pagehide + visibilitychange); a short-lived Redis lock+// keyed on the whole payload collapses the pair.++import { getLock } from '@openpanel/redis';+import fastJsonStableHash from 'fast-json-stable-hash';++// Live Redis key prefix — byte-identical to V1's, so a V1 and a V2 process+// deduplicate against the same locks during the rollout.+const DEDUPLICATE_KEY_PREFIX = 'fastify:deduplicate:';+const DEDUPLICATE_TTL_MS = 100;++export async function isDuplicatedEvent({+ ip,+ origin,+ payload,+ projectId,+}: {+ ip: string;+ origin: string;+ payload: Record<string, unknown>;+ projectId: string;+}): Promise<boolean> {+ const locked = await getLock(+ `${DEDUPLICATE_KEY_PREFIX}${fastJsonStableHash.hash(+ { ...payload, ip, origin, projectId },+ 'md5'+ )}`,+ '1',+ DEDUPLICATE_TTL_MS+ );++ return !locked;+}diff --git a/packages/core/src/modules/ingest/src/device-id.test.ts b/packages/core/src/modules/ingest/src/device-id.test.tsnew file mode 100644index 000000000..26d57db95--- /dev/null+++ b/packages/core/src/modules/ingest/src/device-id.test.ts@@ -0,0 +1,125 @@+/**+ * Tests for getDeviceId's session resolution against the device-keyed session+ * store. The id is reused from the live blob only while it's within the idle+ * window; a lingering (past-window) blob must NOT be reused — see the guard in+ * device-id.ts. Override device ids resolve through the same path with a+ * single read.+ *+ * Moved from apps/api/src/utils/ids.test.ts with M8-002. The buffer is now an+ * argument, so the stub replaces V1's spy on the module singleton — same+ * assertions, and no Redis either way.+ */++import { afterEach, describe, expect, it, mock } from 'bun:test';+import type { IClickhouseSession } from '../../session/session.service';+import { formatClickhouseDate } from '../../session/src/dates';+import { getDeviceId, type SessionBufferReader } from './device-id';++const NOW = new Date('2026-06-08T12:00:00.000Z').getTime();+const MINUTE = 60 * 1000;+const SALTS = { current: 'salt-current', previous: 'salt-previous' };+const BASE = {+ projectId: 'proj-1',+ ip: '1.2.3.4',+ ua: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/148.0.0.0',+ salts: SALTS,+ eventTimeMs: NOW,+};++// withinIdleWindow only reads `id` + `ended_at`; the rest is irrelevant here.+const fakeSession = (id: string, endedAtMs: number): IClickhouseSession =>+ ({+ id,+ ended_at: formatClickhouseDate(new Date(endedAtMs)),+ }) as unknown as IClickhouseSession;++function stubBuffer(session: IClickhouseSession | null) {+ const getExistingSession = mock(async () => session);+ return {+ getExistingSession,+ buffer: { getExistingSession } as unknown as SessionBufferReader,+ };+}++afterEach(() => {+ mock.restore();+});++describe('getDeviceId — session resolution', () => {+ it('mints a deterministic, stable id when no session exists', async () => {+ const { getExistingSession, buffer } = stubBuffer(null);++ const a = await getDeviceId({+ ...BASE,+ overrideDeviceId: 'cookie-abc',+ sessionBuffer: buffer,+ });+ const b = await getDeviceId({+ ...BASE,+ overrideDeviceId: 'cookie-abc',+ sessionBuffer: buffer,+ });++ expect(a.sessionId).toBeTruthy();+ expect(b.sessionId).toBe(a.sessionId); // same window → same id+ expect(a.deviceId).toBe('cookie-abc');+ expect(getExistingSession).toHaveBeenCalled();+ });++ it('reuses the live session id when within the idle window', async () => {+ const { buffer } = stubBuffer(fakeSession('sess-live', NOW - MINUTE));++ const result = await getDeviceId({+ ...BASE,+ overrideDeviceId: 'cookie-abc',+ sessionBuffer: buffer,+ });++ expect(result).toEqual({ deviceId: 'cookie-abc', sessionId: 'sess-live' });+ });++ it('does NOT reuse a session that has lingered past the idle window', async () => {+ // ended 31 min ago — past the 30 min timeout; the reaper just hasn't+ // closed it yet (blobs have no TTL).+ const { buffer } = stubBuffer(fakeSession('sess-stale', NOW - 31 * MINUTE));++ const result = await getDeviceId({+ ...BASE,+ overrideDeviceId: 'cookie-abc',+ sessionBuffer: buffer,+ });++ expect(result.deviceId).toBe('cookie-abc');+ expect(result.sessionId).toBeTruthy();+ expect(result.sessionId).not.toBe('sess-stale'); // a fresh id, not the stale one+ });++ it('reads the store once for an override (no redundant previous lookup)', async () => {+ const { getExistingSession, buffer } = stubBuffer(null);++ await getDeviceId({+ ...BASE,+ overrideDeviceId: 'cookie-abc',+ sessionBuffer: buffer,+ });++ expect(getExistingSession).toHaveBeenCalledTimes(1);+ expect(getExistingSession).toHaveBeenCalledWith({+ projectId: 'proj-1',+ deviceId: 'cookie-abc',+ });+ });++ it('checks both current and previous salt windows for internal ids', async () => {+ const { getExistingSession, buffer } = stubBuffer(null);++ await getDeviceId({ ...BASE, sessionBuffer: buffer }); // no override → IP+UA hashing++ expect(getExistingSession).toHaveBeenCalledTimes(2);+ const deviceIds = getExistingSession.mock.calls.map((call) => {+ const [args] = call as unknown as [{ deviceId?: string }];+ return args.deviceId ?? '';+ });+ expect(new Set(deviceIds).size).toBe(2); // distinct current/previous hashes+ });+});diff --git a/apps/api/src/utils/ids.ts b/packages/core/src/modules/ingest/src/device-id.tssimilarity index 74%rename from apps/api/src/utils/ids.tsrename to packages/core/src/modules/ingest/src/device-id.tsindex fe7ab12ff..2ce76937f 100644--- a/apps/api/src/utils/ids.ts+++ b/packages/core/src/modules/ingest/src/device-id.ts@@ -1,11 +1,32 @@+// Ported from apps/api/src/utils/ids.ts (M8-002). Device and session identity+// for an incoming event: the salted device id, and the session id the API+// answers with synchronously while the worker persists the blob async.+import crypto from 'node:crypto';-import { generateDeviceId } from '@openpanel/core';-import type { IClickhouseSession } from '@openpanel/db';-import {- convertClickhouseDateToJs,- SESSION_TIMEOUT_MS,- sessionBuffer,-} from '@openpanel/db';+import type { SessionBuffer } from '../../../buffers/session-buffer';+import { SESSION_TIMEOUT_MS } from '../../../buffers/session-buffer';+import { generateDeviceId } from '../../../shared/profileId';+import type { IClickhouseSession } from '../../session/session.service';+import { convertClickhouseDateToJs } from '../../session/src/dates';++/** Only the read this module needs, so a caller may pass a narrower stub. */+export type SessionBufferReader = Pick<SessionBuffer, 'getExistingSession'>;++export interface DeviceIdResult {+ deviceId: string;+ sessionId: string;+}++const SESSION_ID_BYTES = 16;+const SESSION_ID_MIN_BYTES = 8;+const SESSION_ID_MAX_BYTES = 32;+const DEFAULT_SESSION_WINDOW_MS = 5 * 60 * 1000;+const DEFAULT_SESSION_GRACE_MS = 60 * 1000;+const SESSION_GRACE_CEILING_MS = 5000;+const SESSION_GRACE_WINDOW_DIVISOR = 6;+const BASE64_PLUS = /\+/g;+const BASE64_SLASH = /\//g;+const BASE64_PADDING = /=+$/g;export async function getDeviceId({projectId,@@ -14,6 +35,7 @@ export async function getDeviceId({salts,overrideDeviceId,eventTimeMs,+ sessionBuffer,}: {projectId: string;ip: string;@@ -23,7 +45,8 @@ export async function getDeviceId({/** Event timestamp (ms). Used to decide whether an existing session is* still within its idle window. Defaults to `Date.now()`. */eventTimeMs?: number;-}) {+ sessionBuffer: SessionBufferReader;+}): Promise<DeviceIdResult> {if (overrideDeviceId) {// A caller-supplied device id is stable (no salt rotation), so it's the only// candidate — resolve it through the same path as internal ids.@@ -31,6 +54,7 @@ export async function getDeviceId({projectId,deviceIds: [overrideDeviceId],eventTimeMs: eventTimeMs ?? Date.now(),+ sessionBuffer,});}@@ -55,14 +79,10 @@ export async function getDeviceId({projectId,deviceIds: [currentDeviceId, previousDeviceId],eventTimeMs: eventTimeMs ?? Date.now(),+ sessionBuffer,});}-interface DeviceIdResult {- deviceId: string;- sessionId: string;-}-/*** Returns true when an existing session is recent enough that the incoming* event should EXTEND it rather than start a new session.@@ -85,12 +105,14 @@ async function getInfoFromSession({projectId,deviceIds,eventTimeMs,+ sessionBuffer,}: {projectId: string;/** Candidate device ids in priority order (e.g. [current, previous] salt* windows, or just [override]). Deduped; the first is canonical. */deviceIds: string[];eventTimeMs: number;+ sessionBuffer: SessionBufferReader;}): Promise<DeviceIdResult> {const candidates = [...new Set(deviceIds.filter(Boolean))];const primary = candidates[0] ?? '';@@ -105,9 +127,9 @@ async function getInfoFromSession({));- for (const [i, session] of sessions.entries()) {+ for (const [index, session] of sessions.entries()) {if (session && withinIdleWindow(session, eventTimeMs)) {- return { deviceId: candidates[i]!, sessionId: session.id };+ return { deviceId: candidates[index]!, sessionId: session.id };}}} catch (error) {@@ -129,7 +151,10 @@ async function getInfoFromSession({projectId,deviceId: primary,eventMs: eventTimeMs,- graceMs: Math.min(5000, Math.floor(SESSION_TIMEOUT_MS / 6)),+ graceMs: Math.min(+ SESSION_GRACE_CEILING_MS,+ Math.floor(SESSION_TIMEOUT_MS / SESSION_GRACE_WINDOW_DIVISOR)+ ),windowMs: SESSION_TIMEOUT_MS,}),};@@ -143,7 +168,7 @@ async function getInfoFromSession({* - graceMs: 1 minute by default (events in first minute of a bucket map to previous bucket)* - Output: base64url, 128-bit (16 bytes) truncated from SHA-256*/-function getSessionId(params: {+export function getSessionId(params: {projectId: string;deviceId: string;eventMs?: number; // use event timestamp; defaults to Date.now()@@ -155,9 +180,9 @@ function getSessionId(params: {projectId,deviceId,eventMs = Date.now(),- windowMs = 5 * 60 * 1000,- graceMs = 60 * 1000,- bytes = 16,+ windowMs = DEFAULT_SESSION_WINDOW_MS,+ graceMs = DEFAULT_SESSION_GRACE_MS,+ bytes = SESSION_ID_BYTES,} = params;if (!projectId) {@@ -172,7 +197,7 @@ function getSessionId(params: {if (graceMs < 0 || graceMs >= windowMs) {throw new Error('graceMs must be >= 0 and < windowMs');}- if (bytes < 8 || bytes > 32) {+ if (bytes < SESSION_ID_MIN_BYTES || bytes > SESSION_ID_MAX_BYTES) {throw new Error('bytes must be between 8 and 32');}@@ -190,7 +215,7 @@ function getSessionId(params: {// base64urlreturn truncated.toString('base64')- .replace(/\+/g, '-')- .replace(/\//g, '_')- .replace(/=+$/g, '');+ .replace(BASE64_PLUS, '-')+ .replace(BASE64_SLASH, '_')+ .replace(BASE64_PADDING, '');}diff --git a/packages/core/src/modules/ingest/src/headers.ts b/packages/core/src/modules/ingest/src/headers.tsnew file mode 100644index 000000000..6a5231b46--- /dev/null+++ b/packages/core/src/modules/ingest/src/headers.ts@@ -0,0 +1,24 @@+// A lowercase header record — Fastify's `IncomingHttpHeaders`, or a `Headers`+// flattened with `Object.fromEntries`. The ingest path reads headers exactly+// as V1 read them, so no transport has to build a `Headers` from a raw record+// (which throws on a malformed name V1 simply ignored).++export type IngestHeaders = Record<string, string | string[] | undefined>;++export function headerValue(+ headers: IngestHeaders,+ name: string+): string | undefined {+ const value = headers[name];+ return Array.isArray(value) ? value[0] : value;+}++/** `Headers` -> the record shape the ingest path reads. Duplicated values are+ * joined by `Headers` itself, exactly as a Fastify record would hold them. */+export function toIngestHeaders(headers: Headers): IngestHeaders {+ const record: IngestHeaders = {};+ for (const [name, value] of headers) {+ record[name] = value;+ }+ return record;+}diff --git a/packages/core/src/modules/ingest/src/incoming-event.ts b/packages/core/src/modules/ingest/src/incoming-event.tsnew file mode 100644index 000000000..5a80de335--- /dev/null+++ b/packages/core/src/modules/ingest/src/incoming-event.ts@@ -0,0 +1,33 @@+// The Kafka topic payload and the producer signature over it.+//+// This is a WIRE FORMAT, not an internal type: a rolling deploy overlaps a V1+// producer with a V2 consumer on the same topic, so it stays byte-compatible+// with @openpanel/queue's `EventsQueuePayloadIncomingEvent['payload']`+// (ADR-004). A leaf file so `context.ts` can name the producer without+// reaching into the ingest service.++import type { GeoLocation } from '../../../clients/geo';+import type { parseUserAgent } from '../../../shared/parser-user-agent';+import type { ITrackPayload } from '../ingest.constants';++export interface IncomingEventPayload {+ // Minted by the producer (/track, /event) so a Kafka redelivery becomes an+ // identical-id row instead of a new one. Optional on the wire: a V1+ // producer omits it and a V1 consumer ignores it.+ id?: string;+ projectId: string;+ event: ITrackPayload & {+ timestamp: string | number;+ isTimestampFromThePast: boolean;+ };+ uaInfo: ReturnType<typeof parseUserAgent>;+ geo: GeoLocation;+ headers: Record<string, string | undefined>;+ deviceId: string;+ sessionId: string;+}++export type IncomingEventProducer = (+ payload: IncomingEventPayload,+ partitionKey: string+) => Promise<unknown>;diff --git a/packages/core/src/modules/insight/insight.jobs.test.ts b/packages/core/src/modules/insight/insight.jobs.test.tsindex cb5793fdf..7d7ecad70 100644--- a/packages/core/src/modules/insight/insight.jobs.test.ts+++ b/packages/core/src/modules/insight/insight.jobs.test.ts@@ -43,6 +43,7 @@ function stubJobCtx(gsc: {} as Services['gsc'],cohort: {} as Services['cohort'],import: {} as Services['import'],+ ingest: {} as Services['ingest'],organization: {} as Services['organization'],onboarding: {} as Services['onboarding'],session: {} as Services['session'],@@ -132,6 +133,7 @@ test('insightsDaily fans out one insightsProject enqueue per candidate, jobId degsc: {} as Services['gsc'],cohort: {} as Services['cohort'],import: {} as Services['import'],+ ingest: {} as Services['ingest'],organization: {} as Services['organization'],onboarding: {} as Services['onboarding'],session: {} as Services['session'],diff --git a/packages/core/src/modules/notification/notification.jobs.test.ts b/packages/core/src/modules/notification/notification.jobs.test.tsindex 5c328f3c7..0f821bcc4 100644--- a/packages/core/src/modules/notification/notification.jobs.test.ts+++ b/packages/core/src/modules/notification/notification.jobs.test.ts@@ -32,6 +32,7 @@ function stubJobCtx(notification: Partial<NotificationService>): JobCtx {cohort: {} as Services['cohort'],gsc: {} as Services['gsc'],import: {} as Services['import'],+ ingest: {} as Services['ingest'],organization: {} as Services['organization'],onboarding: {} as Services['onboarding'],session: {} as Services['session'],diff --git a/packages/core/src/modules/onboarding/onboarding.jobs.test.ts b/packages/core/src/modules/onboarding/onboarding.jobs.test.tsindex 355dec217..23ecb07cc 100644--- a/packages/core/src/modules/onboarding/onboarding.jobs.test.ts+++ b/packages/core/src/modules/onboarding/onboarding.jobs.test.ts@@ -34,6 +34,7 @@ function stubJobCtx(insight: {} as Services['insight'],gsc: {} as Services['gsc'],import: {} as Services['import'],+ ingest: {} as Services['ingest'],cohort: {} as Services['cohort'],organization: {} as Services['organization'],onboarding: onboarding as OnboardingService,diff --git a/packages/core/src/modules/organization/organization.jobs.test.ts b/packages/core/src/modules/organization/organization.jobs.test.tsindex 0b22dfa20..c92495b5e 100644--- a/packages/core/src/modules/organization/organization.jobs.test.ts+++ b/packages/core/src/modules/organization/organization.jobs.test.ts@@ -37,6 +37,7 @@ function stubJobCtx(insight: {} as Services['insight'],gsc: {} as Services['gsc'],import: {} as Services['import'],+ ingest: {} as Services['ingest'],cohort: {} as Services['cohort'],organization: organization as OrganizationService,onboarding: {} as Services['onboarding'],diff --git a/packages/core/src/modules/profile/profile.service.ts b/packages/core/src/modules/profile/profile.service.tsindex d85c590db..762d432b6 100644--- a/packages/core/src/modules/profile/profile.service.ts+++ b/packages/core/src/modules/profile/profile.service.ts@@ -569,7 +569,9 @@ export async function getProfileValues(input: {// ---- apps/api's /profile controller bodiesexport interface IdentifyProfileInput {- profileId: string;+ /** `string | number` because /track's identify payload allows both+ * (ingest.constants' `IProfileId`) and `upsertProfile` stores either. */+ profileId: string | number;firstName?: string;lastName?: string;email?: string;diff --git a/packages/core/src/rest.routes.ts b/packages/core/src/rest.routes.tsindex bd5ea742c..80563b023 100644--- a/packages/core/src/rest.routes.ts+++ b/packages/core/src/rest.routes.ts@@ -15,6 +15,7 @@ import { exportRoutes, insightsRoutes } from './modules/export/export.routes';import { gscRoutes } from './modules/gsc/gsc.routes';import { healthRoutes } from './modules/health/health.routes';import { importRoutes } from './modules/import/import.routes';+import { ingestRoutes } from './modules/ingest/ingest.routes';import { integrationRoutes } from './modules/integration/integration.routes';import { mcpRoutes } from './modules/mcp/mcp.routes';import { miscRoutes } from './modules/misc/misc.routes';@@ -47,9 +48,12 @@ const OPENAPI_EXCLUDED_PATHS = ['/metrics'];// stub that always returns null (import.routes.ts's header), and main.ts// does not mount `publicApiRoutes` until a real `AppDeps` exists (P3/P4/P8).// tools' `/site-checker` + `/ip-lookup` (M7-008) joined it too — unauthenticated-// like V1's, per the module map.+// like V1's, per the module map. ingest's `/track` + `/track/device-id`+// (M8-002) lead the list: the hot path, `clientAuth: { ingest: true }`, and+// the one route surface whose hook ORDER is part of the contract.export const publicApiRoutes = (deps: AppDeps) =>new Elysia({ name: 'core/public-api-routes' })+ .use(ingestRoutes(deps)).use(importRoutes(deps)).use(projectRoutes(deps)).use(clientRoutes(deps))diff --git a/packages/core/src/services.ts b/packages/core/src/services.tsindex 6ffaf85fa..4e9fcc43b 100644--- a/packages/core/src/services.ts+++ b/packages/core/src/services.ts@@ -28,6 +28,10 @@ import {createImportService,type ImportService,} from './modules/import/import.service';+import {+ createIngestService,+ type IngestService,+} from './modules/ingest/ingest.service';import {createInsightService,type InsightService,@@ -78,6 +82,7 @@ export interface Services {gsc: GscService;cohort: CohortService;import: ImportService;+ ingest: IngestService;organization: OrganizationService;onboarding: OnboardingService;notification: NotificationService;@@ -99,6 +104,7 @@ export function createServices(deps: ServiceDeps): Services {gsc: createGscService(deps),cohort: createCohortService(deps),import: createImportService(deps),+ ingest: createIngestService(deps),organization: createOrganizationService(deps),onboarding: createOnboardingService(deps),notification: createNotificationService(deps),diff --git a/packages/core/test/http-fixtures.ts b/packages/core/test/http-fixtures.tsindex b7c8b510b..f5ed3407a 100644--- a/packages/core/test/http-fixtures.ts+++ b/packages/core/test/http-fixtures.ts@@ -44,6 +44,7 @@ export function stubAppDeps(): AppDepsStub {clients: {},buffers: {} as Buffers,producers,+ produceIncomingEvent: () => Promise.resolve(),logger,config: { selfHosted: false },},diff --git a/packages/core/test/request-id-end-to-end.test.ts b/packages/core/test/request-id-end-to-end.test.tsindex 477ae8417..1a8712e6d 100644--- a/packages/core/test/request-id-end-to-end.test.ts+++ b/packages/core/test/request-id-end-to-end.test.ts@@ -146,6 +146,7 @@ function stubDeps() {clients: {},buffers: {} as Buffers,producers: producers as unknown as QueueProducerHandle,+ produceIncomingEvent: () => Promise.resolve(),logger: bindingLogger(lines),config: { selfHosted: false },};diff --git a/packages/core/test/rpc-fixtures.ts b/packages/core/test/rpc-fixtures.tsindex 619092877..4dd67c904 100644--- a/packages/core/test/rpc-fixtures.ts+++ b/packages/core/test/rpc-fixtures.ts@@ -92,6 +92,7 @@ export function stubHttpCtx(clients: {},buffers: {} as Buffers,producers: createRecordingProducers(queues),+ produceIncomingEvent: () => Promise.resolve(),logger,config: { selfHosted: false },};diff --git a/packages/validation/src/event-blocklist.ts b/packages/validation/src/event-blocklist.tsdeleted file mode 100644index e95540659..000000000--- a/packages/validation/src/event-blocklist.ts+++ /dev/null@@ -1,120 +0,0 @@-const MAX_EVENT_LENGTH = 80;--// Substrings that indicate attack/spam payloads-const BLOCKED_SUBSTRINGS = [- // === Security Scanner Domains ===- 'oastify.com',- 'burpcollaborator',- 'interact.sh',- 'oast.me',-- // === SQL Injection ===- 'pg_sleep',- 'waitfor delay',- 'xp_dirtree',- 'load_file(',- 'extractvalue(',- 'dbms_pipe.receive_message',- 'union select',-- // === Command Injection ===- 'nslookup ',- '/bin/sleep',- '/bin/bash',- 'cmd.exe',- 'wget+http',- 'wget http',- 'chmod+777',- 'chmod 777',-- // === Java/Code Execution ===- 'processbuilder',- 'runtime.getruntime',- 'java.lang.processbuilder',- 'eval-stdin.php',-- // === Path Traversal ===- '../',- '..\\',- '%2e%2e',- '%u002e%u002e',- '/etc/passwd',- '/etc/shadow',- 'win.ini',- 'system.ini',-- // === Template/SSTI Injection ===- '${',- '%{',-- // === XXE / XML Attacks ===- '<!doctype',- '<!entity',- '<xi:include',- 'xsi:schemalocation',-- // === SMTP Header Injection ===- '\r\n',- 'bcc:',-- // === Common File Scanning (paths as events) ===- 'phpinfo.php',- 'wp-config.php',- '.git/config',- '.env.backup',- '.env.bak',- '/vendor/phpunit/',-- // === Malware/Botnet Indicators ===- 'mozi.m',- '/setup.cgi?',- '/cgi-bin/',-- // === Ruby Object Inspection Leaks ===- '#<article:0x',- '#<video:0x',- '#<brand:0x',-- // === SQL Injection Patterns ===- 'exec master.dbo',- 'declare @',- "' and '",- "' or '",- "') or ",- "')and ",-];--// Patterns that indicate the "event" is actually a URL path being scanned-const PATH_SCAN_PATTERNS = [- /^\/[a-z_-]+\.(php|env|yml|yaml|json|xml|config|ini|bak|sql|log)/i,- /^\/\.[a-z]/i, // Hidden files like /.env, /.git- /^\/(wp-|wordpress)/i, // WordPress scanning- /^\/phpmyadmin/i,- /^\/.+\.php$/i, // Any .php path-];--/**- * Check if an event name should be blocked- * @param name - The event name to check- * @returns true if the event name should be blocked, false otherwise- */-export function isBlockedEventName(name: string): boolean {- // Length check - attack payloads are often very long- if (name.length > MAX_EVENT_LENGTH) return true;-- // Contains newlines (always suspicious for event names)- if (name.includes('\n') || name.includes('\r')) return true;-- // Substring blocklist (case-insensitive)- const lower = name.toLowerCase();- if (BLOCKED_SUBSTRINGS.some((blocked) => lower.includes(blocked))) {- return true;- }-- // Path scanning patterns- if (PATH_SCAN_PATTERNS.some((pattern) => pattern.test(name))) {- return true;- }-- return false;-}diff --git a/packages/validation/src/index.ts b/packages/validation/src/index.tsindex a5a440f96..2296666cd 100644--- a/packages/validation/src/index.ts+++ b/packages/validation/src/index.ts@@ -146,7 +146,6 @@ export const zEditOrganization = z.object({export * from './chat';export * from './cohort.validation';-export * from './event-blocklist';export * from './import.validation';export * from './track.validation';export * from './types.insights';diff --git a/packages/validation/src/track.validation.ts b/packages/validation/src/track.validation.tsindex 24fd95771..4e1988679 100644--- a/packages/validation/src/track.validation.ts+++ b/packages/validation/src/track.validation.ts@@ -1,259 +1,7 @@-import { RESERVED_EVENT_NAMES } from '@openpanel/constants';-import { z } from 'zod';-import { isBlockedEventName } from './event-blocklist';--// ----- Hand-written types (source of truth) ------//-// These interfaces are duplicated in code that ships in our SDK type-// declarations. We hand-write them (instead of using `z.infer<…>`) so:-// 1. Generated `.d.ts` files for the SDKs are readable plain TypeScript-// with no `import type { ITrackPayload } from '@openpanel/validation'`-// lines (the package isn't published).-// 2. The interfaces stay clean — no zod internals leaking through.-// Each schema below has `satisfies z.ZodType<…>` attached so a drift-// between the interface and the schema fails to compile.--export type IProfileId = string | number;--export interface IGroupPayload {- id: string;- type: string;- name: string;- properties?: Record<string, unknown>;-}--export interface IAssignGroupPayload {- groupIds: string[];- profileId?: IProfileId;-}--export interface ITrackPayload {- name: string;- properties?: Record<string, unknown>;- profileId?: IProfileId;- groups?: string[] | null;-}--export interface IIdentifyPayload {- profileId: IProfileId;- firstName?: string;- lastName?: string;- email?: string;- avatar?: string;- properties?: Record<string, unknown>;-}--export interface IIncrementPayload {- profileId: IProfileId;- property: string;- value?: number;-}--export interface IDecrementPayload {- profileId: IProfileId;- property: string;- value?: number;-}--export interface IAliasPayload {- profileId: IProfileId;- alias: string;-}--export interface IReplayPayload {- chunk_index: number;- events_count: number;- is_full_snapshot: boolean;- started_at: string;- ended_at: string;- payload: string;- // Server-issued session id (from a prior /track response) the SDK echoes back,- // so the chunk is filed under the right session without device resolution.- sessionId?: string;-}--export type ITrackHandlerPayload =- | { type: 'track'; payload: ITrackPayload }- | { type: 'identify'; payload: IIdentifyPayload }- | { type: 'increment'; payload: IIncrementPayload }- | { type: 'decrement'; payload: IDecrementPayload }- | { type: 'alias'; payload: IAliasPayload }- | { type: 'replay'; payload: IReplayPayload }- | { type: 'group'; payload: IGroupPayload }- | { type: 'assign_group'; payload: IAssignGroupPayload };--// ----- Schemas (each `satisfies` its hand-written interface) -------export const zProfileId = z.union([- z.string().min(1),- z.number(),-]) satisfies z.ZodType<IProfileId>;--export const zGroupPayload = z.object({- id: z.string().min(1),- type: z.string().min(1),- name: z.string().min(1),- properties: z.record(z.string(), z.unknown()).optional(),-}) satisfies z.ZodType<IGroupPayload>;--export const zAssignGroupPayload = z.object({- groupIds: z.array(z.string().min(1)),- profileId: zProfileId.optional(),-}) satisfies z.ZodType<IAssignGroupPayload>;--export const zTrackPayload = z- .object({- name: z.string().min(1),- properties: z.record(z.string(), z.unknown()).optional(),- profileId: zProfileId.optional(),- groups: z.array(z.string().min(1)).nullish(),- })- .refine((data) => !RESERVED_EVENT_NAMES.includes(data.name as any), {- message: `Event name cannot be one of the reserved names: ${RESERVED_EVENT_NAMES.join(', ')}`,- path: ['name'],- })- .refine((data) => !isBlockedEventName(data.name), {- message: 'Event name contains blocked content',- path: ['name'],- })- .refine(- (data) => {- if (data.name !== 'revenue') {- return true;- }- const revenue = data.properties?.__revenue;- if (revenue === undefined || revenue === null) {- return true;- }- const isInt = Number.isInteger(revenue);- if (isInt && Number(revenue) < 0) {- return false;- }- return isInt;- },- {- message: '__revenue must be an integer (no floats or strings)',- path: ['properties', '__revenue'],- }- ) satisfies z.ZodType<ITrackPayload>;--export const zIdentifyPayload = z.object({- profileId: zProfileId,- firstName: z.string().optional(),- lastName: z.string().optional(),- email: z.string().email().optional(),- avatar: z.string().url().optional(),- properties: z.record(z.string(), z.unknown()).optional(),-}) satisfies z.ZodType<IIdentifyPayload>;--export const zIncrementPayload = z.object({- profileId: zProfileId,- property: z.string().min(1),- value: z.number().positive().optional(),-}) satisfies z.ZodType<IIncrementPayload>;--export const zDecrementPayload = z.object({- profileId: zProfileId,- property: z.string().min(1),- value: z.number().positive().optional(),-}) satisfies z.ZodType<IDecrementPayload>;--export const zAliasPayload = z.object({- profileId: zProfileId,- alias: z.string().min(1),-}) satisfies z.ZodType<IAliasPayload>;--export const zReplayPayload = z.object({- chunk_index: z.number().int().min(0).max(65_535),- events_count: z.number().int().min(1),- is_full_snapshot: z.boolean(),- started_at: z.string().datetime(),- ended_at: z.string().datetime(),- payload: z.string().max(1_048_576 * 2), // 2MB max- sessionId: z.string().max(64).optional(),-}) satisfies z.ZodType<IReplayPayload>;--export const zTrackHandlerPayload = z.discriminatedUnion('type', [- z- .object({- type: z.enum(['track']),- payload: zTrackPayload,- })- .meta({ title: 'Track' }),- z- .object({- type: z.enum(['identify']),- payload: zIdentifyPayload,- })- .meta({ title: 'Identify' }),- z- .object({- type: z.enum(['increment']),- payload: zIncrementPayload,- })- .meta({ title: 'Increment' }),- z- .object({- type: z.enum(['decrement']),- payload: zDecrementPayload,- })- .meta({ title: 'Decrement' }),- z- .object({- type: z.enum(['alias']),- payload: zAliasPayload,- })- .meta({ title: 'Alias' }),- z- .object({- type: z.enum(['replay']),- payload: zReplayPayload,- })- .meta({ title: 'Replay' }),- z- .object({- type: z.enum(['group']),- payload: zGroupPayload,- })- .meta({ title: 'Group' }),- z- .object({- type: z.enum(['assign_group']),- payload: zAssignGroupPayload,- })- .meta({ title: 'Assign Group' }),-]) satisfies z.ZodType<ITrackHandlerPayload>;--// Deprecated types for beta version of the SDKs--export interface DeprecatedOpenpanelEventOptions {- profileId?: string;-}--export interface DeprecatedPostEventPayload {- name: string;- timestamp: string;- profileId?: string;- properties?: Record<string, unknown> & DeprecatedOpenpanelEventOptions;-}--export interface DeprecatedUpdateProfilePayload {- profileId: string;- firstName?: string;- lastName?: string;- email?: string;- avatar?: string;- properties?: Record<string, unknown>;-}--export interface DeprecatedIncrementProfilePayload {- profileId: string;- property: string;- value: number;-}--export interface DeprecatedDecrementProfilePayload {- profileId?: string;- property: string;- value: number;-}+// Moved into @openpanel/core's ingest module (M8-002, ADR-008's module map:+// ingest owns "C" — track.validation + event-blocklist + RESERVED_EVENT_NAMES+// are one file there). Re-exported here for existing @openpanel/validation+// importers (apps/api's ingest routes and hooks, the SDKs' shared types,+// apps/start) — same shape as ./import.validation.ts since M5-004. The SDK+// deep-import retarget to the constants subpath is P11.+export * from '@openpanel/core/modules/ingest/ingest.constants';diff --git a/pnpm-lock.yaml b/pnpm-lock.yamlindex fbc2a05c1..33abe4199 100644--- a/pnpm-lock.yaml+++ b/pnpm-lock.yaml@@ -206,9 +206,6 @@ importers:'@openpanel/tsconfig':specifier: workspace:*version: link:../../tooling/typescript- '@types/js-yaml':- specifier: ^4.0.9- version: 4.0.9'@types/ramda':specifier: ^0.30.2version: 0.30.2@@ -221,9 +218,6 @@ importers:'@types/ws':specifier: ^8.5.14version: 8.5.14- js-yaml:- specifier: ^4.1.0- version: 4.1.0tsdown:specifier: 0.14.2version: 0.14.2(@emnapi/[email protected])(@emnapi/[email protected])([email protected](@emnapi/[email protected])(@emnapi/[email protected]))([email protected])([email protected])@@ -1166,9 +1160,15 @@ importers:fast-deep-equal:specifier: ^3.1.3version: 3.1.3+ fast-json-stable-hash:+ specifier: ^1.0.3+ version: 1.0.3ipaddr.js:specifier: ^2.5.0version: 2.5.0+ js-yaml:+ specifier: ^4.1.0+ version: 4.1.1lru-cache:specifier: ^11.2.4version: 11.5.1@@ -1220,6 +1220,9 @@ importers:url-metadata:specifier: ^5.4.1version: 5.4.1+ uuid:+ specifier: ^9.0.1+ version: 9.0.1zod:specifier: 'catalog:'version: 4.3.6@@ -1230,6 +1233,9 @@ importers:'@openpanel/tsconfig':specifier: workspace:*version: link:../../tooling/typescript+ '@types/js-yaml':+ specifier: ^4.0.9+ version: 4.0.9'@types/qrcode':specifier: ^1.5.6version: 1.5.6@@ -1242,6 +1248,9 @@ importers:'@types/unzipper':specifier: ^0.10.10version: 0.10.11+ '@types/uuid':+ specifier: ^10.0.0+ version: 10.0.0bun-types:specifier: 1.4.0version: 1.4.0@@ -13829,10 +13838,6 @@ packages:resolution: {integrity: sha512-PMSmkqxr106Xa156c2M265Z+FTrPl+oxd/rgOQy2tijQeK5TxQ43psO1ZCwhVOSdnn+RzkzlRz/eY4BgJBYVpg==}hasBin: true- resolution: {integrity: sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==}- hasBin: true-resolution: {integrity: sha512-qQKT4zQxXl8lLwBtHMWwaTcGfFOZviOJet3Oy/xmGk2gZH677CJM9EvtfdSkgWcATZhj/55JZ0rmy3myCT5lsA==}hasBin: true@@ -34815,10 +34820,6 @@ snapshots:argparse: 1.0.10esprima: 4.0.1- dependencies:- argparse: 2.0.1-dependencies:argparse: 2.0.1