OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.
// Nothing here throws for a caller error: each transport owns its own status
// codes and bodies, so the pipeline returns a `TrackOutcome` instead.
import { generateId } from '@openpanel/shared';
import { parseUserAgent } from '@openpanel/shared/server';
import { assocPath, pathOr, pick } from 'ramda';
import { v4 as uuid } from 'uuid';
import type { Buffers } from '../../buffers/create-buffers';
import { resolveSessionTimeoutMs } from '../../buffers/session-buffer';
import type { DbScope } from '../../cacheable-per-deps';
import {
type AsnInfo,
type GeoLocation,
getAsnInfo,
getGeoLocation,
} from '../../clients/geo';
import type { Logger } from '../../logger';
import type { ServiceDeps, Services } from '../../services';
import { headerValue, type IngestHeaders } from '../../shared/headers';
import { generateDeviceId } from '../../shared/profileId';
import type { IAiPayload } from '../ai/ai.constants';
import { normalizeAiPayload } from '../ai/ai.service';
import { createBotEvent } from '../event/event.service';
import { getOrganizationByProjectIdCached } from '../organization/organization.service';
import {
getProfileById,
identifyProfile,
upsertProfile,
} from '../profile/profile.service';
import { getSalts } from '../salt/salt.service';
import { convertClickhouseDateToJs } from '../session/src/dates';
import {
type DeprecatedPostEventPayload,
type IAssignGroupPayload,
type IDecrementPayload,
type IGroupPayload,
type IIdentifyPayload,
type IIncrementPayload,
type IReplayPayload,
type ITrackHandlerPayload,
type ITrackPayload,
zLooseIdentifyPayload,
} from './ingest.constants';
import { isBot } from './src/bots/detect';
import { applyBotSuspicion, stripBotProperties } from './src/bots/suspicion';
import { isDuplicatedEvent } from './src/deduplicate';
import { getDeviceId } from './src/device-id';
import type {
IncomingEventPayload,
IncomingEventProducer,
} from './src/incoming-event';
export type { IngestHeaders } from '../../shared/headers';
export type { BotMatch } from './src/bots/detect';
export { detectBot, isBot } from './src/bots/detect';
export {
applyBotSuspicion,
BOT_CATEGORY_THRESHOLD,
type BotSuspicion,
stripBotProperties,
summarizeBotSignals,
} from './src/bots/suspicion';
export {
type IngestAuthErrorPayload,
type IngestAuthOutcome,
validateIngestRequest,
} from './src/client-auth';
export { isDuplicatedEvent } from './src/deduplicate';
export { getDeviceId } from './src/device-id';
export type {
IncomingEventPayload,
IncomingEventProducer,
} from './src/incoming-event';
/** The buffers this module touches. A subset of `AppDeps.buffers`. */
export type IngestBuffers = Pick<Buffers, 'session' | 'replay' | 'group'>;
export interface IngestTransport {
buffers: IngestBuffers;
produceIncomingEvent: IncomingEventProducer;
deps: ServiceDeps;
}
/** What the two context builders below need: the Postgres client the salt
* cache keys on, and the idle window every session decision reads. */
type IngestScope = DbScope & Pick<ServiceDeps, 'config' | 'logger'>;
const QUEUE_PAYLOAD_HEADERS = [
'user-agent',
'openpanel-sdk-name',
'openpanel-sdk-version',
'openpanel-client-id',
'request-id',
];
const MAX_OVERRIDE_DEVICE_ID_LENGTH = 64;
const ONE_MINUTE_MS = 60 * 1000;
const FIFTEEN_MINUTES_MS = 15 * ONE_MINUTE_MS;
/**
* A floor, not a window. Back-dating is a documented feature — the tracking
* docs describe it and the Mixpanel migration guide imports years-old events
* through it — so historical imports must keep working. This only catches
* values that cannot be a real event time: `__timestamp: 12345` is read as
* epoch milliseconds and lands on 1970-01-01, after which every date-filled
* series starts there.
*/
const EARLIEST_ACCEPTED_TIMESTAMP_MS = Date.UTC(2000, 0, 1);
const FALLBACK_USER_AGENT = 'unknown/1.0';
const NO_DEVICE = { deviceId: '', sessionId: '' } as const;
/** The whitelisted subset that ships with the queue payload. */
export function getStringHeaders(
headers: IngestHeaders
): Record<string, string | undefined> {
const stringHeaders: Record<string, string | undefined> = {};
for (const [key, value] of Object.entries(
pick(QUEUE_PAYLOAD_HEADERS, headers)
)) {
stringHeaders[key] = value ? String(value) : undefined;
}
return stringHeaders;
}
function getIdentity(body: ITrackHandlerPayload): IIdentifyPayload | undefined {
if (body.type === 'track') {
const submitted = body.payload.properties?.__identify;
if (submitted !== undefined) {
const identity = zLooseIdentifyPayload.safeParse(submitted);
if (identity.success) {
return identity.data;
}
}
return body.payload.profileId
? { profileId: String(body.payload.profileId) }
: undefined;
}
if (body.type === 'ai') {
return body.payload.profileId
? { profileId: String(body.payload.profileId) }
: undefined;
}
return undefined;
}
function sanitizeOverrideDeviceId(raw: unknown): string | undefined {
if (typeof raw !== 'string') {
return undefined;
}
const trimmed = raw.trim();
if (!trimmed || trimmed.length > MAX_OVERRIDE_DEVICE_ID_LENGTH) {
return undefined;
}
return trimmed;
}
/** Resolve a caller-supplied device id from `properties.__deviceId`. */
export function getOverrideDeviceId(
body: ITrackHandlerPayload
): string | undefined {
if (body.type === 'ai') {
return sanitizeOverrideDeviceId(body.payload.deviceId);
}
if (body.type !== 'track') {
return undefined;
}
return sanitizeOverrideDeviceId(body.payload?.properties?.__deviceId);
}
/**
* `properties.__timestamp` off whatever arrived: `/event` has no body schema,
* so `payload` can be a string or null, and `'properties' in payload` throws
* on a primitive.
*/
export function getPropertyTimestamp(payload: unknown): unknown {
if (typeof payload !== 'object' || payload === null) {
return undefined;
}
if (!('properties' in payload)) {
return undefined;
}
const { properties } = payload as { properties?: { __timestamp?: unknown } };
return properties?.__timestamp;
}
/** The back-date a validated `/track` body carries, by its declared type. */
export function getUserDefinedTimestamp(body: ITrackHandlerPayload): unknown {
if (body.type === 'ai') {
return body.payload.timestamp;
}
return getPropertyTimestamp(body.payload);
}
export function getTimestamp(
timestamp: number | undefined,
userDefinedTimestamp: unknown
): { timestamp: number; isTimestampFromThePast: boolean } {
const safeTimestamp = timestamp || Date.now();
const isUsable =
typeof userDefinedTimestamp === 'string' ||
typeof userDefinedTimestamp === 'number';
if (!(isUsable && userDefinedTimestamp)) {
return { timestamp: safeTimestamp, isTimestampFromThePast: false };
}
const clientTimestamp = new Date(userDefinedTimestamp);
const clientTimestampNumber = clientTimestamp.getTime();
if (
Number.isNaN(clientTimestampNumber) ||
clientTimestampNumber < EARLIEST_ACCEPTED_TIMESTAMP_MS ||
clientTimestampNumber > safeTimestamp + ONE_MINUTE_MS
) {
return { timestamp: safeTimestamp, isTimestampFromThePast: false };
}
const isTimestampFromThePast =
clientTimestampNumber < safeTimestamp - FIFTEEN_MINUTES_MS;
return { timestamp: clientTimestampNumber, isTimestampFromThePast };
}
export interface TrackContext {
projectId: string;
ip: string;
ua?: string;
// Whitelisted subset (getStringHeaders) that ships with the queue payload.
headers: Record<string, string | undefined>;
// Full raw request headers — bot signals need sec-ch-ua / sec-fetch-* /
// accept-language, which the whitelist above deliberately drops.
requestHeaders: IngestHeaders;
clientSecretAuth: boolean;
timestamp: { value: number; isFromPast: boolean };
identity?: IIdentifyPayload;
deviceId: string;
sessionId: string;
geo: GeoLocation;
asnInfo: AsnInfo;
}
/** What a transport hands the pipeline, after its own auth ran. */
export interface TrackRequest {
projectId: string | null | undefined;
clientIp: string;
headers: IngestHeaders;
clientSecretAuth: boolean;
/** The request-arrival timestamp. */
timestamp: number | undefined;
body: ITrackHandlerPayload;
}
export type TrackOutcome =
| { status: 'ok'; deviceId: string; sessionId: string }
| { status: 'alias-not-supported' }
| { status: 'invalid-type' }
| { status: 'missing-project-id' }
| { status: 'replay-missing-session-id' }
| { status: 'profile-not-found' }
| { status: 'profile-property-not-a-number' };
async function buildContext(
deps: IngestScope,
request: TrackRequest,
buffers: IngestBuffers
): Promise<TrackContext> {
const sessionTimeoutMs = resolveSessionTimeoutMs(deps.config);
const projectId = request.projectId as string;
const timestamp = getTimestamp(
request.timestamp,
getUserDefinedTimestamp(request.body)
);
const ip =
request.body.type === 'track' && request.body.payload.properties?.__ip
? (request.body.payload.properties.__ip as string)
: request.clientIp;
const ua = headerValue(request.headers, 'user-agent') ?? FALLBACK_USER_AGENT;
const headers = getStringHeaders(request.headers);
const identity = getIdentity(request.body);
const profileId = identity?.profileId;
if (profileId && request.body.type === 'track') {
request.body.payload.profileId = profileId;
}
const overrideDeviceId = getOverrideDeviceId(request.body);
// Get geo location (needed for track and identify) + ASN (bot detection).
// Both hit the same MaxMind readers keyed on the same IP and are cached, so
// resolving them together adds no meaningful latency.
const [geo, asnInfo, salts] = await Promise.all([
getGeoLocation(ip),
getAsnInfo(ip),
getSalts(deps),
]);
// The ip + user-agent hash identifies a browser; for an AI call it would
// only name the backend that made it, and show up as an anonymous user.
const isDeviceless = request.body.type === 'ai' && !overrideDeviceId;
const deviceIdResult = isDeviceless
? NO_DEVICE
: await getDeviceId({
projectId,
ip,
ua,
salts,
overrideDeviceId,
eventTimeMs: timestamp.timestamp,
sessionBuffer: buffers.session,
sessionTimeoutMs,
logger: deps.logger,
});
return {
projectId,
ip,
ua,
headers,
requestHeaders: request.headers,
clientSecretAuth: request.clientSecretAuth,
timestamp: {
value: timestamp.timestamp,
isFromPast: timestamp.isTimestampFromThePast,
},
identity,
deviceId: deviceIdResult.deviceId,
sessionId: deviceIdResult.sessionId,
geo,
asnInfo,
};
}
async function handleTrack(
payload: ITrackPayload,
context: TrackContext,
transport: IngestTransport
): Promise<void> {
const { projectId, deviceId, geo, headers, timestamp, sessionId } = context;
const uaInfo = parseUserAgent(headers['user-agent'], payload.properties);
// Mark (never block) likely bot traffic with __bot / __bot_reasons props.
payload.properties = applyBotSuspicion(payload.properties, {
asnInfo: context.asnInfo,
headers: context.requestHeaders,
clientSecretAuth: context.clientSecretAuth,
isServer: uaInfo.isServer,
});
const groupId = uaInfo.isServer
? payload.profileId
? `${projectId}:${payload.profileId}`
: undefined
: deviceId;
const promises: Promise<unknown>[] = [];
if (context.identity && Object.keys(context.identity).length > 1) {
promises.push(handleIdentify(transport.deps, context.identity, context));
}
const queueData: IncomingEventPayload = {
// The id of the ClickHouse row this event becomes, minted here so a
// redelivered Kafka message is a duplicate we can recognise.
id: uuid(),
projectId,
headers,
event: {
...payload,
groups: payload.groups ?? [],
timestamp: timestamp.value,
isTimestampFromThePast: timestamp.isFromPast,
},
uaInfo,
geo,
deviceId,
sessionId,
};
const partitionKey = groupId || generateId();
promises.push(transport.produceIncomingEvent(queueData, partitionKey));
await Promise.all(promises);
}
/**
* An AI event is a server event in all but name: no bot scoring, no session
* of its own. It keys the partition on the profile so one user's trace stays
* ordered, and ships the typed row draft next to the slim `events` payload.
*/
async function handleAi(
payload: IAiPayload,
context: TrackContext,
transport: IngestTransport
): Promise<void> {
const { projectId, deviceId, geo, headers, timestamp, sessionId } = context;
const normalized = normalizeAiPayload(payload);
const uaInfo = parseUserAgent(headers['user-agent'], undefined);
const profileId =
payload.profileId === undefined ? undefined : String(payload.profileId);
const queueData: IncomingEventPayload = {
id: uuid(),
projectId,
headers,
event: {
name: normalized.name,
properties: normalized.properties,
profileId,
groups: payload.groups ?? [],
timestamp: timestamp.value,
isTimestampFromThePast: timestamp.isFromPast,
},
uaInfo,
geo,
deviceId,
sessionId,
ai: normalized.draft,
};
const partitionKey = profileId
? `${projectId}:${profileId}`
: deviceId || generateId();
await transport.produceIncomingEvent(queueData, partitionKey);
}
async function handleIdentify(
deps: ServiceDeps,
payload: IIdentifyPayload,
context: TrackContext
): Promise<void> {
// Profiles must not carry forged bot verdicts either.
stripBotProperties(payload.properties);
const userAgent = parseUserAgent(context.ua, payload.properties);
await identifyProfile(deps, context.projectId, payload, {
geo: context.geo,
userAgent,
});
}
async function adjustProfileProperty(
deps: ServiceDeps,
payload: IIncrementPayload | IDecrementPayload,
projectId: string,
direction: 1 | -1
): Promise<TrackOutcome | null> {
const { profileId, property, value } = payload;
const profile = await getProfileById(deps, String(profileId), projectId);
if (!profile) {
return { status: 'profile-not-found' };
}
const parsed = Number.parseInt(
pathOr<string>('0', property.split('.'), profile.properties),
10
);
if (Number.isNaN(parsed)) {
return { status: 'profile-property-not-a-number' };
}
await upsertProfile(deps, {
id: profile.id,
projectId,
properties: assocPath(
property.split('.'),
parsed + direction * (value || 1),
profile.properties
),
isExternal: true,
});
return null;
}
/** Replay only needs the server-issued session id (the SDK echoes it back).
* Trust it — scoped to the authed project, unguessable, same trust level as
* event data. */
export async function handleReplay(
payload: IReplayPayload,
{
projectId,
sessionId,
}: { projectId: string; sessionId: string | undefined },
replayBuffer: IngestBuffers['replay']
): Promise<TrackOutcome | null> {
if (!sessionId) {
return { status: 'replay-missing-session-id' };
}
await replayBuffer.add({
project_id: projectId,
session_id: sessionId,
chunk_index: payload.chunk_index,
started_at: payload.started_at,
ended_at: payload.ended_at,
events_count: payload.events_count,
is_full_snapshot: payload.is_full_snapshot,
payload: payload.payload,
});
return null;
}
async function handleGroup(
payload: IGroupPayload,
context: TrackContext,
groupBuffer: IngestBuffers['group']
): Promise<void> {
const { id, type, name, properties = {} } = payload;
await groupBuffer.add({
id,
projectId: context.projectId,
type,
name,
properties,
});
}
async function handleAssignGroup(
deps: ServiceDeps,
payload: IAssignGroupPayload,
context: TrackContext
): Promise<void> {
const profileId = payload.profileId ?? context.deviceId;
if (!profileId) {
return;
}
await upsertProfile(deps, {
id: String(profileId),
projectId: context.projectId,
isExternal: !!payload.profileId,
groups: payload.groupIds,
});
}
/** `POST /track`. An unknown type still resolves geo/device before it is
* refused. */
export async function ingestTrack(
request: TrackRequest,
transport: IngestTransport
): Promise<TrackOutcome> {
const body = request.body;
if (body.type === 'alias') {
return { status: 'alias-not-supported' };
}
if (!request.projectId) {
return { status: 'missing-project-id' };
}
const context = await buildContext(
transport.deps,
request,
transport.buffers
);
switch (body.type) {
case 'track':
await handleTrack(body.payload, context, transport);
break;
case 'identify':
await handleIdentify(transport.deps, body.payload, context);
break;
case 'increment': {
const failure = await adjustProfileProperty(
transport.deps,
body.payload,
context.projectId,
1
);
if (failure) {
return failure;
}
break;
}
case 'decrement': {
const failure = await adjustProfileProperty(
transport.deps,
body.payload,
context.projectId,
-1
);
if (failure) {
return failure;
}
break;
}
case 'replay': {
// BACKCOMPAT(replay-sessionid): TEMPORARY legacy branch, remove when new SDK is fully deployed
const failure = await handleReplay(
body.payload,
{
projectId: context.projectId,
sessionId: body.payload.sessionId || context.sessionId,
},
transport.buffers.replay
);
if (failure) {
return failure;
}
break;
}
case 'group':
await handleGroup(body.payload, context, transport.buffers.group);
break;
case 'assign_group':
await handleAssignGroup(transport.deps, body.payload, context);
break;
case 'ai':
await handleAi(body.payload, context, transport);
break;
default:
return { status: 'invalid-type' };
}
return {
status: 'ok',
deviceId: context.deviceId,
sessionId: context.sessionId,
};
}
/**
* Two things worth knowing about the shape of this:
*
* It gates on `windDownStep`, not on `subscriptionState`. Every expired trial
* is already in `trial_expired`, so gating on the state would block thousands
* of orgs at once, skipping the four warning emails entirely.
*
* It answers 202, not 402 or 403 (the caller does; this returns the verdict).
* The SDKs treat only 401 and 2xx as terminal, so a "correct" status code
* would multiply traffic from exactly the clients we are trying to quiet down.
*/
const WIND_DOWN_BLOCKED_STEPS = new Set(['blocked', 'final_warning']);
export async function isIngestionWoundDown(
deps: ServiceDeps,
request: {
projectId: string | null | undefined;
selfHosted: boolean;
logger: Logger;
}
): Promise<boolean> {
if (request.selfHosted || !request.projectId) {
return false;
}
const { projectId, logger } = request;
try {
// Cached for 5 minutes and invalidated by the Polar webhook, so paying
// again lifts the block on the next checkout rather than on a TTL.
const organization = await getOrganizationByProjectIdCached(
deps,
projectId
);
if (
!(
organization?.windDownStep &&
WIND_DOWN_BLOCKED_STEPS.has(organization.windDownStep)
)
) {
return false;
}
logger.info(
{
organizationId: organization.id,
projectId,
windDownStep: organization.windDownStep,
},
'Ingestion blocked by wind-down'
);
return true;
} catch (error) {
// Fail open. Dropping a paying customer's events because Redis or Postgres
// hiccuped is far worse than letting a blocked org through for a tick.
logger.error(
{ err: error, projectId },
'Wind-down check failed, allowing ingestion'
);
return false;
}
}
/**
* `POST /event` — the legacy compat route, kept because production still has
* projects posting to it.
*
* It is NOT `ingestTrack` with a different body: the legacy payload carries a
* client-supplied ISO `timestamp`, has no `type` discriminator, no schema, and
* no identify/increment/replay branches, so it builds its own queue payload
* and mints the event id at the producer like `/track` does.
*/
export type LegacyEventOutcome =
| { status: 'ok' }
| { status: 'missing-project-id' };
function legacyProfileId(raw: unknown): string | undefined {
return typeof raw === 'string' || typeof raw === 'number'
? String(raw)
: undefined;
}
export async function ingestLegacyEvent(
request: {
projectId: string | null | undefined;
clientIp: string;
headers: IngestHeaders;
clientSecretAuth: boolean;
/** The request-arrival timestamp. */
timestamp: number | undefined;
body: DeprecatedPostEventPayload | null | undefined;
},
transport: IngestTransport
): Promise<LegacyEventOutcome> {
const { projectId } = request;
if (!projectId) {
return { status: 'missing-project-id' };
}
// An event with no name is not an event. Writing one with `name = ''` shows
// up in every picker and breakdown. The status stays 202 — this route has
// no body schema and the HTTP contract must not change — but nothing is
// written.
const name = (request.body as { name?: unknown } | null | undefined)?.name;
if (typeof name !== 'string' || name.trim() === '') {
return { status: 'ok' };
}
const { timestamp, isTimestampFromThePast } = getTimestamp(
request.timestamp,
getPropertyTimestamp(request.body)
);
const ip = request.clientIp;
const ua = headerValue(request.headers, 'user-agent') ?? FALLBACK_USER_AGENT;
const headers = getStringHeaders(request.headers);
const [salts, geo, asnInfo] = await Promise.all([
getSalts(transport.deps),
getGeoLocation(ip),
getAsnInfo(ip),
]);
const { deviceId, sessionId } = await getDeviceId({
projectId,
ip,
ua,
salts,
sessionBuffer: transport.buffers.session,
sessionTimeoutMs: resolveSessionTimeoutMs(transport.deps.config),
logger: transport.deps.logger,
});
const uaInfo = parseUserAgent(ua, request.body?.properties);
// Mark (never block) likely bot traffic with __bot / __bot_reasons props.
// This deprecated route has no body schema, so body can be null/undefined.
if (request.body) {
request.body.properties = applyBotSuspicion(request.body.properties, {
asnInfo,
headers: request.headers,
clientSecretAuth: request.clientSecretAuth,
isServer: uaInfo.isServer,
});
}
const groupId = uaInfo.isServer
? `${projectId}:${request.body?.profileId ?? generateId()}`
: deviceId;
await transport.produceIncomingEvent(
{
// See handleTrack: the producer mints the event id.
id: uuid(),
projectId,
headers,
// Picked, not spread: this route has no body schema, and a stray field
// (`groups` as an object, the body's `clientSecret`) would otherwise
// ride into Kafka and onto the shared buffers' insert batches.
event: {
name,
profileId: legacyProfileId(request.body?.profileId),
properties: request.body?.properties,
groups: [],
timestamp,
isTimestampFromThePast,
},
uaInfo,
geo,
deviceId,
sessionId: sessionId ?? '',
} as IncomingEventPayload,
groupId
);
return { status: 'ok' };
}
export type DeviceIdentity =
| { status: 'ok'; deviceId: string; sessionId: string; message: string }
| { status: 'missing-project-id' }
| { status: 'missing-ip' }
| { status: 'missing-user-agent' };
/** `GET /track/device-id`. */
export async function fetchDeviceIdentity(
deps: IngestScope,
request: {
projectId: string | null | undefined;
clientIp: string | undefined;
headers: IngestHeaders;
},
buffers: Pick<IngestBuffers, 'session'>,
logger: Logger
): Promise<DeviceIdentity> {
const salts = await getSalts(deps);
const sessionTimeoutMs = resolveSessionTimeoutMs(deps.config);
const projectId = request.projectId;
if (!projectId) {
return { status: 'missing-project-id' };
}
const ip = request.clientIp;
if (!ip) {
return { status: 'missing-ip' };
}
const ua = headerValue(request.headers, 'user-agent');
if (!ua) {
return { status: 'missing-user-agent' };
}
const currentDeviceId = generateDeviceId({
salt: salts.current,
origin: projectId,
ip,
ua,
});
const previousDeviceId = generateDeviceId({
salt: salts.previous,
origin: projectId,
ip,
ua,
});
try {
const [current, previous] = await buffers.session.getExistingSessions(
projectId,
[currentDeviceId, previousDeviceId]
);
// Blob has no TTL — only treat the session as "current" if its last
// event is within the idle window. Otherwise the SDK should ask the
// server to start a fresh session id on the next event.
const now = Date.now();
if (current && isLiveSession(current, now, sessionTimeoutMs)) {
return {
status: 'ok',
deviceId: currentDeviceId,
sessionId: current.id,
message: 'current session exists for this device id',
};
}
if (previous && isLiveSession(previous, now, sessionTimeoutMs)) {
return {
status: 'ok',
deviceId: previousDeviceId,
sessionId: previous.id,
message: 'previous session exists for this device id',
};
}
} catch (error) {
logger.error(
{ err: error },
'Error getting session end GET /track/device-id'
);
}
return {
status: 'ok',
deviceId: currentDeviceId,
sessionId: '',
message: 'No session exists for this device id',
};
}
/** The web SDK can fire the same event twice; this catches the duplicate. */
export async function isDuplicateIngestRequest(request: {
method: string;
clientIp: string | undefined;
headers: IngestHeaders;
body: unknown;
}): Promise<boolean> {
const ip = request.clientIp;
const origin = headerValue(request.headers, 'origin');
const clientId = headerValue(request.headers, 'openpanel-client-id');
const isReplay = isTrackBody(request) && request.body.type === 'replay';
const shouldCheck = ip && origin && clientId && !isReplay;
if (!shouldCheck) {
return false;
}
return await isDuplicatedEvent({
ip,
origin,
payload: request.body as Record<string, unknown>,
projectId: clientId,
});
}
function isTrackBody(request: {
method: string;
body: unknown;
}): request is { method: string; body: ITrackHandlerPayload } {
if (request.method !== 'POST') {
return false;
}
if (!request.body) {
return false;
}
if (typeof request.body !== 'object' || Array.isArray(request.body)) {
return false;
}
return 'type' in request.body;
}
export type BotVerdict = { name: string; type: string } | null;
/**
* Requests authenticated with a client secret come from server-side SDKs
* (node, php, go, rust, java, python, …). That auth is a far
* stronger signal of legitimate first-party traffic than the user agent, so
* never treat them as bots — bot detection is for public/frontend
* (origin-authenticated) traffic.
*/
export async function checkIngestBot(
deps: ServiceDeps,
request: {
headers: IngestHeaders;
clientSecretAuth: boolean;
projectId: string | null | undefined;
body: unknown;
}
): Promise<BotVerdict> {
if (request.clientSecretAuth) {
return null;
}
const userAgent = headerValue(request.headers, 'user-agent');
const bot = userAgent ? await isBot(userAgent) : null;
if (!(bot && request.projectId)) {
return null;
}
const path = getBotEventPath(request.body);
if (path) {
await createBotEvent(deps, {
...bot,
projectId: request.projectId,
path,
createdAt: new Date(),
});
}
return bot;
}
function getBotEventPath(body: unknown): string | undefined {
if (!body || typeof body !== 'object') {
return undefined;
}
if ('type' in body && (body as ITrackHandlerPayload).type === 'track') {
const properties = (
body as { payload?: { properties?: Record<string, unknown> } }
).payload?.properties;
return (properties?.__path || properties?.path) as string | undefined;
}
// Handle deprecated events (v1)
if ('name' in body && 'properties' in body) {
const properties = (body as { properties?: Record<string, unknown> })
.properties;
return (properties?.__path || properties?.path) as string | undefined;
}
return undefined;
}
function isLiveSession(
session: { ended_at: string },
now: number,
sessionTimeoutMs: number
): boolean {
return (
now - convertClickhouseDateToJs(session.ended_at).getTime() <
sessionTimeoutMs
);
}
export function createIngestService(
deps: ServiceDeps,
_services: () => Services
) {
const track = (
request: TrackRequest,
produceIncomingEvent: IncomingEventProducer
): Promise<TrackOutcome> =>
ingestTrack(request, {
buffers: deps.buffers,
produceIncomingEvent,
deps,
});
const checkBot = (
request: Parameters<typeof checkIngestBot>[1]
): Promise<BotVerdict> => checkIngestBot(deps, request);
const isDuplicate = isDuplicateIngestRequest;
return { track, checkBot, isDuplicate };
}