Skip to content

🐸 Rybbit - open-source and privacy-friendly alternative to Google Analytics that is 10x more intuitive.

refactor(tracker): resolve the Tracking Request once, behind one ingestion seam (#1118)

Goldflag4ca4117a17cb
diff --git a/server/src/api/featureFlags/index.ts b/server/src/api/featureFlags/index.ts
index 5c7605e46..c86db73ff 100644
--- a/server/src/api/featureFlags/index.ts
+++ b/server/src/api/featureFlags/index.ts
@@ -8,9 +8,8 @@ import { getLocation } from "../../db/geolocation/geolocation.js";
import { db } from "../../db/postgres/postgres.js";
import { featureFlags, userProfiles } from "../../db/postgres/schema.js";
import { siteConfig } from "../../lib/siteConfig.js";
-import { getRequestUserAgent } from "../../services/tracker/requestIdentity.js";
import { processResults } from "../analytics/utils/utils.js";
-import { getDeviceType } from "../../utils.js";
+import { getDeviceType, getRequestUserAgent } from "../../utils.js";
import { resolveClientIp } from "../../services/tracker/resolveClientIp.js";
import {
getFeatureFlagDefinitionsForRuntime,
@@ -324,7 +323,7 @@ async function evaluateFeatureFlagsForRuntime(
]);
const location = locationByIp[ipAddress];
- const ua = userAgentParser(getRequestUserAgent(request));
+ const ua = userAgentParser(getRequestUserAgent(request.headers));
const deviceType = getDeviceType(body.screenWidth || 0, body.screenHeight || 0, ua);
const assignments = evaluateFeatureFlagDefinitions(
diff --git a/server/src/db/geolocation/asn.ts b/server/src/db/geolocation/asn.ts
index 6e22723b2..e5b656044 100644
--- a/server/src/db/geolocation/asn.ts
+++ b/server/src/db/geolocation/asn.ts
@@ -44,3 +44,34 @@ export function lookupAsn(ip: string): AsnInfo | null {
return null;
}
}
+
+/**
+ * An ASN resolver. Everything that needs an ASN takes one of these rather than
+ * importing `lookupAsn` directly, so a caller can hand the same memoised
+ * resolver to every module involved in one request β€” and tests can inject a
+ * fake without the MaxMind DB.
+ */
+export type AsnLookup = (ip: string) => AsnInfo | null;
+
+/**
+ * A per-request ASN resolver that answers each IP once.
+ *
+ * A single tracking request asks about the same handful of IPs from IP
+ * resolution, identity bucketing, sticky re-attachment, exclusion matching and
+ * bot detection. The reader is an in-memory mmdb, so each lookup is cheap, but
+ * repeating it several times per event on the hot ingestion path is pure waste.
+ * Memoise for the life of one request only β€” never longer, since the DB is
+ * reloaded in place.
+ */
+export function createAsnLookup(lookup: AsnLookup = lookupAsn): AsnLookup {
+ const resolved = new Map<string, AsnInfo | null>();
+
+ return ip => {
+ const cached = resolved.get(ip);
+ if (cached !== undefined) return cached;
+
+ const info = lookup(ip);
+ resolved.set(ip, info);
+ return info;
+ };
+}
diff --git a/server/src/services/sites/siteExclusionDecision.ts b/server/src/services/sites/siteExclusionDecision.ts
index 1de79ee50..2c24a8bff 100644
--- a/server/src/services/sites/siteExclusionDecision.ts
+++ b/server/src/services/sites/siteExclusionDecision.ts
@@ -1,4 +1,4 @@
-import { lookupAsn } from "../../db/geolocation/asn.js";
+import { lookupAsn, type AsnLookup } from "../../db/geolocation/asn.js";
import { getLocation } from "../../db/geolocation/geolocation.js";
import { matchesCIDR, matchesRange } from "../../lib/ipUtils.js";
import { logger } from "../../lib/logger/logger.js";
@@ -29,6 +29,12 @@ export type SiteExclusionRequest = {
querystring?: string;
hostname?: string;
userAgent?: string;
+ /**
+ * ASN resolver for ASN exclusions. Tracking ingestion passes its
+ * request-scoped one so candidate IPs already looked up elsewhere in the
+ * request aren't resolved a second time.
+ */
+ lookupAsn?: AsnLookup;
};
export type SiteExclusionReason = "ip" | "asn" | "country" | "path" | "query_param" | "hostname" | "user_agent";
@@ -167,8 +173,9 @@ export async function decideSiteExclusion(
configuration.excludedASNs.map(parseAsnPattern).filter((asn): asn is number => asn !== null)
);
+ const asnLookup = request.lookupAsn ?? lookupAsn;
for (const ip of ipsToCheck) {
- const asnInfo = lookupAsn(ip);
+ const asnInfo = asnLookup(ip);
if (asnInfo && excludedAsnNumbers.has(asnInfo.asn)) {
return excluded("asn", "ASN", `AS${asnInfo.asn}`);
}
diff --git a/server/src/services/tracker/botBlocking/headerHeuristics.test.ts b/server/src/services/tracker/botBlocking/headerHeuristics.test.ts
index 0e73f3e27..1a5d318a6 100644
--- a/server/src/services/tracker/botBlocking/headerHeuristics.test.ts
+++ b/server/src/services/tracker/botBlocking/headerHeuristics.test.ts
@@ -1,17 +1,12 @@
import { describe, expect, it } from "vitest";
-import { FastifyRequest } from "fastify";
import { detectBot } from "./headerHeuristics.js";
-function requestWithHeaders(headers: Record<string, string | string[]>): FastifyRequest {
- return { headers } as unknown as FastifyRequest;
-}
-
const browserUserAgent =
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36";
describe("header heuristic bot detection", () => {
it("detects scripting framework user agents", () => {
- const result = detectBot(requestWithHeaders({}), "python-requests/2.31.0");
+ const result = detectBot({}, "python-requests/2.31.0");
expect(result).toEqual({
isBot: true,
@@ -21,7 +16,7 @@ describe("header heuristic bot detection", () => {
});
it("detects default OkHttp React Native requests as scripting framework traffic", () => {
- const result = detectBot(requestWithHeaders({}), "okhttp/4.12.0");
+ const result = detectBot({}, "okhttp/4.12.0");
expect(result).toEqual({
isBot: true,
@@ -32,10 +27,10 @@ describe("header heuristic bot detection", () => {
it("does not block React Native SDK requests that send SDK HTTP headers", () => {
const result = detectBot(
- requestWithHeaders({
+ {
accept: "application/json",
"accept-language": "en-US,en;q=0.9",
- }),
+ },
"Mozilla/5.0 (Linux; Android 36) AppleWebKit/537.36 (KHTML, like Gecko) RybbitReactNative/0.1.1"
);
@@ -47,7 +42,7 @@ describe("header heuristic bot detection", () => {
});
it("scores missing browser headers", () => {
- const result = detectBot(requestWithHeaders({}), browserUserAgent);
+ const result = detectBot({}, browserUserAgent);
expect(result).toMatchObject({
isBot: true,
@@ -60,14 +55,14 @@ describe("header heuristic bot detection", () => {
});
it("does not score complete browser fetch headers", () => {
- const request = requestWithHeaders({
+ const browserHeaders = {
accept: "*/*",
"accept-encoding": "gzip, br",
"accept-language": "en-US,en;q=0.9",
"sec-fetch-site": "cross-site",
- });
+ };
- expect(detectBot(request, browserUserAgent)).toEqual({
+ expect(detectBot(browserHeaders, browserUserAgent)).toEqual({
isBot: false,
score: 0,
reason: undefined,
diff --git a/server/src/services/tracker/botBlocking/headerHeuristics.ts b/server/src/services/tracker/botBlocking/headerHeuristics.ts
index 94ea8eacf..c5a62b603 100644
--- a/server/src/services/tracker/botBlocking/headerHeuristics.ts
+++ b/server/src/services/tracker/botBlocking/headerHeuristics.ts
@@ -1,4 +1,4 @@
-import { FastifyRequest } from "fastify";
+import type { IncomingHttpHeaders } from "http";
import { BOT_SCORE_THRESHOLD } from "./config.js";
interface BotDetectionResult {
@@ -71,10 +71,9 @@ function claimsModernOS(ua: string): boolean {
* Supplements the UA-pattern check (uaBots/classifyUA) with analysis of headers
* that real browsers always send but scripting clients typically miss.
*/
-export function detectBot(request: FastifyRequest, userAgent: string): BotDetectionResult {
+export function detectBot(headers: IncomingHttpHeaders, userAgent: string): BotDetectionResult {
let score = 0;
const reasons: string[] = [];
- const headers = request.headers;
const lowerUA = userAgent.toLowerCase();
// 1. Known bot framework signatures in UA (instant kill)
diff --git a/server/src/services/tracker/botBlocking/index.test.ts b/server/src/services/tracker/botBlocking/index.test.ts
index 1234e2e92..0bee1fb63 100644
--- a/server/src/services/tracker/botBlocking/index.test.ts
+++ b/server/src/services/tracker/botBlocking/index.test.ts
@@ -1,4 +1,3 @@
-import { FastifyRequest } from "fastify";
import { beforeEach, describe, expect, it, vi } from "vitest";
import { lookupAsn } from "../../../db/geolocation/asn.js";
import { resetAnomalyScorerForTests, setRedisAnomalyEnabledForTests } from "./anomalyScorer.js";
@@ -16,10 +15,6 @@ vi.mock("../../../db/redis/redis.js", () => ({
anomalyObserve: vi.fn(),
}));
-function requestWithHeaders(headers: Record<string, string | string[]>): FastifyRequest {
- return { headers } as unknown as FastifyRequest;
-}
-
const basePayload = {
siteId: "site_123",
ipAddress: "203.0.113.10",
@@ -49,7 +44,7 @@ describe("checkBotBlocking", () => {
it("does nothing when bot blocking is disabled", async () => {
const result = await checkBotBlocking({
- request: requestWithHeaders({}),
+ headers: {},
blockBots: false,
payload: basePayload,
});
@@ -59,7 +54,7 @@ describe("checkBotBlocking", () => {
it("skips verified trusted server-side ingestion requests", async () => {
const result = await checkBotBlocking({
- request: requestWithHeaders({}),
+ headers: {},
blockBots: true,
trustedServerSideIngestion: true,
payload: basePayload,
@@ -70,7 +65,7 @@ describe("checkBotBlocking", () => {
it("does not bypass bot blocking for an unverified bearer header", async () => {
const result = await checkBotBlocking({
- request: requestWithHeaders({ authorization: "Bearer token" }),
+ headers: { authorization: "Bearer token" },
blockBots: true,
payload: basePayload,
});
@@ -83,12 +78,12 @@ describe("checkBotBlocking", () => {
it("counts requests before bot-blocking bypasses for percentage stats", async () => {
await checkBotBlocking({
- request: requestWithHeaders({}),
+ headers: {},
blockBots: false,
payload: basePayload,
});
await checkBotBlocking({
- request: requestWithHeaders({}),
+ headers: {},
blockBots: true,
trustedServerSideIngestion: true,
payload: { ...basePayload, clientBotScore: 0, clientBotSignalMask: 0 },
@@ -110,7 +105,7 @@ describe("checkBotBlocking", () => {
it("returns bot event properties for detected bots", async () => {
const result = await checkBotBlocking({
- request: requestWithHeaders({}),
+ headers: {},
blockBots: true,
payload: basePayload,
});
@@ -145,7 +140,7 @@ describe("checkBotBlocking", () => {
// Android React Native ships requests through okhttp with no browser headers.
// On a web site this trips both the UA-pattern and header-heuristic layers...
const asWeb = await checkBotBlocking({
- request: requestWithHeaders({ "user-agent": "okhttp/4.12.0" }),
+ headers: { "user-agent": "okhttp/4.12.0" },
blockBots: true,
payload: { ...basePayload, userAgent: "okhttp/4.12.0", clientBotScore: 0, clientBotSignalMask: 0 },
});
@@ -153,7 +148,7 @@ describe("checkBotBlocking", () => {
// ...but a mobile/app site treats it as legitimate first-party traffic.
const asMobile = await checkBotBlocking({
- request: requestWithHeaders({ "user-agent": "okhttp/4.12.0" }),
+ headers: { "user-agent": "okhttp/4.12.0" },
blockBots: true,
isMobileSite: true,
payload: { ...basePayload, userAgent: "okhttp/4.12.0", clientBotScore: 0, clientBotSignalMask: 0 },
@@ -163,7 +158,7 @@ describe("checkBotBlocking", () => {
it("still flags mobile traffic through client signals, ASN, and rate anomaly", async () => {
const result = await checkBotBlocking({
- request: requestWithHeaders({ "user-agent": "okhttp/4.12.0" }),
+ headers: { "user-agent": "okhttp/4.12.0" },
blockBots: true,
isMobileSite: true,
payload: {
@@ -183,25 +178,25 @@ describe("checkBotBlocking", () => {
});
it("records client bot score and signal aggregates for inspected requests", async () => {
- const request = requestWithHeaders(browserHeaders);
+ const headers = browserHeaders;
await checkBotBlocking({
- request,
+ headers,
blockBots: true,
payload: basePayload,
});
await checkBotBlocking({
- request,
+ headers,
blockBots: true,
payload: { ...basePayload, clientBotScore: 0, clientBotSignalMask: 0 },
});
await checkBotBlocking({
- request,
+ headers,
blockBots: true,
payload: { ...basePayload, clientBotScore: 1, clientBotSignalMask: clientBotSignalMasks.swiftShader },
});
await checkBotBlocking({
- request,
+ headers,
blockBots: true,
payload: {
...basePayload,
@@ -210,7 +205,7 @@ describe("checkBotBlocking", () => {
},
});
await checkBotBlocking({
- request,
+ headers,
blockBots: true,
payload: { ...basePayload, clientBotScore: 3, clientBotSignalMask: clientBotSignalMasks.automationApi },
});
@@ -245,10 +240,10 @@ describe("checkBotBlocking", () => {
it("collects every matching bot signal before returning", async () => {
const result = await checkBotBlocking({
- request: requestWithHeaders({
+ headers: {
"user-agent":
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/120.0.0.0 Safari/537.36",
- }),
+ },
blockBots: true,
payload: {
...basePayload,
@@ -279,10 +274,10 @@ describe("checkBotBlocking", () => {
it("moves default viewport fingerprints into client signals", async () => {
const result = await checkBotBlocking({
- request: requestWithHeaders({
+ headers: {
...browserHeaders,
"user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/120 Safari/537.36",
- }),
+ },
blockBots: true,
payload: {
...basePayload,
@@ -308,10 +303,10 @@ describe("checkBotBlocking", () => {
it("convicts the 1280x1200 headless window geometry on a desktop UA", async () => {
const result = await checkBotBlocking({
- request: requestWithHeaders({
+ headers: {
...browserHeaders,
"user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/116 Safari/537.36",
- }),
+ },
blockBots: true,
payload: {
...basePayload,
@@ -330,11 +325,11 @@ describe("checkBotBlocking", () => {
it("leaves 1280x1200 alone on a mobile UA, where it is not a headless tell", async () => {
const result = await checkBotBlocking({
- request: requestWithHeaders({
+ headers: {
...browserHeaders,
"user-agent":
"Mozilla/5.0 (iPhone; CPU iPhone OS 18_5 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.5 Mobile/15E148 Safari/604.1",
- }),
+ },
blockBots: true,
payload: {
...basePayload,
@@ -353,7 +348,7 @@ describe("checkBotBlocking", () => {
// but both occur on real devices (prerendering, low-end GPUs) β€” weak
// signals corroborate, never convict.
const result = await checkBotBlocking({
- request: requestWithHeaders(browserHeaders),
+ headers: browserHeaders,
blockBots: true,
payload: {
...basePayload,
@@ -369,7 +364,7 @@ describe("checkBotBlocking", () => {
// Without the mask the score cannot be decomposed into strong vs. weak
// signals, so it can only ever corroborate.
const result = await checkBotBlocking({
- request: requestWithHeaders(browserHeaders),
+ headers: browserHeaders,
blockBots: true,
payload: { ...basePayload, clientBotScore: 5 },
});
@@ -379,11 +374,11 @@ describe("checkBotBlocking", () => {
it("uses weak client signals as supporting evidence when another layer matched", async () => {
const result = await checkBotBlocking({
- request: requestWithHeaders({
+ headers: {
...browserHeaders,
"user-agent":
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/120.0.0.0 Safari/537.36",
- }),
+ },
blockBots: true,
payload: {
...basePayload,
@@ -414,7 +409,7 @@ describe("checkBotBlocking", () => {
});
const result = await checkBotBlocking({
- request: requestWithHeaders(browserHeaders),
+ headers: browserHeaders,
blockBots: true,
payload: {
...basePayload,
@@ -434,7 +429,7 @@ describe("checkBotBlocking", () => {
});
const result = await checkBotBlocking({
- request: requestWithHeaders(browserHeaders),
+ headers: browserHeaders,
blockBots: true,
payload: {
...basePayload,
@@ -461,7 +456,7 @@ describe("checkBotBlocking", () => {
});
const result = await checkBotBlocking({
- request: requestWithHeaders(browserHeaders),
+ headers: browserHeaders,
blockBots: true,
payload: {
...basePayload,
@@ -490,7 +485,7 @@ describe("checkBotBlocking", () => {
});
const result = await checkBotBlocking({
- request: requestWithHeaders(browserHeaders),
+ headers: browserHeaders,
blockBots: true,
payload: {
...basePayload,
@@ -512,12 +507,12 @@ describe("checkBotBlocking", () => {
});
it("adds a rate anomaly layer after a request burst", async () => {
- const request = requestWithHeaders(browserHeaders);
+ const headers = browserHeaders;
let result: Awaited<ReturnType<typeof checkBotBlocking>> = null;
for (let i = 0; i < 31; i++) {
result = await checkBotBlocking({
- request,
+ headers,
blockBots: true,
payload: {
...basePayload,
@@ -541,7 +536,7 @@ describe("checkBotBlocking", () => {
"Mozilla/5.0 (Linux; Android 8.0; SM-G930F) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/40.0.0.0 Mobile Safari/537.36";
const result = await checkBotBlocking({
- request: requestWithHeaders({ ...browserHeaders, "user-agent": userAgent }),
+ headers: { ...browserHeaders, "user-agent": userAgent },
blockBots: true,
payload: {
...basePayload,
@@ -569,7 +564,7 @@ describe("checkBotBlocking", () => {
const userAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) HeadlessChrome/40.0.0.0 Safari/537.36";
const result = await checkBotBlocking({
- request: requestWithHeaders({ ...browserHeaders, "user-agent": userAgent }),
+ headers: { ...browserHeaders, "user-agent": userAgent },
blockBots: true,
payload: { ...basePayload, userAgent, clientBotScore: 0, clientBotSignalMask: 0 },
});
@@ -584,7 +579,7 @@ describe("checkBotBlocking", () => {
"Mozilla/5.0 (Linux; Android 8.0; SM-G930F) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/40.0.0.0 Mobile Safari/537.36";
const result = await checkBotBlocking({
- request: requestWithHeaders({ ...browserHeaders, "user-agent": userAgent }),
+ headers: { ...browserHeaders, "user-agent": userAgent },
blockBots: true,
isMobileSite: true,
payload: { ...basePayload, userAgent, clientBotScore: 0, clientBotSignalMask: 0 },
@@ -595,7 +590,7 @@ describe("checkBotBlocking", () => {
it("convicts a square screen, which no shipping display reports", async () => {
const result = await checkBotBlocking({
- request: requestWithHeaders(browserHeaders),
+ headers: browserHeaders,
blockBots: true,
payload: {
...basePayload,
@@ -618,11 +613,11 @@ describe("checkBotBlocking", () => {
// The square fleets claim Android and iOS as well as desktop, so unlike the
// default-viewport rules this one is not gated on a desktop user-agent.
const result = await checkBotBlocking({
- request: requestWithHeaders({
+ headers: {
...browserHeaders,
"user-agent":
"Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/150.0.0.0 Mobile Safari/537.36",
- }),
+ },
blockBots: true,
payload: { ...basePayload, clientBotScore: 0, clientBotSignalMask: 0, screenWidth: 2000, screenHeight: 2000 },
});
@@ -638,7 +633,7 @@ describe("checkBotBlocking", () => {
]) {
resetBotDetectionStatsForTests();
const result = await checkBotBlocking({
- request: requestWithHeaders(browserHeaders),
+ headers: browserHeaders,
blockBots: true,
payload: { ...basePayload, clientBotScore: 0, clientBotSignalMask: 0, screenWidth, screenHeight },
});
@@ -656,7 +651,7 @@ describe("checkBotBlocking", () => {
[1920, 1080],
]) {
const result = await checkBotBlocking({
- request: requestWithHeaders(browserHeaders),
+ headers: browserHeaders,
blockBots: true,
payload: { ...basePayload, clientBotScore: 0, clientBotSignalMask: 0, screenWidth, screenHeight },
});
diff --git a/server/src/services/tracker/botBlocking/index.ts b/server/src/services/tracker/botBlocking/index.ts
index 421167325..f31cc1c6d 100644
--- a/server/src/services/tracker/botBlocking/index.ts
+++ b/server/src/services/tracker/botBlocking/index.ts
@@ -1,5 +1,5 @@
-import { FastifyRequest } from "fastify";
-import { lookupAsn, type AsnInfo } from "../../../db/geolocation/asn.js";
+import type { IncomingHttpHeaders } from "http";
+import { lookupAsn, type AsnInfo, type AsnLookup } from "../../../db/geolocation/asn.js";
import { logger } from "../../../lib/logger/logger.js";
import type { AnomalyCounters } from "./anomalyScorer.js";
import { observeTrackingAnomaly } from "./anomalyScorer.js";
@@ -29,7 +29,12 @@ interface BotBlockingPayload {
}
interface BotBlockingInput {
- request: FastifyRequest;
+ /**
+ * The request's headers. Header heuristics are the only thing this needs from
+ * the HTTP request, so it takes headers rather than a `FastifyRequest` β€”
+ * ingestion can then be driven from a plain value.
+ */
+ headers: IncomingHttpHeaders;
blockBots: boolean;
trustedServerSideIngestion?: boolean;
/**
@@ -39,6 +44,8 @@ interface BotBlockingInput {
*/
isMobileSite?: boolean;
payload: BotBlockingPayload;
+ /** Request-scoped ASN resolver, shared with the rest of the ingestion path. */
+ lookupAsn?: AsnLookup;
}
interface AnomalyReason {
@@ -263,13 +270,14 @@ function getClientSignalResult(payload: BotBlockingPayload, userAgent: string) {
}
export async function checkBotBlocking({
- request,
+ headers,
blockBots,
trustedServerSideIngestion = false,
isMobileSite = false,
payload,
+ lookupAsn: asnLookup = lookupAsn,
}: BotBlockingInput): Promise<BotDetectionResult | null> {
- const userAgent = payload.userAgent || (request.headers["user-agent"] as string) || "";
+ const userAgent = payload.userAgent || (headers["user-agent"] as string) || "";
const clientSignalResult = getClientSignalResult(payload, userAgent);
recordBotBlockingRequest(clientSignalResult.scoreForStats, clientSignalResult.maskForStats);
@@ -315,7 +323,7 @@ export async function checkBotBlocking({
}
// Layer 2: Header heuristic bot detection
- const detection = detectBot(request, userAgent);
+ const detection = detectBot(headers, userAgent);
if (detection.isBot) {
addDetection("Bot detected using header heuristics", {
layer: "header_heuristics",
@@ -350,7 +358,7 @@ export async function checkBotBlocking({
let asnInfo: AsnInfo | null = null;
let supportingHostingAsnDetection: BotBlockingDetection | null = null;
if (ipForAsn) {
- asnInfo = lookupAsn(ipForAsn);
+ asnInfo = asnLookup(ipForAsn);
const botAsnMatch = classifyBotAsn(asnInfo?.asn);
if (asnInfo && botAsnMatch.isBotInfrastructure) {
const asnDetection: BotBlockingDetection = {
diff --git a/server/src/services/tracker/ingestEvent.test.ts b/server/src/services/tracker/ingestEvent.test.ts
new file mode 100644
index 000000000..27ba55767
--- /dev/null
+++ b/server/src/services/tracker/ingestEvent.test.ts
@@ -0,0 +1,239 @@
+import { beforeEach, describe, expect, it, vi } from "vitest";
+import type { SiteConfigData } from "../../lib/siteConfig.js";
+import type { TrackingRequest } from "./trackingRequest.js";
+
+const mocks = vi.hoisted(() => ({
+ addBotEvent: vi.fn(),
+ addPageview: vi.fn(),
+ checkBotBlocking: vi.fn(),
+ decideSiteExclusion: vi.fn(),
+ isSiteOverLimit: vi.fn(),
+ updateSession: vi.fn(),
+ generateUserId: vi.fn(),
+ generateUserIdFromClientId: vi.fn(),
+}));
+
+vi.mock("../usageService.js", () => ({
+ usageService: { isSiteOverLimit: mocks.isSiteOverLimit },
+}));
+
+vi.mock("./pageviewQueue.js", () => ({
+ pageviewQueue: { add: mocks.addPageview },
+}));
+
+vi.mock("../sessions/sessionsService.js", () => ({
+ sessionsService: { updateSession: mocks.updateSession },
+}));
+
+vi.mock("./botBlocking/index.js", () => ({
+ checkBotBlocking: mocks.checkBotBlocking,
+}));
+
+vi.mock("./botBlocking/botEventQueue.js", () => ({
+ botEventQueue: { add: mocks.addBotEvent },
+}));
+
+vi.mock("../sites/siteExclusionDecision.js", () => ({
+ decideSiteExclusion: mocks.decideSiteExclusion,
+}));
+
+vi.mock("../userId/userIdService.js", () => ({
+ userIdService: {
+ generateUserId: mocks.generateUserId,
+ generateUserIdFromClientId: mocks.generateUserIdFromClientId,
+ },
+}));
+
+import { ingestEvent } from "./ingestEvent.js";
+
+const site = {
+ id: "site_abc",
+ siteId: 42,
+ type: "web",
+ blockBots: true,
+ saltUserIds: false,
+ trackIp: false,
+ firstPartyProxy: false,
+ excludedIPs: [],
+ excludedCountries: [],
+ excludedPaths: [],
+ excludedHostnames: [],
+ excludedUserAgents: [],
+ excludedASNs: [],
+ excludedQueryParams: [],
+} as unknown as SiteConfigData;
+
+const lookupAsn = vi.fn(() => null);
+
+function trackingRequest(overrides: Partial<TrackingRequest> = {}): TrackingRequest {
+ return {
+ payload: {
+ type: "pageview",
+ site_id: "site_abc",
+ pathname: "/pricing",
+ hostname: "example.com",
+ } as TrackingRequest["payload"],
+ site,
+ ipAddress: "198.51.100.10",
+ userAgent: "Mozilla/5.0",
+ candidateIps: ["198.51.100.10"],
+ trustedServerSideIngestion: false,
+ headers: { "user-agent": "Mozilla/5.0" },
+ lookupAsn,
+ receivedAt: new Date("2026-08-14T10:38:37.000Z"),
+ ...overrides,
+ };
+}
+
+describe("ingestEvent", () => {
+ beforeEach(() => {
+ vi.clearAllMocks();
+ mocks.decideSiteExclusion.mockResolvedValue({ excluded: false });
+ mocks.isSiteOverLimit.mockReturnValue(false);
+ mocks.checkBotBlocking.mockResolvedValue(null);
+ mocks.updateSession.mockResolvedValue({ sessionId: "session-alice" });
+ mocks.generateUserId.mockResolvedValue("shared-fingerprint");
+ mocks.addPageview.mockResolvedValue(undefined);
+ });
+
+ it("queues an accepted event with its session and resolved identity", async () => {
+ const outcome = await ingestEvent(
+ trackingRequest({
+ payload: {
+ type: "pageview",
+ site_id: "site_abc",
+ user_id: "employee-alice",
+ } as TrackingRequest["payload"],
+ })
+ );
+
+ expect(outcome).toEqual({ status: "tracked", sessionId: "session-alice" });
+ expect(mocks.updateSession).toHaveBeenCalledWith({
+ userId: "shared-fingerprint",
+ identifiedUserId: "employee-alice",
+ siteId: 42,
+ });
+ expect(mocks.addPageview).toHaveBeenCalledWith(
+ expect.objectContaining({
+ site_id: 42,
+ sessionId: "session-alice",
+ userId: "shared-fingerprint",
+ identifiedUserId: "employee-alice",
+ ipAddress: "198.51.100.10",
+ timestamp: "2026-08-14T10:38:37.000Z",
+ })
+ );
+ });
+
+ it("passes the resolved request to the Site Exclusion Decision without re-deriving it", async () => {
+ await ingestEvent(
+ trackingRequest({
+ ipAddress: "203.0.113.7",
+ candidateIps: ["203.0.113.7", "198.51.100.10"],
+ userAgent: "ServerSDK/1.0",
+ payload: {
+ type: "pageview",
+ site_id: "site_abc",
+ pathname: "/admin/users",
+ querystring: "utm_source=newsletter",
+ hostname: "example.com",
+ } as TrackingRequest["payload"],
+ })
+ );
+
+ expect(mocks.decideSiteExclusion).toHaveBeenCalledWith(site, {
+ ipAddress: "203.0.113.7",
+ candidateIps: ["203.0.113.7", "198.51.100.10"],
+ pathname: "/admin/users",
+ querystring: "utm_source=newsletter",
+ hostname: "example.com",
+ userAgent: "ServerSDK/1.0",
+ lookupAsn,
+ });
+ });
+
+ // The ordering fix: exclusion used to run *after* bot detection, so a Site
+ // owner's own excluded office IP still drove every anomaly counter.
+ it("decides exclusion before bot detection, so excluded traffic leaves no trace", async () => {
+ mocks.decideSiteExclusion.mockResolvedValue({
+ excluded: true,
+ reason: "path",
+ label: "path",
+ value: "/admin/users",
+ });
+
+ const outcome = await ingestEvent(trackingRequest());
+
+ expect(outcome).toEqual({
+ status: "excluded",
+ exclusion: { excluded: true, reason: "path", label: "path", value: "/admin/users" },
+ });
+ expect(mocks.checkBotBlocking).not.toHaveBeenCalled();
+ expect(mocks.isSiteOverLimit).not.toHaveBeenCalled();
+ expect(mocks.generateUserId).not.toHaveBeenCalled();
+ expect(mocks.addPageview).not.toHaveBeenCalled();
+ expect(mocks.addBotEvent).not.toHaveBeenCalled();
+ });
+
+ it("stops at the monthly limit before charging bot detection a Redis round-trip", async () => {
+ mocks.isSiteOverLimit.mockReturnValue(true);
+
+ const outcome = await ingestEvent(trackingRequest());
+
+ expect(outcome).toEqual({ status: "over_limit" });
+ expect(mocks.checkBotBlocking).not.toHaveBeenCalled();
+ expect(mocks.addPageview).not.toHaveBeenCalled();
+ });
+
+ it("routes detected bots to the bot queue instead of a session", async () => {
+ mocks.checkBotBlocking.mockResolvedValue({
+ eventProperties: { bot_layer: "ua_pattern" },
+ });
+
+ const outcome = await ingestEvent(trackingRequest());
+
+ expect(outcome).toEqual({ status: "bot" });
+ expect(mocks.addBotEvent).toHaveBeenCalledWith(
+ expect.objectContaining({
+ bot_layer: "ua_pattern",
+ sessionId: "bot:shared-fingerprint",
+ })
+ );
+ expect(mocks.updateSession).not.toHaveBeenCalled();
+ expect(mocks.addPageview).not.toHaveBeenCalled();
+ });
+
+ it("hands bot detection the request's own headers and ASN resolver", async () => {
+ await ingestEvent(
+ trackingRequest({
+ headers: { "user-agent": "Mozilla/5.0", "accept-language": "en-US" },
+ trustedServerSideIngestion: true,
+ })
+ );
+
+ expect(mocks.checkBotBlocking).toHaveBeenCalledWith(
+ expect.objectContaining({
+ headers: { "user-agent": "Mozilla/5.0", "accept-language": "en-US" },
+ blockBots: true,
+ trustedServerSideIngestion: true,
+ isMobileSite: false,
+ lookupAsn,
+ })
+ );
+ });
+
+ it("reuses the request's Site Configuration for identity instead of re-reading it", async () => {
+ await ingestEvent(
+ trackingRequest({
+ site: { ...site, saltUserIds: true } as SiteConfigData,
+ })
+ );
+
+ expect(mocks.generateUserId).toHaveBeenCalledWith("198.51.100.10", "Mozilla/5.0", 42, {
+ saltUserIds: true,
+ lookupAsn,
+ // The event's own instant decides its salt day, not the clock at hash time.
+ receivedAt: new Date("2026-08-14T10:38:37.000Z"),
+ });
+ });
+});
diff --git a/server/src/services/tracker/ingestEvent.ts b/server/src/services/tracker/ingestEvent.ts
new file mode 100644
index 000000000..6b7e145af
--- /dev/null
+++ b/server/src/services/tracker/ingestEvent.ts
@@ -0,0 +1,111 @@
+import { createServiceLogger } from "../../lib/logger/logger.js";
+import { sessionsService } from "../sessions/sessionsService.js";
+import { decideSiteExclusion, type SiteExclusionDecision } from "../sites/siteExclusionDecision.js";
+import { usageService } from "../usageService.js";
+import { botEventQueue } from "./botBlocking/botEventQueue.js";
+import { checkBotBlocking } from "./botBlocking/index.js";
+import { pageviewQueue } from "./pageviewQueue.js";
+import type { TrackingRequest } from "./trackingRequest.js";
+import { createBasePayload } from "./utils.js";
+
+const logger = createServiceLogger("ingest-event");
+
+type MatchedExclusion = Extract<SiteExclusionDecision, { excluded: true }>;
+
+/**
+ * What ingestion did with an event. Every outcome is a success from the
+ * caller's point of view β€” the HTTP layer turns these into status codes and
+ * bodies, and holds no ingestion logic of its own.
+ */
+export type IngestOutcome =
+ | { status: "excluded"; exclusion: MatchedExclusion }
+ | { status: "over_limit" }
+ | { status: "bot" }
+ | { status: "tracked"; sessionId: string };
+
+/**
+ * Take one resolved Tracking Request through the ingestion pipeline.
+ *
+ * The order of the stages is the interesting part, and it is the reason this is
+ * a module rather than a handler body: each stage can veto, and until they sat
+ * together the order was an accident of how the handler had been edited.
+ *
+ * 1. Site Exclusion Decision β€” first, so traffic the Site owner has disowned
+ * (their own office IP, a staging hostname) leaves no trace at all. It
+ * used to run third, after bot detection, which meant an excluded visitor
+ * still moved every anomaly counter for the Site.
+ * 2. Over-limit β€” before any Redis or ClickHouse work, since the event is
+ * going to be dropped regardless.
+ * 3. Bot detection β€” needs a real event; charges a Redis round-trip.
+ * 4. Identity and session β€” the only stage that hashes a fingerprint.
+ * 5. Enqueue.
+ */
+export async function ingestEvent(trackingRequest: TrackingRequest): Promise<IngestOutcome> {
+ const { payload, site } = trackingRequest;
+
+ const exclusionDecision = await decideSiteExclusion(site, {
+ ipAddress: trackingRequest.ipAddress,
+ candidateIps: trackingRequest.candidateIps,
+ pathname: payload.pathname,
+ querystring: payload.querystring,
+ hostname: payload.hostname,
+ userAgent: trackingRequest.userAgent,
+ lookupAsn: trackingRequest.lookupAsn,
+ });
+
+ if (exclusionDecision.excluded) {
+ logger.info({ siteId: payload.site_id, exclusionReason: exclusionDecision.reason }, "Site exclusion matched");
+ return { status: "excluded", exclusion: exclusionDecision };
+ }
+
+ if (usageService.isSiteOverLimit(site.siteId)) {
+ logger.info({ siteId: payload.site_id }, "Skipping event - site over monthly limit");
+ return { status: "over_limit" };
+ }
+
+ const botDetectionResult = await checkBotBlocking({
+ headers: trackingRequest.headers,
+ blockBots: site.blockBots,
+ trustedServerSideIngestion: trackingRequest.trustedServerSideIngestion,
+ isMobileSite: site.type === "mobile",
+ lookupAsn: trackingRequest.lookupAsn,
+ payload: {
+ siteId: payload.site_id,
+ userAgent: trackingRequest.userAgent,
+ clientBotScore: payload._bs,
+ clientBotSignalMask: payload._bsm,
+ screenWidth: payload.screenWidth,
+ screenHeight: payload.screenHeight,
+ language: payload.language,
+ hostname: payload.hostname,
+ pathname: payload.pathname,
+ eventType: payload.type,
+ ipAddress: trackingRequest.ipAddress,
+ },
+ });
+
+ const eventPayload = await createBasePayload(trackingRequest);
+
+ if (botDetectionResult) {
+ await botEventQueue.add({
+ ...eventPayload,
+ ...botDetectionResult.eventProperties,
+ sessionId: `bot:${eventPayload.userId}`,
+ });
+
+ return { status: "bot" };
+ }
+
+ const { sessionId } = await sessionsService.updateSession({
+ userId: eventPayload.userId,
+ identifiedUserId: eventPayload.identifiedUserId,
+ siteId: site.siteId,
+ });
+
+ await pageviewQueue.add({
+ ...eventPayload,
+ sessionId,
+ });
+
+ return { status: "tracked", sessionId };
+}
diff --git a/server/src/services/tracker/requestIdentity.test.ts b/server/src/services/tracker/requestIdentity.test.ts
deleted file mode 100644
index a034b1b40..000000000
--- a/server/src/services/tracker/requestIdentity.test.ts
+++ /dev/null
@@ -1,54 +0,0 @@
-import { FastifyRequest } from "fastify";
-import { describe, expect, it } from "vitest";
-import { resolveTrackingIdentity } from "./requestIdentity.js";
-
-function requestWithHeaders(headers: Record<string, string | string[]>, ip = "198.51.100.10"): FastifyRequest {
- return { headers, ip } as unknown as FastifyRequest;
-}
-
-describe("resolveTrackingIdentity", () => {
- it("ignores public payload IP and user-agent overrides", () => {
- const request = requestWithHeaders({
- "user-agent": "Mozilla/5.0 Chrome/120 Safari/537.36",
- "x-forwarded-for": "198.51.100.20, 10.0.0.1",
- });
-
- expect(
- resolveTrackingIdentity(
- request,
- {
- ip_address: "203.0.113.10",
- user_agent: "SpoofedBot/1.0",
- },
- false
- )
- ).toEqual({
- ipAddress: "198.51.100.20",
- userAgent: "Mozilla/5.0 Chrome/120 Safari/537.36",
- // Exclusion candidates still include everything the request presented.
- candidateIps: ["198.51.100.20", "10.0.0.1", "198.51.100.10"],
- });
- });
-
- it("allows payload IP and user-agent overrides for trusted server-side ingestion", () => {
- const request = requestWithHeaders({
- "user-agent": "ServerSDK/1.0",
- "x-forwarded-for": "198.51.100.20",
- });
-
- expect(
- resolveTrackingIdentity(
- request,
- {
- ip_address: "203.0.113.10",
- user_agent: "Mozilla/5.0 Chrome/120 Safari/537.36",
- },
- true
- )
- ).toEqual({
- ipAddress: "203.0.113.10",
- userAgent: "Mozilla/5.0 Chrome/120 Safari/537.36",
- candidateIps: ["203.0.113.10", "198.51.100.20", "198.51.100.10"],
- });
- });
-});
diff --git a/server/src/services/tracker/requestIdentity.ts b/server/src/services/tracker/requestIdentity.ts
deleted file mode 100644
index fd61b1fdd..000000000
--- a/server/src/services/tracker/requestIdentity.ts
+++ /dev/null
@@ -1,40 +0,0 @@
-import { FastifyRequest } from "fastify";
-import { collectCandidateClientIps, resolveClientIp } from "./resolveClientIp.js";
-
-interface TrackingIdentityPayload {
- ip_address?: string;
- user_agent?: string;
-}
-
-interface TrackingIdentity {
- ipAddress: string;
- userAgent: string;
- /** All plausible client IPs for this request β€” exclusion matching only. */
- candidateIps: string[];
-}
-
-export function getRequestUserAgent(request: FastifyRequest): string {
- const userAgentHeader = request.headers["user-agent"];
- if (Array.isArray(userAgentHeader)) {
- return userAgentHeader[0] || "";
- }
- return userAgentHeader || "";
-}
-
-export function resolveTrackingIdentity(
- request: FastifyRequest,
- payload: TrackingIdentityPayload,
- trustedServerSideIngestion: boolean,
- firstPartyProxy = false
-): TrackingIdentity {
- const requestIpAddress = resolveClientIp(request, { firstPartyProxy });
- const requestUserAgent = getRequestUserAgent(request);
-
- const ipAddress = trustedServerSideIngestion ? payload.ip_address || requestIpAddress : requestIpAddress;
-
- return {
- ipAddress,
- userAgent: trustedServerSideIngestion ? payload.user_agent || requestUserAgent : requestUserAgent,
- candidateIps: collectCandidateClientIps(request, [ipAddress, requestIpAddress]),
- };
-}
diff --git a/server/src/services/tracker/resolveClientIp.ts b/server/src/services/tracker/resolveClientIp.ts
index bcf3c8ab9..be4ce58a0 100644
--- a/server/src/services/tracker/resolveClientIp.ts
+++ b/server/src/services/tracker/resolveClientIp.ts
@@ -1,5 +1,5 @@
import { FastifyRequest } from "fastify";
-import { lookupAsn } from "../../db/geolocation/asn.js";
+import { lookupAsn, type AsnLookup } from "../../db/geolocation/asn.js";
import { getIpAddress } from "../../utils.js";
import { isDatacenterAsn } from "./botBlocking/datacenterAsns.js";
@@ -79,10 +79,9 @@ const CF_WORKER_SUBREQUEST_IP = "2a06:98c0:3600::103";
* to `true` on a null lookup instead (favouring proxied correctness over
* direct-path anti-spoofing).
*/
-function isProxiedEdge(ip: string): boolean {
+function isProxiedEdge(ip: string, asnLookup: AsnLookup): boolean {
if (ip.toLowerCase() === CF_WORKER_SUBREQUEST_IP) return true;
- const asn = lookupAsn(ip);
- return isDatacenterAsn(asn?.asn);
+ return isDatacenterAsn(asnLookup(ip)?.asn);
}
export interface ResolveClientIpOptions {
@@ -92,12 +91,18 @@ export interface ResolveClientIpOptions {
* unconditionally, exactly like the pre-ASN `getIpAddress` precedence.
*/
firstPartyProxy?: boolean;
+ /**
+ * ASN resolver to infer the topology with. Tracking ingestion passes the
+ * request-scoped one so the edge IP is looked up once for the whole request.
+ */
+ lookupAsn?: AsnLookup;
/** Injectable for tests so branching can be exercised without the MaxMind DB. */
proxiedEdge?: (ip: string) => boolean;
}
export function resolveClientIp(request: FastifyRequest, options: ResolveClientIpOptions = {}): string {
- const proxiedEdge = options.proxiedEdge ?? isProxiedEdge;
+ const asnLookup = options.lookupAsn ?? lookupAsn;
+ const proxiedEdge = options.proxiedEdge ?? (ip => isProxiedEdge(ip, asnLookup));
const realIp = realIpHeader(request);
const xffFirst = firstForwardedFor(request);
const cfIp = cfConnectingIp(request);
diff --git a/server/src/services/tracker/trackEvent.test.ts b/server/src/services/tracker/trackEvent.test.ts
index 6370ecf0b..96b1098d0 100644
--- a/server/src/services/tracker/trackEvent.test.ts
+++ b/server/src/services/tracker/trackEvent.test.ts
@@ -2,75 +2,25 @@ import type { FastifyReply, FastifyRequest } from "fastify";
import { beforeEach, describe, expect, it, vi } from "vitest";
const mocks = vi.hoisted(() => ({
- addBotEvent: vi.fn(),
- addPageview: vi.fn(),
- checkBotBlocking: vi.fn(),
- createBasePayload: vi.fn(),
- decideSiteExclusion: vi.fn(),
- getConfig: vi.fn(),
- isSiteOverLimit: vi.fn(),
- updateSession: vi.fn(),
+ resolveTrackingRequest: vi.fn(),
+ ingestEvent: vi.fn(),
}));
-vi.mock("../../lib/siteConfig.js", () => ({
- siteConfig: {
- getConfig: mocks.getConfig,
- },
+vi.mock("./trackingRequest.js", () => ({
+ resolveTrackingRequest: mocks.resolveTrackingRequest,
}));
-vi.mock("../usageService.js", () => ({
- usageService: {
- isSiteOverLimit: mocks.isSiteOverLimit,
- },
-}));
-
-vi.mock("./pageviewQueue.js", () => ({
- pageviewQueue: {
- add: mocks.addPageview,
- },
-}));
-
-vi.mock("../sessions/sessionsService.js", () => ({
- sessionsService: {
- updateSession: mocks.updateSession,
- },
-}));
-
-vi.mock("./botBlocking/index.js", () => ({
- checkBotBlocking: mocks.checkBotBlocking,
-}));
-
-vi.mock("./botBlocking/botEventQueue.js", () => ({
- botEventQueue: {
- add: mocks.addBotEvent,
- },
-}));
-
-vi.mock("../../lib/auth-utils.js", () => ({
- checkApiKey: vi.fn(),
-}));
-
-vi.mock("./utils.js", () => ({
- createBasePayload: mocks.createBasePayload,
-}));
-
-vi.mock("../sites/siteExclusionDecision.js", () => ({
- decideSiteExclusion: mocks.decideSiteExclusion,
+vi.mock("./ingestEvent.js", () => ({
+ ingestEvent: mocks.ingestEvent,
}));
import { trackEvent } from "./trackEvent.js";
-const siteConfiguration = {
- id: "site_abc",
- siteId: 42,
- type: "web",
- blockBots: false,
- excludedIPs: [],
- excludedCountries: [],
- excludedPaths: [],
- excludedHostnames: [],
- excludedUserAgents: [],
-};
+const resolvedRequest = { payload: { site_id: "site_abc" }, site: { siteId: 42 } };
+
+function requestWithBody(body: unknown): FastifyRequest {
+ return { body, headers: {}, ip: "198.51.100.10" } as unknown as FastifyRequest;
+}
function replyStub(): FastifyReply {
const reply = {
@@ -84,80 +34,87 @@ function replyStub(): FastifyReply {
return reply as unknown as FastifyReply;
}
-describe("trackEvent session identity", () => {
+describe("trackEvent", () => {
beforeEach(() => {
vi.clearAllMocks();
- mocks.getConfig.mockResolvedValue(siteConfiguration);
- mocks.isSiteOverLimit.mockReturnValue(false);
- mocks.checkBotBlocking.mockResolvedValue(null);
- mocks.decideSiteExclusion.mockResolvedValue({ excluded: false });
- mocks.createBasePayload.mockResolvedValue({
- site_id: 42,
- userId: "shared-fingerprint",
- identifiedUserId: "employee-alice",
- });
- mocks.updateSession.mockResolvedValue({ sessionId: "session-alice" });
- mocks.addPageview.mockResolvedValue(undefined);
+ mocks.resolveTrackingRequest.mockResolvedValue(resolvedRequest);
+ mocks.ingestEvent.mockResolvedValue({ status: "tracked", sessionId: "session-alice" });
});
- it("passes the identified user into session assignment", async () => {
- const request = {
- body: {
- type: "pageview",
- site_id: "site_abc",
- },
- headers: {},
- ip: "198.51.100.10",
- } as unknown as FastifyRequest;
-
- await trackEvent(request, replyStub());
-
- expect(mocks.updateSession).toHaveBeenCalledWith({
- userId: "shared-fingerprint",
- identifiedUserId: "employee-alice",
- siteId: 42,
- });
- expect(mocks.addPageview).toHaveBeenCalledWith(
- expect.objectContaining({
- sessionId: "session-alice",
- userId: "shared-fingerprint",
- identifiedUserId: "employee-alice",
- })
- );
+ it("rejects a payload that fails validation without resolving anything", async () => {
+ const reply = replyStub();
+
+ await trackEvent(requestWithBody({ type: "pageview" }), reply);
+
+ expect(reply.status).toHaveBeenCalledWith(400);
+ expect(mocks.resolveTrackingRequest).not.toHaveBeenCalled();
+ expect(mocks.ingestEvent).not.toHaveBeenCalled();
+ });
+
+ it("answers 404 when the Site is unknown", async () => {
+ mocks.resolveTrackingRequest.mockResolvedValue(null);
+ const reply = replyStub();
+
+ await trackEvent(requestWithBody({ type: "pageview", site_id: "nope" }), reply);
+
+ expect(reply.status).toHaveBeenCalledWith(404);
+ expect(reply.send).toHaveBeenCalledWith({ success: false, error: "Site not found" });
+ expect(mocks.ingestEvent).not.toHaveBeenCalled();
+ });
+
+ it("hands the resolved Tracking Request to ingestion and reports success", async () => {
+ const reply = replyStub();
+
+ await trackEvent(requestWithBody({ type: "pageview", site_id: "site_abc" }), reply);
+
+ expect(mocks.ingestEvent).toHaveBeenCalledWith(resolvedRequest);
+ expect(reply.status).toHaveBeenCalledWith(200);
+ expect(reply.send).toHaveBeenCalledWith({ success: true });
});
it("maps a Site Exclusion Decision to the event skip response", async () => {
- mocks.decideSiteExclusion.mockResolvedValue({
- excluded: true,
- reason: "path",
- label: "path",
- value: "/admin/users",
+ mocks.ingestEvent.mockResolvedValue({
+ status: "excluded",
+ exclusion: { excluded: true, reason: "path", label: "path", value: "/admin/users" },
});
- const request = {
- body: {
- type: "pageview",
- site_id: "site_abc",
- pathname: "/admin/users",
- hostname: "example.com",
- },
- headers: { "user-agent": "Mozilla/5.0" },
- ip: "198.51.100.10",
- } as unknown as FastifyRequest;
const reply = replyStub();
- await trackEvent(request, reply);
+ await trackEvent(requestWithBody({ type: "pageview", site_id: "site_abc" }), reply);
- expect(mocks.decideSiteExclusion).toHaveBeenCalledWith(siteConfiguration, {
- ipAddress: "198.51.100.10",
- candidateIps: ["198.51.100.10"],
- pathname: "/admin/users",
- hostname: "example.com",
- userAgent: "Mozilla/5.0",
- });
+ expect(reply.status).toHaveBeenCalledWith(200);
expect(reply.send).toHaveBeenCalledWith({
success: true,
message: "Event not tracked - path excluded",
});
- expect(mocks.createBasePayload).not.toHaveBeenCalled();
+ });
+
+ it("reports the monthly limit without failing the request", async () => {
+ mocks.ingestEvent.mockResolvedValue({ status: "over_limit" });
+ const reply = replyStub();
+
+ await trackEvent(requestWithBody({ type: "pageview", site_id: "site_abc" }), reply);
+
+ expect(reply.status).toHaveBeenCalledWith(200);
+ expect(reply.send).toHaveBeenCalledWith("Site over monthly limit, event not tracked");
+ });
+
+ it("answers a detected bot with the same success shape as a tracked event", async () => {
+ mocks.ingestEvent.mockResolvedValue({ status: "bot" });
+ const reply = replyStub();
+
+ await trackEvent(requestWithBody({ type: "pageview", site_id: "site_abc" }), reply);
+
+ expect(reply.status).toHaveBeenCalledWith(200);
+ expect(reply.send).toHaveBeenCalledWith({ success: true });
+ });
+
+ it("turns an ingestion failure into a 500", async () => {
+ mocks.ingestEvent.mockRejectedValue(new Error("ClickHouse unavailable"));
+ const reply = replyStub();
+
+ await trackEvent(requestWithBody({ type: "pageview", site_id: "site_abc" }), reply);
+
+ expect(reply.status).toHaveBeenCalledWith(500);
+ expect(reply.send).toHaveBeenCalledWith({ success: false, error: "Failed to track event" });
});
});
diff --git a/server/src/services/tracker/trackEvent.ts b/server/src/services/tracker/trackEvent.ts
index 7f54c6b39..b22c9c629 100644
--- a/server/src/services/tracker/trackEvent.ts
+++ b/server/src/services/tracker/trackEvent.ts
@@ -1,270 +1,21 @@
import { FastifyReply, FastifyRequest } from "fastify";
-import { z, ZodError } from "zod";
+import { ZodError } from "zod";
import { createServiceLogger } from "../../lib/logger/logger.js";
-import { siteConfig } from "../../lib/siteConfig.js";
-import { sessionsService } from "../sessions/sessionsService.js";
-import { usageService } from "../usageService.js";
-import { pageviewQueue } from "./pageviewQueue.js";
-import { createBasePayload } from "./utils.js";
-import { checkApiKey } from "../../lib/auth-utils.js";
-import { hasScope } from "../../lib/scopes.js";
-import { botEventQueue } from "./botBlocking/botEventQueue.js";
-import { checkBotBlocking } from "./botBlocking/index.js";
-import { resolveTrackingIdentity } from "./requestIdentity.js";
-import { decideSiteExclusion } from "../sites/siteExclusionDecision.js";
-
-// Shared fields for all event types
-const baseEventFields = {
- site_id: z.string().min(1),
- hostname: z.string().max(253).optional(),
- pathname: z.string().max(2048).optional(),
- querystring: z.string().max(2048).optional(),
- screenWidth: z.number().int().nonnegative().optional(),
- screenHeight: z.number().int().nonnegative().optional(),
- language: z.string().max(35).optional(),
- page_title: z.string().max(512).optional(),
- referrer: z.string().max(2048).optional(),
- anonymous_id: z.string().min(1).max(255).optional(),
- user_id: z.string().max(255).optional(),
- tag: z.string().max(256).optional(),
- feature_flags: z.record(z.string().max(100), z.string().max(2048)).optional(),
- ip_address: z.string().ip().optional(),
- user_agent: z.string().max(512).optional(),
- _bs: z.number().int().min(0).max(10).optional(),
- _bsm: z.number().int().min(0).max(2047).optional(),
-};
-
-// Default event_name and properties used by pageview and performance
-const defaultEventProps = {
- event_name: z.string().max(256).optional(),
- properties: z.string().max(2048).optional(),
-};
-
-// Reusable JSON validation refine
-const jsonStringRefine = (message: string) =>
- z
- .string()
- .max(2048)
- .refine(
- val => {
- try {
- JSON.parse(val);
- return true;
- } catch {
- return false;
- }
- },
- { message }
- )
- .optional();
-
-// Define Zod schema for validation
-export const trackingPayloadSchema = z.discriminatedUnion("type", [
- z
- .object({
- type: z.literal("pageview"),
- ...baseEventFields,
- ...defaultEventProps,
- })
- .strict(),
- z
- .object({
- type: z.literal("custom_event"),
- ...baseEventFields,
- event_name: z.string().min(1).max(256),
- properties: jsonStringRefine("Properties must be a valid JSON string"),
- })
- .strict(),
- z
- .object({
- type: z.literal("performance"),
- ...baseEventFields,
- ...defaultEventProps,
- // Performance metrics (can be null if not collected)
- lcp: z.number().min(0).nullable().optional(),
- cls: z.number().min(0).nullable().optional(),
- inp: z.number().min(0).nullable().optional(),
- fcp: z.number().min(0).nullable().optional(),
- ttfb: z.number().min(0).nullable().optional(),
- })
- .strict(),
- z
- .object({
- type: z.literal("outbound"),
- ...baseEventFields,
- event_name: z.string().max(256).optional(),
- properties: z
- .string()
- .max(2048)
- .refine(
- val => {
- try {
- const parsed = JSON.parse(val);
- if (typeof parsed.url !== "string" || parsed.url.length === 0) return false;
- if (parsed.text && typeof parsed.text !== "string") return false;
- if (parsed.target && typeof parsed.target !== "string") return false;
- try {
- new URL(parsed.url);
- } catch {
- return false;
- }
- return true;
- } catch {
- return false;
- }
- },
- {
- message: "Properties must be valid JSON with outbound link fields (url required, text and target optional)",
- }
- ),
- })
- .strict(),
- z
- .object({
- type: z.literal("error"),
- ...baseEventFields,
- event_name: z.string().min(1).max(256), // Error type (TypeError, ReferenceError, etc.)
- properties: z
- .string()
- .max(4096) // Larger limit for error details
- .refine(
- val => {
- try {
- const parsed = JSON.parse(val);
- if (typeof parsed.message !== "string") return false;
- if (parsed.stack && typeof parsed.stack !== "string") return false;
- if (parsed.fileName && typeof parsed.fileName !== "string") return false;
- if (parsed.lineNumber && typeof parsed.lineNumber !== "number") return false;
- if (parsed.columnNumber && typeof parsed.columnNumber !== "number") return false;
- // Apply truncation limits
- if (parsed.message && parsed.message.length > 500) {
- parsed.message = parsed.message.substring(0, 500);
- }
- if (parsed.stack && parsed.stack.length > 2000) {
- parsed.stack = parsed.stack.substring(0, 2000);
- }
- return true;
- } catch {
- return false;
- }
- },
- {
- message:
- "Properties must be valid JSON with error fields (message, stack, fileName, lineNumber, columnNumber)",
- }
- ),
- })
- .strict(),
- z
- .object({
- type: z.literal("button_click"),
- ...baseEventFields,
- event_name: z.string().max(256).optional(),
- properties: jsonStringRefine("Properties must be valid JSON"),
- })
- .strict(),
- z
- .object({
- type: z.literal("copy"),
- ...baseEventFields,
- event_name: z.string().max(256).optional(),
- properties: z
- .string()
- .max(2048)
- .refine(
- val => {
- try {
- const parsed = JSON.parse(val);
- if (typeof parsed.sourceElement !== "string") return false;
- if (parsed.text !== undefined && typeof parsed.text !== "string") return false;
- if (parsed.textLength !== undefined && (typeof parsed.textLength !== "number" || parsed.textLength < 0))
- return false;
- return true;
- } catch {
- return false;
- }
- },
- {
- message:
- "Properties must be valid JSON with copy fields (sourceElement required, text and textLength optional)",
- }
- ),
- })
- .strict(),
- z
- .object({
- type: z.literal("form_submit"),
- ...baseEventFields,
- event_name: z.string().max(256).optional(),
- properties: z
- .string()
- .max(2048)
- .refine(
- val => {
- try {
- const parsed = JSON.parse(val);
- if (typeof parsed.formId !== "string") return false;
- if (typeof parsed.formName !== "string") return false;
- if (typeof parsed.formAction !== "string") return false;
- if (typeof parsed.method !== "string") return false;
- if (typeof parsed.fieldCount !== "number" || parsed.fieldCount < 0) return false;
- return true;
- } catch {
- return false;
- }
- },
- {
- message:
- "Properties must be valid JSON with form_submit fields (formId, formName, formAction, method, fieldCount required)",
- }
- ),
- })
- .strict(),
- z
- .object({
- type: z.literal("input_change"),
- ...baseEventFields,
- event_name: z.string().max(256).optional(),
- properties: z
- .string()
- .max(2048)
- .refine(
- val => {
- try {
- const parsed = JSON.parse(val);
- if (typeof parsed.element !== "string") return false;
- if (typeof parsed.inputName !== "string") return false;
- return true;
- } catch {
- return false;
- }
- },
- {
- message: "Properties must be valid JSON with input_change fields (element, inputName required)",
- }
- ),
- })
- .strict(),
-]);
+import { ingestEvent } from "./ingestEvent.js";
+import { trackingPayloadSchema } from "./trackingPayload.js";
+import { resolveTrackingRequest } from "./trackingRequest.js";
const logger = createServiceLogger("track-event");
-async function isTrustedServerSideIngestion(request: FastifyRequest, siteId: number): Promise<boolean> {
- const authHeader = request.headers["authorization"];
- if (typeof authHeader !== "string" || !authHeader.startsWith("Bearer ")) {
- return false;
- }
-
- // An under-scoped bearer degrades to untrusted client-side traffic (no 4xx)
- // β€” this is a public ingestion endpoint, not an authenticated API.
- const apiKeyResult = await checkApiKey(request, { siteId });
- return apiKeyResult.valid && hasScope(apiKeyResult.statements, { resource: "ingest", action: "write" });
-}
-
-// Unified handler for all events (pageviews and custom events)
+/**
+ * HTTP entry point for all event types.
+ *
+ * Parse, resolve, ingest, answer β€” nothing else lives here. The ingestion rules
+ * are in `ingestEvent`, and everything the request means is resolved once by
+ * `resolveTrackingRequest`.
+ */
export async function trackEvent(request: FastifyRequest, reply: FastifyReply) {
try {
- // Validate request body using Zod
const validationResult = trackingPayloadSchema.safeParse(request.body);
if (!validationResult.success) {
@@ -275,108 +26,29 @@ export async function trackEvent(request: FastifyRequest, reply: FastifyReply) {
});
}
- // Use validated data
- const validatedPayload = validationResult.data;
+ const trackingRequest = await resolveTrackingRequest(request, validationResult.data);
- // Get the site configuration to get the numeric siteId
- const siteConfiguration = await siteConfig.getConfig(validatedPayload.site_id);
- if (!siteConfiguration) {
- logger.warn({ siteId: validatedPayload.site_id }, "Site not found");
+ if (!trackingRequest) {
+ logger.warn({ siteId: validationResult.data.site_id }, "Site not found");
return reply.status(404).send({
success: false,
error: "Site not found",
});
}
- const trustedServerSideIngestion = await isTrustedServerSideIngestion(request, siteConfiguration.siteId);
- const trackingIdentity = resolveTrackingIdentity(
- request,
- validatedPayload,
- trustedServerSideIngestion,
- siteConfiguration.firstPartyProxy
- );
- const requestIP = trackingIdentity.ipAddress;
-
- const botDetectionResult = await checkBotBlocking({
- request,
- blockBots: siteConfiguration.blockBots,
- trustedServerSideIngestion,
- isMobileSite: siteConfiguration.type === "mobile",
- payload: {
- siteId: validatedPayload.site_id,
- userAgent: trackingIdentity.userAgent,
- clientBotScore: validatedPayload._bs,
- clientBotSignalMask: validatedPayload._bsm,
- screenWidth: validatedPayload.screenWidth,
- screenHeight: validatedPayload.screenHeight,
- language: validatedPayload.language,
- hostname: validatedPayload.hostname,
- pathname: validatedPayload.pathname,
- eventType: validatedPayload.type,
- ipAddress: requestIP,
- },
- });
+ const outcome = await ingestEvent(trackingRequest);
- // Check if the site has exceeded its monthly limit (using numeric siteId)
- if (usageService.isSiteOverLimit(siteConfiguration.siteId)) {
- logger.info({ siteId: validatedPayload.site_id }, "Skipping event - site over monthly limit");
+ if (outcome.status === "over_limit") {
return reply.status(200).send("Site over monthly limit, event not tracked");
}
- const exclusionDecision = await decideSiteExclusion(siteConfiguration, {
- ipAddress: requestIP,
- candidateIps: trackingIdentity.candidateIps,
- pathname: validatedPayload.pathname,
- querystring: validatedPayload.querystring,
- hostname: validatedPayload.hostname,
- userAgent: trackingIdentity.userAgent,
- });
-
- if (exclusionDecision.excluded) {
- logger.info(
- { siteId: validatedPayload.site_id, exclusionReason: exclusionDecision.reason },
- "Site exclusion matched"
- );
- return reply.status(200).send({
- success: true,
- message: `Event not tracked - ${exclusionDecision.label} excluded`,
- });
- }
-
- // Create base payload for the event using validated data
- const payload = await createBasePayload(
- request, // Pass request for IP/UA
- validatedPayload.type,
- validatedPayload, // Pass original validated payload
- siteConfiguration,
- trustedServerSideIngestion
- );
-
- if (botDetectionResult) {
- await botEventQueue.add({
- ...payload,
- ...botDetectionResult.eventProperties,
- sessionId: `bot:${payload.userId}`,
- });
-
+ if (outcome.status === "excluded") {
return reply.status(200).send({
success: true,
+ message: `Event not tracked - ${outcome.exclusion.label} excluded`,
});
}
- // Update session (use numeric siteId)
- const { sessionId } = await sessionsService.updateSession({
- userId: payload.userId,
- identifiedUserId: payload.identifiedUserId,
- siteId: siteConfiguration.siteId,
- });
-
- // Add to queue for processing (payload already has numeric siteId)
- await pageviewQueue.add({
- ...payload,
- sessionId,
- });
-
return reply.status(200).send({
success: true,
});
diff --git a/server/src/services/tracker/trackingPayload.ts b/server/src/services/tracker/trackingPayload.ts
new file mode 100644
index 000000000..6286963dd
--- /dev/null
+++ b/server/src/services/tracker/trackingPayload.ts
@@ -0,0 +1,238 @@
+import { z } from "zod";
+
+// Shared fields for all event types
+const baseEventFields = {
+ site_id: z.string().min(1),
+ hostname: z.string().max(253).optional(),
+ pathname: z.string().max(2048).optional(),
+ querystring: z.string().max(2048).optional(),
+ screenWidth: z.number().int().nonnegative().optional(),
+ screenHeight: z.number().int().nonnegative().optional(),
+ language: z.string().max(35).optional(),
+ page_title: z.string().max(512).optional(),
+ referrer: z.string().max(2048).optional(),
+ anonymous_id: z.string().min(1).max(255).optional(),
+ user_id: z.string().max(255).optional(),
+ tag: z.string().max(256).optional(),
+ feature_flags: z.record(z.string().max(100), z.string().max(2048)).optional(),
+ ip_address: z.string().ip().optional(),
+ user_agent: z.string().max(512).optional(),
+ _bs: z.number().int().min(0).max(10).optional(),
+ _bsm: z.number().int().min(0).max(2047).optional(),
+};
+
+// Default event_name and properties used by pageview and performance
+const defaultEventProps = {
+ event_name: z.string().max(256).optional(),
+ properties: z.string().max(2048).optional(),
+};
+
+// Reusable JSON validation refine
+const jsonStringRefine = (message: string) =>
+ z
+ .string()
+ .max(2048)
+ .refine(
+ val => {
+ try {
+ JSON.parse(val);
+ return true;
+ } catch {
+ return false;
+ }
+ },
+ { message }
+ )
+ .optional();
+
+// Define Zod schema for validation
+export const trackingPayloadSchema = z.discriminatedUnion("type", [
+ z
+ .object({
+ type: z.literal("pageview"),
+ ...baseEventFields,
+ ...defaultEventProps,
+ })
+ .strict(),
+ z
+ .object({
+ type: z.literal("custom_event"),
+ ...baseEventFields,
+ event_name: z.string().min(1).max(256),
+ properties: jsonStringRefine("Properties must be a valid JSON string"),
+ })
+ .strict(),
+ z
+ .object({
+ type: z.literal("performance"),
+ ...baseEventFields,
+ ...defaultEventProps,
+ // Performance metrics (can be null if not collected)
+ lcp: z.number().min(0).nullable().optional(),
+ cls: z.number().min(0).nullable().optional(),
+ inp: z.number().min(0).nullable().optional(),
+ fcp: z.number().min(0).nullable().optional(),
+ ttfb: z.number().min(0).nullable().optional(),
+ })
+ .strict(),
+ z
+ .object({
+ type: z.literal("outbound"),
+ ...baseEventFields,
+ event_name: z.string().max(256).optional(),
+ properties: z
+ .string()
+ .max(2048)
+ .refine(
+ val => {
+ try {
+ const parsed = JSON.parse(val);
+ if (typeof parsed.url !== "string" || parsed.url.length === 0) return false;
+ if (parsed.text && typeof parsed.text !== "string") return false;
+ if (parsed.target && typeof parsed.target !== "string") return false;
+ try {
+ new URL(parsed.url);
+ } catch {
+ return false;
+ }
+ return true;
+ } catch {
+ return false;
+ }
+ },
+ {
+ message: "Properties must be valid JSON with outbound link fields (url required, text and target optional)",
+ }
+ ),
+ })
+ .strict(),
+ z
+ .object({
+ type: z.literal("error"),
+ ...baseEventFields,
+ event_name: z.string().min(1).max(256), // Error type (TypeError, ReferenceError, etc.)
+ properties: z
+ .string()
+ .max(4096) // Larger limit for error details
+ .refine(
+ val => {
+ try {
+ const parsed = JSON.parse(val);
+ if (typeof parsed.message !== "string") return false;
+ if (parsed.stack && typeof parsed.stack !== "string") return false;
+ if (parsed.fileName && typeof parsed.fileName !== "string") return false;
+ if (parsed.lineNumber && typeof parsed.lineNumber !== "number") return false;
+ if (parsed.columnNumber && typeof parsed.columnNumber !== "number") return false;
+ // Apply truncation limits
+ if (parsed.message && parsed.message.length > 500) {
+ parsed.message = parsed.message.substring(0, 500);
+ }
+ if (parsed.stack && parsed.stack.length > 2000) {
+ parsed.stack = parsed.stack.substring(0, 2000);
+ }
+ return true;
+ } catch {
+ return false;
+ }
+ },
+ {
+ message:
+ "Properties must be valid JSON with error fields (message, stack, fileName, lineNumber, columnNumber)",
+ }
+ ),
+ })
+ .strict(),
+ z
+ .object({
+ type: z.literal("button_click"),
+ ...baseEventFields,
+ event_name: z.string().max(256).optional(),
+ properties: jsonStringRefine("Properties must be valid JSON"),
+ })
+ .strict(),
+ z
+ .object({
+ type: z.literal("copy"),
+ ...baseEventFields,
+ event_name: z.string().max(256).optional(),
+ properties: z
+ .string()
+ .max(2048)
+ .refine(
+ val => {
+ try {
+ const parsed = JSON.parse(val);
+ if (typeof parsed.sourceElement !== "string") return false;
+ if (parsed.text !== undefined && typeof parsed.text !== "string") return false;
+ if (parsed.textLength !== undefined && (typeof parsed.textLength !== "number" || parsed.textLength < 0))
+ return false;
+ return true;
+ } catch {
+ return false;
+ }
+ },
+ {
+ message:
+ "Properties must be valid JSON with copy fields (sourceElement required, text and textLength optional)",
+ }
+ ),
+ })
+ .strict(),
+ z
+ .object({
+ type: z.literal("form_submit"),
+ ...baseEventFields,
+ event_name: z.string().max(256).optional(),
+ properties: z
+ .string()
+ .max(2048)
+ .refine(
+ val => {
+ try {
+ const parsed = JSON.parse(val);
+ if (typeof parsed.formId !== "string") return false;
+ if (typeof parsed.formName !== "string") return false;
+ if (typeof parsed.formAction !== "string") return false;
+ if (typeof parsed.method !== "string") return false;
+ if (typeof parsed.fieldCount !== "number" || parsed.fieldCount < 0) return false;
+ return true;
+ } catch {
+ return false;
+ }
+ },
+ {
+ message:
+ "Properties must be valid JSON with form_submit fields (formId, formName, formAction, method, fieldCount required)",
+ }
+ ),
+ })
+ .strict(),
+ z
+ .object({
+ type: z.literal("input_change"),
+ ...baseEventFields,
+ event_name: z.string().max(256).optional(),
+ properties: z
+ .string()
+ .max(2048)
+ .refine(
+ val => {
+ try {
+ const parsed = JSON.parse(val);
+ if (typeof parsed.element !== "string") return false;
+ if (typeof parsed.inputName !== "string") return false;
+ return true;
+ } catch {
+ return false;
+ }
+ },
+ {
+ message: "Properties must be valid JSON with input_change fields (element, inputName required)",
+ }
+ ),
+ })
+ .strict(),
+]);
+
+/** The event body after validation β€” the only shape ingestion ever sees. */
+export type ValidatedTrackingPayload = z.infer<typeof trackingPayloadSchema>;
diff --git a/server/src/services/tracker/trackingRequest.test.ts b/server/src/services/tracker/trackingRequest.test.ts
new file mode 100644
index 000000000..b4e5c1271
--- /dev/null
+++ b/server/src/services/tracker/trackingRequest.test.ts
@@ -0,0 +1,149 @@
+import type { FastifyRequest } from "fastify";
+import { beforeEach, describe, expect, it, vi } from "vitest";
+import type { ValidatedTrackingPayload } from "./trackingPayload.js";
+
+const mocks = vi.hoisted(() => ({
+ getConfig: vi.fn(),
+ checkApiKey: vi.fn(),
+ lookupAsn: vi.fn(),
+}));
+
+vi.mock("../../lib/siteConfig.js", () => ({
+ siteConfig: { getConfig: mocks.getConfig },
+}));
+
+vi.mock("../../lib/auth-utils.js", () => ({
+ checkApiKey: mocks.checkApiKey,
+}));
+
+// The memoising factory is the real one, wired to a fake reader β€” these tests
+// exercise the shipped memoisation, not a stand-in for it.
+vi.mock("../../db/geolocation/asn.js", async importOriginal => {
+ const actual = await importOriginal<typeof import("../../db/geolocation/asn.js")>();
+ return {
+ ...actual,
+ lookupAsn: mocks.lookupAsn,
+ createAsnLookup: (lookup = mocks.lookupAsn) => actual.createAsnLookup(lookup),
+ };
+});
+
+import { resolveTrackingRequest } from "./trackingRequest.js";
+
+const site = {
+ id: "site_abc",
+ siteId: 42,
+ firstPartyProxy: false,
+ saltUserIds: false,
+};
+
+function request(headers: Record<string, string | string[]>, ip = "198.51.100.10"): FastifyRequest {
+ return { headers, ip } as unknown as FastifyRequest;
+}
+
+const payload = (overrides: Record<string, unknown> = {}) =>
+ ({ type: "pageview", site_id: "site_abc", ...overrides }) as ValidatedTrackingPayload;
+
+describe("resolveTrackingRequest", () => {
+ beforeEach(() => {
+ vi.clearAllMocks();
+ mocks.getConfig.mockResolvedValue(site);
+ mocks.lookupAsn.mockReturnValue(null);
+ });
+
+ it("returns null for an unknown Site", async () => {
+ mocks.getConfig.mockResolvedValue(undefined);
+
+ expect(await resolveTrackingRequest(request({}), payload())).toBeNull();
+ });
+
+ it("reads the Site Configuration once for the whole request", async () => {
+ await resolveTrackingRequest(request({}), payload());
+
+ expect(mocks.getConfig).toHaveBeenCalledOnce();
+ expect(mocks.getConfig).toHaveBeenCalledWith("site_abc");
+ });
+
+ it("ignores public payload IP and user-agent overrides", async () => {
+ const resolved = await resolveTrackingRequest(
+ request({
+ "user-agent": "Mozilla/5.0 Chrome/120 Safari/537.36",
+ "x-forwarded-for": "198.51.100.20, 10.0.0.1",
+ }),
+ payload({ ip_address: "203.0.113.10", user_agent: "SpoofedBot/1.0" })
+ );
+
+ expect(resolved).toMatchObject({
+ ipAddress: "198.51.100.20",
+ userAgent: "Mozilla/5.0 Chrome/120 Safari/537.36",
+ trustedServerSideIngestion: false,
+ // Exclusion candidates still include everything the request presented.
+ candidateIps: ["198.51.100.20", "10.0.0.1", "198.51.100.10"],
+ });
+ });
+
+ it("allows payload IP and user-agent overrides for trusted server-side ingestion", async () => {
+ mocks.checkApiKey.mockResolvedValue({
+ valid: true,
+ statements: { ingest: ["write"] },
+ });
+
+ const resolved = await resolveTrackingRequest(
+ request({
+ authorization: "Bearer sk_test",
+ "user-agent": "ServerSDK/1.0",
+ "x-forwarded-for": "198.51.100.20",
+ }),
+ payload({ ip_address: "203.0.113.10", user_agent: "Mozilla/5.0 Chrome/120 Safari/537.36" })
+ );
+
+ expect(resolved).toMatchObject({
+ ipAddress: "203.0.113.10",
+ userAgent: "Mozilla/5.0 Chrome/120 Safari/537.36",
+ trustedServerSideIngestion: true,
+ candidateIps: ["203.0.113.10", "198.51.100.20", "198.51.100.10"],
+ });
+ });
+
+ it("degrades an under-scoped bearer to untrusted client-side traffic", async () => {
+ mocks.checkApiKey.mockResolvedValue({
+ valid: true,
+ statements: { site: ["read"] },
+ });
+
+ const resolved = await resolveTrackingRequest(
+ request({ authorization: "Bearer sk_test", "user-agent": "ServerSDK/1.0" }),
+ payload({ ip_address: "203.0.113.10" })
+ );
+
+ expect(resolved).toMatchObject({
+ ipAddress: "198.51.100.10",
+ trustedServerSideIngestion: false,
+ });
+ });
+
+ it("carries an ASN resolver that answers each IP once per request", async () => {
+ mocks.lookupAsn.mockReturnValue({ asn: 13335, organization: "Cloudflare" });
+
+ const resolved = await resolveTrackingRequest(request({}), payload());
+ mocks.lookupAsn.mockClear();
+
+ expect(resolved!.lookupAsn("203.0.113.7")).toEqual({ asn: 13335, organization: "Cloudflare" });
+ expect(resolved!.lookupAsn("203.0.113.7")).toEqual({ asn: 13335, organization: "Cloudflare" });
+ expect(mocks.lookupAsn).toHaveBeenCalledOnce();
+ });
+
+ it("resolves the client IP through the same request-scoped ASN resolver", async () => {
+ // A datacenter edge IP means a first-party proxy is in front, so the
+ // forwarded visitor IP wins over the Cloudflare edge IP.
+ mocks.lookupAsn.mockImplementation((ip: string) =>
+ ip === "203.0.113.99" ? { asn: 16509, organization: "AMAZON-02" } : null
+ );
+
+ const resolved = await resolveTrackingRequest(
+ request({ "cf-connecting-ip": "203.0.113.99", "x-forwarded-for": "198.51.100.20" }),
+ payload()
+ );
+
+ expect(resolved!.ipAddress).toBe("198.51.100.20");
+ });
+});
diff --git a/server/src/services/tracker/trackingRequest.ts b/server/src/services/tracker/trackingRequest.ts
new file mode 100644
index 000000000..e6fd98a39
--- /dev/null
+++ b/server/src/services/tracker/trackingRequest.ts
@@ -0,0 +1,113 @@
+import type { FastifyRequest } from "fastify";
+import { createAsnLookup, type AsnLookup } from "../../db/geolocation/asn.js";
+import { checkApiKey } from "../../lib/auth-utils.js";
+import { hasScope } from "../../lib/scopes.js";
+import { siteConfig, type SiteConfigData } from "../../lib/siteConfig.js";
+import { getRequestUserAgent } from "../../utils.js";
+import { collectCandidateClientIps, resolveClientIp } from "./resolveClientIp.js";
+import type { ValidatedTrackingPayload } from "./trackingPayload.js";
+
+/**
+ * One tracking request, fully resolved.
+ *
+ * Everything downstream of ingestion needs the same handful of facts about a
+ * request β€” who sent it, from where, for which Site β€” and each of them used to
+ * re-derive those facts from the raw `FastifyRequest` on its own. That made the
+ * derivations impossible to keep consistent (identity was resolved twice, per
+ * different rules) and impossible to test without an HTTP stub.
+ *
+ * This is that shared value: resolved once at the edge of the handler, then
+ * passed by value. It carries no Fastify types, so ingestion can be exercised
+ * directly with an object literal.
+ */
+export interface TrackingRequest {
+ /** The event body, already validated. */
+ payload: ValidatedTrackingPayload;
+ /** Site Configuration for `payload.site_id`, fetched once for the request. */
+ site: SiteConfigData;
+ /**
+ * The authoritative client IP: identity, geolocation and bot detection all
+ * use this one. Never re-resolve it from headers downstream.
+ */
+ ipAddress: string;
+ /** The authoritative user agent, under the same rule as `ipAddress`. */
+ userAgent: string;
+ /**
+ * Every plausible client IP for this request (edge, forwarded hops, socket).
+ * Exclusion matching only β€” over-matching is the safe failure mode there.
+ */
+ candidateIps: string[];
+ /**
+ * A bearer token with `ingest:write` scope for this Site let the payload
+ * override IP and user agent. Bot detection is skipped for these.
+ */
+ trustedServerSideIngestion: boolean;
+ /** Request headers, for the layers that read beyond the user agent. */
+ headers: FastifyRequest["headers"];
+ /** Request-scoped ASN resolver: each IP is looked up at most once per event. */
+ lookupAsn: AsnLookup;
+ /**
+ * When the request arrived. This is the event's timestamp *and* the instant
+ * that decides which UTC day its salted fingerprint belongs to, so the two can
+ * never disagree about the day.
+ */
+ receivedAt: Date;
+}
+
+/**
+ * Whether a bearer token authorises this request to declare its own client IP
+ * and user agent.
+ *
+ * An under-scoped bearer degrades to untrusted client-side traffic (no 4xx) β€”
+ * this is a public ingestion endpoint, not an authenticated API.
+ */
+async function isTrustedServerSideIngestion(request: FastifyRequest, siteId: number): Promise<boolean> {
+ const authHeader = request.headers["authorization"];
+ if (typeof authHeader !== "string" || !authHeader.startsWith("Bearer ")) {
+ return false;
+ }
+
+ const apiKeyResult = await checkApiKey(request, { siteId });
+ return apiKeyResult.valid && hasScope(apiKeyResult.statements, { resource: "ingest", action: "write" });
+}
+
+/**
+ * Resolve a validated payload and its HTTP request into a Tracking Request.
+ *
+ * Returns null when the Site is unknown β€” the only way resolution fails, and
+ * the caller's cue to answer 404.
+ */
+export async function resolveTrackingRequest(
+ request: FastifyRequest,
+ payload: ValidatedTrackingPayload
+): Promise<TrackingRequest | null> {
+ // Stamped before the first await: a cold Site Configuration read or a slow
+ // API-key check must not push the event's time forward into another bucket.
+ const receivedAt = new Date();
+
+ const site = await siteConfig.getConfig(payload.site_id);
+ if (!site) return null;
+
+ const lookupAsn = createAsnLookup();
+ const trustedServerSideIngestion = await isTrustedServerSideIngestion(request, site.siteId);
+
+ const requestIpAddress = resolveClientIp(request, { firstPartyProxy: site.firstPartyProxy, lookupAsn });
+ const requestUserAgent = getRequestUserAgent(request.headers);
+
+ // Only a trusted bearer may speak for someone else; for everyone else the
+ // payload's ip_address/user_agent are ignored rather than trusted.
+ const ipAddress = trustedServerSideIngestion ? payload.ip_address || requestIpAddress : requestIpAddress;
+ const userAgent = trustedServerSideIngestion ? payload.user_agent || requestUserAgent : requestUserAgent;
+
+ return {
+ payload,
+ site,
+ ipAddress,
+ userAgent,
+ candidateIps: collectCandidateClientIps(request, [ipAddress, requestIpAddress]),
+ trustedServerSideIngestion,
+ headers: request.headers,
+ lookupAsn,
+ receivedAt,
+ };
+}
diff --git a/server/src/services/tracker/utils.test.ts b/server/src/services/tracker/utils.test.ts
new file mode 100644
index 000000000..a74ae2d16
--- /dev/null
+++ b/server/src/services/tracker/utils.test.ts
@@ -0,0 +1,145 @@
+import { beforeEach, describe, expect, it, vi } from "vitest";
+import type { SiteConfigData } from "../../lib/siteConfig.js";
+import type { TrackingRequest } from "./trackingRequest.js";
+
+const mocks = vi.hoisted(() => ({
+ generateUserId: vi.fn(),
+ generateUserIdFromClientId: vi.fn(),
+}));
+
+vi.mock("../userId/userIdService.js", () => ({
+ userIdService: {
+ generateUserId: mocks.generateUserId,
+ generateUserIdFromClientId: mocks.generateUserIdFromClientId,
+ },
+}));
+
+import { createBasePayload } from "./utils.js";
+
+const site = {
+ id: "site_abc",
+ siteId: 42,
+ saltUserIds: false,
+ trackIp: false,
+} as unknown as SiteConfigData;
+
+const lookupAsn = vi.fn(() => null);
+
+function trackingRequest(overrides: Partial<TrackingRequest> = {}): TrackingRequest {
+ return {
+ payload: { type: "pageview", site_id: "site_abc" } as TrackingRequest["payload"],
+ site,
+ ipAddress: "198.51.100.10",
+ userAgent: "Mozilla/5.0 (Macintosh) Chrome/120 Safari/537.36",
+ candidateIps: ["198.51.100.10"],
+ trustedServerSideIngestion: false,
+ headers: {},
+ lookupAsn,
+ receivedAt: new Date("2026-08-14T10:38:37.000Z"),
+ ...overrides,
+ };
+}
+
+describe("createBasePayload", () => {
+ beforeEach(() => {
+ vi.clearAllMocks();
+ mocks.generateUserId.mockResolvedValue("device-fingerprint");
+ mocks.generateUserIdFromClientId.mockResolvedValue("client-id-fingerprint");
+ });
+
+ it("uses the request's resolved identity rather than re-deriving one", async () => {
+ const payload = await createBasePayload(trackingRequest({ ipAddress: "203.0.113.10", userAgent: "ServerSDK/1.0" }));
+
+ expect(mocks.generateUserId).toHaveBeenCalledWith("203.0.113.10", "ServerSDK/1.0", 42, {
+ saltUserIds: false,
+ lookupAsn,
+ receivedAt: new Date("2026-08-14T10:38:37.000Z"),
+ });
+ expect(payload).toMatchObject({
+ ipAddress: "203.0.113.10",
+ userAgent: "ServerSDK/1.0",
+ userId: "device-fingerprint",
+ identifiedUserId: "",
+ });
+ });
+
+ it("stamps the event with the time the request was accepted", async () => {
+ const payload = await createBasePayload(trackingRequest());
+
+ expect(payload.timestamp).toBe("2026-08-14T10:38:37.000Z");
+ });
+
+ it("resolves the numeric Site id and defaults the optional string fields", async () => {
+ const payload = await createBasePayload(trackingRequest());
+
+ expect(payload).toMatchObject({
+ site_id: 42,
+ hostname: "",
+ pathname: "",
+ querystring: "",
+ page_title: "",
+ referrer: "",
+ language: "",
+ screenWidth: 0,
+ screenHeight: 0,
+ type: "pageview",
+ });
+ });
+
+ it("never leaks the payload's IP and user-agent overrides into the event", async () => {
+ const payload = await createBasePayload(
+ trackingRequest({
+ payload: {
+ type: "pageview",
+ site_id: "site_abc",
+ ip_address: "203.0.113.10",
+ user_agent: "SpoofedBot/1.0",
+ } as TrackingRequest["payload"],
+ })
+ );
+
+ expect(payload).not.toHaveProperty("ip_address");
+ expect(payload).not.toHaveProperty("user_agent");
+ });
+
+ it("prefers an anonymous id over the IP/user-agent fingerprint", async () => {
+ const payload = await createBasePayload(
+ trackingRequest({
+ site: { ...site, saltUserIds: true } as SiteConfigData,
+ payload: {
+ type: "pageview",
+ site_id: "site_abc",
+ anonymous_id: "consented-visitor",
+ } as TrackingRequest["payload"],
+ })
+ );
+
+ expect(mocks.generateUserIdFromClientId).toHaveBeenCalledWith("consented-visitor", 42, {
+ saltUserIds: true,
+ receivedAt: new Date("2026-08-14T10:38:37.000Z"),
+ });
+ expect(mocks.generateUserId).not.toHaveBeenCalled();
+ expect(payload.userId).toBe("client-id-fingerprint");
+ });
+
+ it("keeps the device fingerprint separate from an identified user", async () => {
+ const payload = await createBasePayload(
+ trackingRequest({
+ payload: {
+ type: "pageview",
+ site_id: "site_abc",
+ user_id: " employee-alice ",
+ } as TrackingRequest["payload"],
+ })
+ );
+
+ expect(payload.userId).toBe("device-fingerprint");
+ expect(payload.identifiedUserId).toBe("employee-alice");
+ });
+
+ it("carries the Site's IP storage setting onto the event", async () => {
+ const payload = await createBasePayload(trackingRequest({ site: { ...site, trackIp: true } as SiteConfigData }));
+
+ expect(payload.storeIp).toBe(true);
+ });
+});
diff --git a/server/src/services/tracker/utils.ts b/server/src/services/tracker/utils.ts
index c20252b55..19d6c37bb 100644
--- a/server/src/services/tracker/utils.ts
+++ b/server/src/services/tracker/utils.ts
@@ -1,11 +1,7 @@
-import { FastifyRequest } from "fastify";
import UAParser, { UAParser as userAgentParser } from "ua-parser-js";
-import { z } from "zod";
import { userIdService } from "../userId/userIdService.js";
-import { trackingPayloadSchema } from "./trackEvent.js";
+import type { TrackingRequest } from "./trackingRequest.js";
import { TrackingPayload } from "./types.js";
-import { SiteConfigData } from "../../lib/siteConfig.js";
-import { resolveTrackingIdentity } from "./requestIdentity.js";
export type TotalTrackingPayload = TrackingPayload & {
userId: string; // Always the device fingerprint
@@ -28,9 +24,6 @@ export type TotalTrackingPayload = TrackingPayload & {
feature_flags?: Record<string, string>;
};
-// Infer type from Zod schema
-type ValidatedTrackingPayload = z.infer<typeof trackingPayloadSchema>;
-
// ua-parser-js is regex-heavy and runs on every event. User-agent strings repeat
// constantly across requests, so memoize parsing in a bounded LRU-ish cache to
// keep this off the per-event hot path.
@@ -117,57 +110,50 @@ export function clearSelfReferrer(referrer: string, hostname: string): string {
return referrer;
}
-// Create base tracking payload from request
-export async function createBasePayload(
- request: FastifyRequest,
- eventType:
- | "pageview"
- | "custom_event"
- | "performance"
- | "error"
- | "outbound"
- | "button_click"
- | "copy"
- | "form_submit"
- | "input_change" = "pageview",
- validatedBody: ValidatedTrackingPayload,
- siteConfiguration: SiteConfigData,
- trustedServerSideIngestion = false
-): Promise<TotalTrackingPayload> {
- const { ipAddress, userAgent } = resolveTrackingIdentity(
- request,
- validatedBody,
- trustedServerSideIngestion,
- siteConfiguration.firstPartyProxy
- );
- const { ip_address: _ipAddressOverride, user_agent: _userAgentOverride, ...payloadBody } = validatedBody;
-
- const anonymousId = validatedBody.anonymous_id
- ? await userIdService.generateUserIdFromClientId(validatedBody.anonymous_id, siteConfiguration.siteId)
- : await userIdService.generateUserId(ipAddress, userAgent, siteConfiguration.siteId);
+/**
+ * Build the queued event from an already-resolved Tracking Request.
+ *
+ * Identity, IP, user agent and the Site Configuration all arrive resolved β€”
+ * this does not re-derive any of them, so the fingerprint hashed here is by
+ * construction the same one bot detection and exclusion matching just saw.
+ */
+export async function createBasePayload(trackingRequest: TrackingRequest): Promise<TotalTrackingPayload> {
+ const { payload, site, ipAddress, userAgent } = trackingRequest;
+ const { ip_address: _ipAddressOverride, user_agent: _userAgentOverride, ...payloadBody } = payload;
+
+ const anonymousId = payload.anonymous_id
+ ? await userIdService.generateUserIdFromClientId(payload.anonymous_id, site.siteId, {
+ saltUserIds: site.saltUserIds,
+ receivedAt: trackingRequest.receivedAt,
+ })
+ : await userIdService.generateUserId(ipAddress, userAgent, site.siteId, {
+ saltUserIds: site.saltUserIds,
+ lookupAsn: trackingRequest.lookupAsn,
+ receivedAt: trackingRequest.receivedAt,
+ });
// userId is always the device fingerprint
// identifiedUserId is the custom user ID when provided, empty string otherwise
- const identifiedUserId = validatedBody.user_id ? validatedBody.user_id.trim() : "";
+ const identifiedUserId = payload.user_id ? payload.user_id.trim() : "";
return {
...payloadBody,
- site_id: siteConfiguration.siteId, // Use the numeric site ID
- hostname: validatedBody.hostname || "",
- pathname: validatedBody.pathname || "",
- querystring: validatedBody.querystring || "",
- screenWidth: validatedBody.screenWidth || 0,
- screenHeight: validatedBody.screenHeight || 0,
- language: validatedBody.language || "",
- page_title: validatedBody.page_title || "",
- referrer: validatedBody.referrer || "",
- type: eventType,
+ site_id: site.siteId, // Use the numeric site ID
+ hostname: payload.hostname || "",
+ pathname: payload.pathname || "",
+ querystring: payload.querystring || "",
+ screenWidth: payload.screenWidth || 0,
+ screenHeight: payload.screenHeight || 0,
+ language: payload.language || "",
+ page_title: payload.page_title || "",
+ referrer: payload.referrer || "",
+ type: payload.type,
ipAddress: ipAddress,
- timestamp: new Date().toISOString(),
+ timestamp: trackingRequest.receivedAt.toISOString(),
ua: parseUserAgent(userAgent),
userAgent,
userId: anonymousId, // Always the device fingerprint
identifiedUserId: identifiedUserId, // Custom user ID when identified
- storeIp: siteConfiguration.trackIp,
+ storeIp: site.trackIp,
} as any;
}
diff --git a/server/src/services/userId/userIdService.test.ts b/server/src/services/userId/userIdService.test.ts
new file mode 100644
index 000000000..1017569a9
--- /dev/null
+++ b/server/src/services/userId/userIdService.test.ts
@@ -0,0 +1,133 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+
+const mocks = vi.hoisted(() => ({
+ getConfig: vi.fn(),
+ resolveStickyUserId: vi.fn(),
+}));
+
+vi.mock("../../lib/const.js", () => ({
+ SECRET: "test-secret",
+}));
+
+vi.mock("../../lib/siteConfig.js", () => ({
+ siteConfig: { getConfig: mocks.getConfig },
+}));
+
+// Sticky re-attachment has its own tests; here it is a passthrough that lets us
+// observe the saltScope the fingerprint was built under.
+vi.mock("./stickyUserId.js", () => ({
+ resolveStickyUserId: mocks.resolveStickyUserId,
+}));
+
+vi.mock("../../db/geolocation/asn.js", () => ({
+ lookupAsn: vi.fn(() => null),
+}));
+
+import { userIdService } from "./userIdService.js";
+
+const DAY_END = new Date("2026-08-14T23:59:59.900Z");
+const NEXT_DAY = new Date("2026-08-15T00:00:00.100Z");
+
+describe("userIdService", () => {
+ beforeEach(() => {
+ vi.clearAllMocks();
+ mocks.getConfig.mockResolvedValue({ saltUserIds: true });
+ mocks.resolveStickyUserId.mockImplementation(async input => input.rawUserId);
+ });
+
+ afterEach(() => {
+ vi.useRealTimers();
+ });
+
+ it("reads the Site Configuration when the caller does not supply saltUserIds", async () => {
+ await userIdService.generateUserId("198.51.100.10", "Mozilla/5.0", 42);
+
+ expect(mocks.getConfig).toHaveBeenCalledWith(42);
+ });
+
+ it("skips the Site Configuration read when the caller already knows the setting", async () => {
+ await userIdService.generateUserId("198.51.100.10", "Mozilla/5.0", 42, { saltUserIds: true });
+
+ expect(mocks.getConfig).not.toHaveBeenCalled();
+ });
+
+ // The event's UTC day comes from receivedAt, not from the clock at hashing
+ // time β€” otherwise an event accepted at 23:59:59.9 gets stamped with day D
+ // but fingerprinted with day D+1's salt.
+ it("salts against the day the event arrived, not the day it reached the hash", async () => {
+ vi.useFakeTimers();
+ vi.setSystemTime(DAY_END);
+ const beforeMidnight = await userIdService.generateUserId("198.51.100.10", "Mozilla/5.0", 42, {
+ saltUserIds: true,
+ receivedAt: DAY_END,
+ });
+
+ // Ingestion crossed midnight between accepting the event and hashing it.
+ vi.setSystemTime(NEXT_DAY);
+ const acrossMidnight = await userIdService.generateUserId("198.51.100.10", "Mozilla/5.0", 42, {
+ saltUserIds: true,
+ receivedAt: DAY_END,
+ });
+
+ expect(acrossMidnight).toBe(beforeMidnight);
+ expect(mocks.resolveStickyUserId).toHaveBeenLastCalledWith(expect.objectContaining({ saltScope: "2026-08-14" }));
+ });
+
+ it("still rotates the fingerprint for an event that genuinely belongs to the next day", async () => {
+ const dayOne = await userIdService.generateUserId("198.51.100.10", "Mozilla/5.0", 42, {
+ saltUserIds: true,
+ receivedAt: DAY_END,
+ });
+ const dayTwo = await userIdService.generateUserId("198.51.100.10", "Mozilla/5.0", 42, {
+ saltUserIds: true,
+ receivedAt: NEXT_DAY,
+ });
+
+ expect(dayTwo).not.toBe(dayOne);
+ expect(mocks.resolveStickyUserId).toHaveBeenLastCalledWith(expect.objectContaining({ saltScope: "2026-08-15" }));
+ });
+
+ it("leaves an unsalted Site's fingerprint stable across the day boundary", async () => {
+ const dayOne = await userIdService.generateUserId("198.51.100.10", "Mozilla/5.0", 42, {
+ saltUserIds: false,
+ receivedAt: DAY_END,
+ });
+ const dayTwo = await userIdService.generateUserId("198.51.100.10", "Mozilla/5.0", 42, {
+ saltUserIds: false,
+ receivedAt: NEXT_DAY,
+ });
+
+ expect(dayTwo).toBe(dayOne);
+ expect(mocks.resolveStickyUserId).toHaveBeenLastCalledWith(expect.objectContaining({ saltScope: "" }));
+ });
+
+ it("salts an anonymous id against the event's day too", async () => {
+ vi.useFakeTimers();
+ vi.setSystemTime(DAY_END);
+ const beforeMidnight = await userIdService.generateUserIdFromClientId("consented-visitor", 42, {
+ saltUserIds: true,
+ receivedAt: DAY_END,
+ });
+
+ vi.setSystemTime(NEXT_DAY);
+ const acrossMidnight = await userIdService.generateUserIdFromClientId("consented-visitor", 42, {
+ saltUserIds: true,
+ receivedAt: DAY_END,
+ });
+
+ expect(acrossMidnight).toBe(beforeMidnight);
+ });
+
+ it("keeps an unsalted anonymous id independent of the day", async () => {
+ const dayOne = await userIdService.generateUserIdFromClientId("consented-visitor", 42, {
+ saltUserIds: false,
+ receivedAt: DAY_END,
+ });
+ const dayTwo = await userIdService.generateUserIdFromClientId("consented-visitor", 42, {
+ saltUserIds: false,
+ receivedAt: NEXT_DAY,
+ });
+
+ expect(dayTwo).toBe(dayOne);
+ });
+});
diff --git a/server/src/services/userId/userIdService.ts b/server/src/services/userId/userIdService.ts
index 4d42bb07b..d55b78169 100644
--- a/server/src/services/userId/userIdService.ts
+++ b/server/src/services/userId/userIdService.ts
@@ -1,22 +1,52 @@
import crypto from "crypto";
+import { lookupAsn, type AsnLookup } from "../../db/geolocation/asn.js";
import { SECRET } from "../../lib/const.js";
import { siteConfig } from "../../lib/siteConfig.js";
+import { isDatacenterAsn } from "../tracker/botBlocking/datacenterAsns.js";
import { bucketIpForIdentity } from "./identityIpBucket.js";
import { resolveStickyUserId } from "./stickyUserId.js";
+export interface UserIdOptions {
+ /**
+ * The Site's salting setting when the caller already holds its Site
+ * Configuration. Omit it and the config is fetched β€” a second read of a row
+ * the tracking path has usually just loaded.
+ */
+ saltUserIds?: boolean;
+ /** Request-scoped ASN resolver, shared with the rest of the ingestion path. */
+ lookupAsn?: AsnLookup;
+ /**
+ * The moment the event belongs to β€” the same instant it will be timestamped
+ * with. The daily salt is picked for *this* instant's UTC day rather than for
+ * "now", so an event accepted at 23:59:59.9 can't be stamped with day D and
+ * fingerprinted with day D+1's salt because ingestion took a millisecond to
+ * reach the hash. Defaults to now for callers outside the tracking path.
+ */
+ receivedAt?: Date;
+}
+
+/** The UTC day (YYYY-MM-DD) an instant falls in. */
+function utcDayOf(instant: Date): string {
+ return instant.toISOString().split("T")[0];
+}
+
class UserIdService {
private cachedSalt: string | null = null;
private cacheDate: string | null = null; // Store the date the salt was generated for (YYYY-MM-DD format)
/**
- * Generates a deterministic daily salt based on a secret environment variable.
- * The salt remains the same for the entire UTC day and changes automatically
- * when the UTC date changes. Caches the salt in memory for efficiency.
+ * Generates a deterministic salt for a given UTC day from a secret environment
+ * variable. The salt is the same for the whole day and changes when the day
+ * does. Caches the most recent day's salt in memory for efficiency.
+ *
+ * The day is a parameter, not a reading of the clock: the caller decides which
+ * day an event belongs to, so a request straddling midnight hashes against one
+ * day throughout.
*
* @throws {Error} If the BETTER_AUTH_SECRET environment variable is not set.
* @returns {string} The daily salt as a hex string.
*/
- private getDailySalt(): string {
+ private getDailySalt(utcDay: string): string {
if (!SECRET) {
console.error(
"FATAL: BETTER_AUTH_SECRET environment variable is not set. User ID generation will be insecure or fail."
@@ -24,22 +54,28 @@ class UserIdService {
throw new Error("BETTER_AUTH_SECRET environment variable is missing.");
}
- // Use UTC date to ensure consistency across timezones and server restarts
- const currentDate = new Date().toISOString().split("T")[0]; // Gets 'YYYY-MM-DD' in UTC
-
- // Check if the cached salt is still valid for the current UTC date
- if (this.cachedSalt && this.cacheDate === currentDate) {
+ if (this.cachedSalt && this.cacheDate === utcDay) {
return this.cachedSalt;
}
- const input = SECRET + currentDate;
+ const input = SECRET + utcDay;
const newSalt = crypto.createHash("sha256").update(input).digest("hex");
this.cachedSalt = newSalt;
- this.cacheDate = currentDate;
+ this.cacheDate = utcDay;
return newSalt;
}
+ /**
+ * Whether this Site salts user IDs β€” from the caller's already-loaded Site
+ * Configuration when it has one, otherwise fetched.
+ */
+ private async isSalted(siteId: number, options: UserIdOptions): Promise<boolean> {
+ if (options.saltUserIds !== undefined) return options.saltUserIds;
+ const config = await siteConfig.getConfig(siteId);
+ return !!config?.saltUserIds;
+ }
+
/**
* Generate a user ID based on IP and user agent
* If the site has salting enabled, also includes a daily rotating salt
@@ -47,19 +83,23 @@ class UserIdService {
* @param ip User's IP address
* @param userAgent User's user agent string
* @param siteId The site ID to check for salting configuration
+ * @param options Facts the caller already holds, so they aren't looked up again
* @returns A truncated sha256 hash (12 chars) to identify the user
*/
- async generateUserId(ip: string, userAgent: string, siteId: number): Promise<string> {
+ async generateUserId(ip: string, userAgent: string, siteId: number, options: UserIdOptions = {}): Promise<string> {
+ const asnLookup = options.lookupAsn ?? lookupAsn;
+ const isDatacenterEgress = (candidate: string) => isDatacenterAsn(asnLookup(candidate)?.asn);
+
// Datacenter egress (corporate proxies, WARP, Private Relay) rotates IPs
// between requests; hash a coarse bucket so one visitor stays one user.
- const identityIp = bucketIpForIdentity(ip);
+ const identityIp = bucketIpForIdentity(ip, isDatacenterEgress);
- // Only apply salt if the site has salting enabled
- const config = await siteConfig.getConfig(siteId);
- const salted = !!config?.saltUserIds;
+ // The event's own day, fixed before anything is hashed against it.
+ const saltDay = utcDayOf(options.receivedAt ?? new Date());
+ const salted = await this.isSalted(siteId, options);
const rawUserId = crypto
.createHash("sha256")
- .update(identityIp + userAgent + (salted ? this.getDailySalt() : ""))
+ .update(identityIp + userAgent + (salted ? this.getDailySalt(saltDay) : ""))
.digest("hex")
.substring(0, 12);
@@ -70,14 +110,15 @@ class UserIdService {
rawUserId,
ipAddress: ip,
userAgent,
- // cacheDate is the UTC date the salt in rawUserId was built from.
- saltScope: salted ? (this.cacheDate ?? "") : "",
+ // The UTC day the salt in rawUserId was built from.
+ saltScope: salted ? saltDay : "",
+ isDatacenterEgress,
});
}
- async generateUserIdFromClientId(clientId: string, siteId: number): Promise<string> {
- const config = await siteConfig.getConfig(siteId);
- const salt = config?.saltUserIds ? this.getDailySalt() : "";
+ async generateUserIdFromClientId(clientId: string, siteId: number, options: UserIdOptions = {}): Promise<string> {
+ const saltDay = utcDayOf(options.receivedAt ?? new Date());
+ const salt = (await this.isSalted(siteId, options)) ? this.getDailySalt(saltDay) : "";
return crypto.createHash("sha256").update(`${siteId}:${clientId}:${salt}`).digest("hex").substring(0, 12);
}
diff --git a/server/src/utils.ts b/server/src/utils.ts
index 4946e68c0..465d425a6 100644
--- a/server/src/utils.ts
+++ b/server/src/utils.ts
@@ -287,3 +287,13 @@ export const getIpAddress = (request: FastifyRequest): string => {
return request.ip;
};
+
+// The request's user agent as a single string. Node exposes a repeated header as
+// an array; take the first rather than the joined form so parsing sees one UA.
+export const getRequestUserAgent = (headers: FastifyRequest["headers"]): string => {
+ const userAgentHeader = headers["user-agent"];
+ if (Array.isArray(userAgentHeader)) {
+ return userAgentHeader[0] || "";
+ }
+ return userAgentHeader || "";
+};