🐸 Rybbit - open-source and privacy-friendly alternative to Google Analytics that is 10x more intuitive.
import UAParser, { UAParser as userAgentParser } from "ua-parser-js";
import { userIdService } from "../userId/userIdService.js";
import type { TrackingRequest } from "./trackingRequest.js";
import { TrackingPayload } from "./types.js";
export type TotalTrackingPayload = TrackingPayload & {
userId: string; // Always the device fingerprint
identifiedUserId: string; // Custom user ID when identified, empty string otherwise
timestamp: string;
type?: string;
event_name?: string;
properties?: string;
ua: UAParser.IResult;
userAgent: string;
referrer: string;
ipAddress: string;
storeIp?: boolean;
lcp?: number;
cls?: number;
inp?: number;
fcp?: number;
ttfb?: number;
tag?: string;
feature_flags?: Record<string, string>;
visitor_id?: string;
};
// ua-parser-js is regex-heavy and runs on every event. User-agent strings repeat
// constantly across requests, so memoize parsing in a bounded LRU-ish cache to
// keep this off the per-event hot path.
const UA_CACHE_MAX = 10_000;
const uaCache = new Map<string, UAParser.IResult>();
export function parseUserAgent(userAgent: string): UAParser.IResult {
const cached = uaCache.get(userAgent);
if (cached) {
// Refresh recency: re-insert so frequently-seen UAs survive eviction.
uaCache.delete(userAgent);
uaCache.set(userAgent, cached);
return cached;
}
const result = userAgentParser(userAgent);
uaCache.set(userAgent, result);
if (uaCache.size > UA_CACHE_MAX) {
// Evict the oldest (least recently used) entry.
const oldest = uaCache.keys().next().value;
if (oldest !== undefined) uaCache.delete(oldest);
}
return result;
}
// UTM and URL parameter parsing utilities
export function getUTMParams(querystring: string): Record<string, string> {
const params: Record<string, string> = {};
if (!querystring) return params;
try {
const searchParams = new URLSearchParams(querystring);
// Extract UTM parameters
for (const [key, value] of searchParams.entries()) {
if (key.startsWith("utm_") || key === "gclid" || key === "gad_source") {
params[key.toLowerCase()] = value.toLowerCase();
}
}
} catch (e) {
console.error("Error parsing query string:", e);
}
return params;
}
// Parse all URL parameters from querystring
export function getAllUrlParams(querystring: string): Record<string, string> {
const params: Record<string, string> = {};
if (!querystring) return params;
// If querystring starts with ?, remove it
const cleanQuerystring = querystring.startsWith("?") ? querystring.substring(1) : querystring;
try {
const searchParams = new URLSearchParams(cleanQuerystring);
// Extract all parameters
for (const [key, value] of searchParams.entries()) {
params[key.toLowerCase()] = value;
}
} catch (e) {
console.error("Error parsing query string for URL parameters:", e);
}
return params;
}
// Clear referrer if it's from the same domain
export function clearSelfReferrer(referrer: string, hostname: string): string {
if (!referrer || !hostname) return referrer;
try {
const referrerUrl = new URL(referrer);
if (referrerUrl.hostname === hostname) {
// Internal navigation, clear the referrer
return "";
}
} catch (e) {
// Invalid URL, return original referrer
}
return referrer;
}
/**
* Build the queued event from an already-resolved Tracking Request.
*
* Identity, IP, user agent and the Site Configuration all arrive resolved —
* this does not re-derive any of them, so the fingerprint hashed here is by
* construction the same one bot detection and exclusion matching just saw.
*/
export async function createBasePayload(trackingRequest: TrackingRequest): Promise<TotalTrackingPayload> {
const { payload, site, ipAddress, userAgent } = trackingRequest;
const { ip_address: _ipAddressOverride, user_agent: _userAgentOverride, ...payloadBody } = payload;
const anonymousId = payload.anonymous_id
? await userIdService.generateUserIdFromClientId(payload.anonymous_id, site.siteId, {
saltUserIds: site.saltUserIds,
receivedAt: trackingRequest.receivedAt,
})
: await userIdService.generateUserId(ipAddress, userAgent, site.siteId, {
saltUserIds: site.saltUserIds,
lookupAsn: trackingRequest.lookupAsn,
receivedAt: trackingRequest.receivedAt,
});
// userId is always the device fingerprint
// identifiedUserId is the custom user ID when provided, empty string otherwise
const identifiedUserId = payload.user_id ? payload.user_id.trim() : "";
return {
...payloadBody,
site_id: site.siteId, // Use the numeric site ID
hostname: payload.hostname || "",
pathname: payload.pathname || "",
querystring: payload.querystring || "",
screenWidth: payload.screenWidth || 0,
screenHeight: payload.screenHeight || 0,
language: payload.language || "",
page_title: payload.page_title || "",
referrer: payload.referrer || "",
type: payload.type,
ipAddress: ipAddress,
timestamp: trackingRequest.receivedAt.toISOString(),
ua: parseUserAgent(userAgent),
userAgent,
userId: anonymousId, // Always the device fingerprint
identifiedUserId: identifiedUserId, // Custom user ID when identified
storeIp: site.trackIp,
} as any;
}
// The props column is ClickHouse JSON, which only accepts objects. Validation
// lets any JSON through (e.g. rybbit.event("download", "file.pdf") sends a
// string), and one non-object row fails the whole insert, so drop those props
// here and keep the event.
export function parseEventProperties(properties: string | undefined): Record<string, unknown> | undefined {
if (!properties) return undefined;
try {
const parsed = JSON.parse(properties);
return parsed && typeof parsed === "object" && !Array.isArray(parsed) ? parsed : undefined;
} catch (error) {
return undefined;
}
}
// A tracking batch mixes every site's events, so one row ClickHouse can't parse
// must not take the rest down with it. Conflicting paths like {"a.b":1,"a":{"b":2}} are
// merged instead of rejected, and up to 100 other unparseable rows are skipped.
// A failure on more rows than that still throws.
export const EVENTS_INSERT_SETTINGS = {
type_json_skip_duplicated_paths: 1,
input_format_allow_errors_num: "100",
};