Comp AI CRM is an open source, CRM designed for AI agents. Agentic-first CRM.
TypeScript
import type { Db } from "@crm/db";
import { classifyTouch, type RawTouch, type Touch } from "@crm/db/attribution";
import {
dedupeKey,
EVENTS_PER_MINUTE,
hostAllowed,
MAX_EVENTS_PER_BATCH,
matchedHost,
normalizePath,
originAllowed,
rateWindowKey,
stripQuery,
type TrackingConfig,
} from "@crm/db/tracking";
import { Injectable, Logger } from "@nestjs/common";
import { z } from "zod";
import { normalizeEmail } from "../crm/values";
import { InjectDatabase } from "../database/database.constants";
import { TrackingConfigService } from "./tracking-config.service";
import { TrackingCounterService } from "./tracking-counter.service";
import { TrackingFilingService } from "./tracking-filing.service";
const MAX_LABEL = 80;
const MAX_PATH = 512;
const MAX_HOST = 253;
const KEPT = new Set(["page_view", "click", "form_submit"]);
const BOT = /bot|crawler|spider|crawling|headlesschrome|lighthouse|preview/i;
const SENSITIVE = /pass|secret|token|card|cvv|cvc|ssn|iban|routing/i;
const CARD = /^[0-9 -]{12,25}$/;
const ADDRESS = /^[^\s@]+@[^\s@.]+\.[^\s@]+$/;
const fieldText = z.string().nullable().catch(null);
type FormFields = Record<string, string>;
type StoredTouch = {
source: string;
medium: string;
campaign: string | null;
term: string | null;
content: string | null;
referrer: string | null;
landing: string | null;
at: string;
};
export interface IncomingEvent {
type: string;
host: string;
path: string;
referrer?: string;
label?: string;
at?: number;
fields?: FormFields;
touch?: RawTouch;
firstTouch?: RawTouch;
}
export interface IncomingBatch {
siteId: string;
visitorId: string;
events: IncomingEvent[];
}
interface AcceptedEvent {
event: IncomingEvent;
host: string;
}
@Injectable()
export class TrackingIngestService {
private readonly logger = new Logger(TrackingIngestService.name);
constructor(
@InjectDatabase() private readonly db: Db,
private readonly config: TrackingConfigService,
private readonly counters: TrackingCounterService,
private readonly filing: TrackingFilingService,
) {}
async accept(
batch: IncomingBatch,
request: { origin: string | null; userAgent: string | null },
): Promise<void> {
if (request.userAgent && BOT.test(request.userAgent)) return;
const compiled = await this.config.forSite(batch.siteId);
if (!compiled) return;
if (!originAllowed(request.origin, compiled.config)) return;
const visitorId = sanitizeId(batch.visitorId);
if (!visitorId) return;
const events = batch.events.slice(0, MAX_EVENTS_PER_BATCH);
if (scripted(events)) return;
const accepted = events.flatMap<AcceptedEvent>((event) => {
if (!KEPT.has(event.type)) return [];
const host = event.host?.toLowerCase().trim().slice(0, MAX_HOST);
return host && hostAllowed(host, compiled.config)
? [{ event, host }]
: [];
});
if (accepted.length === 0) return;
if (!(await this.withinRate(accepted.length))) return;
const pageViews = accepted.filter(
({ event }) => event.type === "page_view" || event.type === "click",
);
const forms = accepted.filter(({ event }) => event.type === "form_submit");
if (pageViews.length > 0) {
await this.events(visitorId, pageViews, compiled.config);
}
for (const form of forms) {
await this.submission(visitorId, form);
}
}
private async events(
visitorId: string,
accepted: AcceptedEvent[],
config: TrackingConfig,
): Promise<void> {
const rows = accepted.map(({ event, host }) => {
const touch =
event.type === "page_view" && event.touch
? classifyTouch(arriving(event.touch))
: null;
const referrer = stripQuery(event.referrer);
return {
visitorId,
type: event.type,
host,
path: trim(normalizePath(event.path), MAX_PATH),
referrer: referrer ? trim(referrer, MAX_PATH) : null,
label: event.label ? trim(event.label, MAX_LABEL) : null,
source: touch?.source ?? null,
medium: touch?.medium ?? null,
campaign: touch?.campaign ?? null,
occurredAt: occurredAt(event.at),
};
});
await this.db.trackedEvent.createMany({ data: rows });
await this.countViews(rows, config);
}
private async countViews(
rows: { host: string; type: string }[],
config: TrackingConfig,
): Promise<void> {
const tallies = new Map<string, number>();
for (const row of rows) {
const entry = matchedHost(row.host, config);
if (!entry) continue;
const views = tallies.get(entry.host) ?? 0;
tallies.set(entry.host, views + (row.type === "page_view" ? 1 : 0));
}
const lastSeenAt = new Date();
await Promise.all(
[...tallies].map(([host, views]) =>
this.db.trackedDomain.updateMany({
where: { host },
data: { pageViews: { increment: views }, lastSeenAt },
}),
),
);
}
private async submission(
visitorId: string,
{ event, host }: AcceptedEvent,
): Promise<void> {
const fields = clean(event.fields ?? {});
const email = emailFrom(fields);
const path = trim(normalizePath(event.path), MAX_PATH);
const at = occurredAt(event.at);
const lastTouch = classifyTouch(arriving(event.touch ?? {}), at);
const firstTouch = event.firstTouch
? classifyTouch(arriving(event.firstTouch), at)
: lastTouch;
const key = dedupeKey({ host, path, email, at });
const created = await this.db.formSubmission.createMany({
data: [
{
visitorId,
host,
path,
email,
fields,
firstTouch: stored(firstTouch),
lastTouch: stored(lastTouch),
dedupeKey: key,
},
],
skipDuplicates: true,
});
const submission = await this.db.formSubmission.findUnique({
where: { dedupeKey: key },
select: { id: true, filedAt: true, skipReason: true },
});
if (!submission) return;
if (created.count === 0 && !unfiled(submission)) return;
const outcome = await this.filing.file({
id: submission.id,
email,
host,
visitorId,
name: nameFrom(fields),
firstTouch,
lastTouch,
});
if (!outcome.filed) {
this.logger.log({
message: "Form submission stored but not filed",
host,
reason: outcome.reason,
});
}
}
private async withinRate(events: number): Promise<boolean> {
return this.counters.take(rateWindowKey(), EVENTS_PER_MINUTE, events);
}
}
function unfiled(submission: {
filedAt: Date | null;
skipReason: string | null;
}): boolean {
return submission.filedAt === null && submission.skipReason === null;
}
function arriving(touch: RawTouch): RawTouch {
return {
...touch,
referrer: stripQuery(touch.referrer) ?? undefined,
landing: stripQuery(touch.landing) ?? undefined,
};
}
function stored(touch: Touch): StoredTouch {
return {
source: touch.source,
medium: touch.medium,
campaign: touch.campaign,
term: touch.term,
content: touch.content,
referrer: touch.referrer,
landing: touch.landing,
at: touch.at.toISOString(),
};
}
function scripted(events: IncomingEvent[]): boolean {
if (events.length < 3) return false;
const stamps = events.flatMap((event) =>
event.at !== undefined && Number.isFinite(event.at) ? [event.at] : [],
);
if (stamps.length !== events.length) return false;
return new Set(stamps).size === 1;
}
function occurredAt(at: number | undefined): Date {
const now = Date.now();
if (at === undefined || !Number.isFinite(at)) return new Date(now);
const bounded = Math.min(Math.max(at, now - 86_400_000), now);
return new Date(bounded);
}
function trim(value: string, max: number): string {
return value.length > max ? value.slice(0, max) : value;
}
function sanitizeId(value: string | undefined): string | null {
const trimmed = fieldText.parse(value)?.trim() ?? "";
return /^[a-zA-Z0-9_-]{8,64}$/.test(trimmed) ? trimmed : null;
}
function clean(fields: FormFields): FormFields {
const kept: FormFields = {};
for (const [key, value] of Object.entries(fields).slice(0, 40)) {
const text = fieldText.parse(value);
if (text === null) continue;
if (SENSITIVE.test(key)) continue;
if (CARD.test(text.trim())) continue;
kept[trim(key, 64)] = trim(text, 512);
}
return kept;
}
function emailFrom(fields: FormFields): string | null {
for (const [key, value] of Object.entries(fields)) {
if (!/mail/i.test(key)) continue;
const email = address(value);
if (email) return email;
}
for (const value of Object.values(fields)) {
const email = address(value);
if (email) return email;
}
return null;
}
function address(value: string): string | null {
const email = normalizeEmail(value);
return email && ADDRESS.test(email) ? email : null;
}
function nameFrom(fields: FormFields): string | null {
const first = pick(fields, /^(first[\s_-]?name|fname|given)/i);
const last = pick(fields, /^(last[\s_-]?name|lname|surname|family)/i);
if (first) return last ? `${first} ${last}` : first;
return pick(fields, /^(full[\s_-]?name|name)$/i) ?? pick(fields, /name/i);
}
function pick(fields: FormFields, pattern: RegExp): string | null {
for (const [key, value] of Object.entries(fields)) {
if (pattern.test(key) && value.trim()) return value.trim();
}
return null;
}