Skip to content

Open-source agentic AI Search Execution Platform. Track AI visibility, find opportunities, and let agents turn signals into prioritized actions and drafted fixes across ChatGPT, Claude, Gemini, Google AI Overviews, Google AI Mode, Perplexity, Grok, and Microsoft Copilot.

feat(traffic): take a journey's pageviews and conversions from the AI Traffic snippet (CE-017, part 1: server) (#1527)

Cihan55e6af354e84
diff --git a/server/src/lib/traffic-events.js b/server/src/lib/traffic-events.js
new file mode 100644
index 000000000..9e14ecff9
--- /dev/null
+++ b/server/src/lib/traffic-events.js
@@ -0,0 +1,171 @@
+/**
+ * A journey's events from the AI Traffic snippet (CE-017, 00286): the pages
+ * a visitor who came from an AI assistant goes on to (k: "pv") and the goal
+ * they reach (k: "cv": a signup, a demo form, an order…), each with the
+ * first AI visit it is credited to. Sent only by brands with conversions on.
+ *
+ * The body is untrusted, from any page: read with a schema, and a row is
+ * made only from what it allows. What can't be credited to an AI visit
+ * (a source not on the list, a first visit past the window or in the
+ * future) is dropped, not stored as unattributed.
+ */
+
+import { createHash } from 'node:crypto';
+import { z } from 'zod';
+import { classifyAiSource } from './ai-sources.js';
+
+/** Goals a conversion can be (00286), per site type in the Content Editor. */
+export const CONVERSION_GOALS = ['signup', 'demo', 'order', 'booking', 'newsletter', 'custom'];
+
+/** Where a conversion came from: the site's own call, a form it caught, Shopify's pixel. */
+export const CONVERSION_ORIGINS = ['call', 'form', 'shopify'];
+
+/** How long after the first AI visit a journey is still credited to it (first touch). */
+export const ATTRIBUTION_DAYS = 30;
+
+const DAY_MS = 24 * 60 * 60 * 1000;
+
+/** A browser's clock may run a little ahead of ours. */
+const CLOCK_SKEW_MS = DAY_MS;
+
+/**
+ * The most one visitor id stores (routes/traffic.js), by kind: pageviews in
+ * the attribution window, conversions in a day. Far above what a person
+ * does, so a journey's reads stay bounded whatever a sender does.
+ */
+export const VISITOR_LIMITS = {
+ page: { max: 300, windowMs: ATTRIBUTION_DAYS * DAY_MS },
+ conversion: { max: 10, windowMs: DAY_MS },
+};
+
+const visitorId = z.string().regex(/^[A-Za-z0-9_-]{16,64}$/);
+// Long ad and tracking tags are cut off with the query string (eventUrl).
+const pageUrl = z.string().min(1).max(16384);
+
+const eventBody = z.object({
+ k: z.enum(['pv', 'cv']),
+ id: visitorId,
+ u: pageUrl,
+ // The first AI visit: its source key, landing page and time (ms).
+ f: z.object({
+ s: z.string().min(1).max(255),
+ u: pageUrl,
+ at: z.number().int().positive(),
+ }),
+ g: z.enum(CONVERSION_GOALS).optional(),
+ n: z.string().max(200).optional(),
+ o: z.enum(CONVERSION_ORIGINS).optional(),
+ val: z.number().finite().min(0).max(1e9).optional(),
+ cur: z.string().max(10).optional(),
+ q: z.number().int().min(0).max(100000).optional(),
+ ref: z.string().max(200).optional(),
+});
+
+/** What a path segment holding an email is stored as. */
+export const EMAIL_SEGMENT = '[email]';
+
+/**
+ * Whether a path segment holds an email, raw or encoded (an encoded dot too):
+ * a name, an @, then a dot past it. An @ at the start is a handle ("/@acme",
+ * "/@acme.dev"). Read without decoding: an escape always stands for at least
+ * one character, and only %40 and %2E stand for an @ or a dot, so the answer
+ * is the decoded one's, and a malformed escape costs nothing (a decode would
+ * throw on each of an address's many segments).
+ */
+export function holdsEmail(segment) {
+ const text = segment.replace(/%40/gi, '@').replace(/%2e/gi, '.');
+ const at = text.indexOf('@', 1);
+ return at > 0 && text.indexOf('.', at + 2) > 0;
+}
+
+/**
+ * A page's address as stored: its origin and path, no query string or
+ * fragment (a thank-you page's ?email=… never gets in), and a path segment
+ * that holds an email, raw or encoded (/unsubscribe/ann%40example.com), as
+ * EMAIL_SEGMENT: the page still counts, the address stays out. A handle
+ * ("/@acme/post") stays. Null when it isn't an http(s) address.
+ */
+export function eventUrl(raw) {
+ try {
+ const url = new URL(raw);
+ if (url.protocol !== 'http:' && url.protocol !== 'https:') return null;
+ const path = url.pathname
+ .split('/')
+ .map((segment) => (holdsEmail(segment) ? EMAIL_SEGMENT : segment))
+ .join('/');
+ return `${url.origin}${path}`.slice(0, 2048);
+ } catch {
+ return null;
+ }
+}
+
+/**
+ * A form's or a call's own name ("pricing-demo"), or null: never what looks
+ * like a person's details (an email, a phone number), should a site put one
+ * there.
+ */
+export function eventLabel(value) {
+ const text = typeof value === 'string' ? value.trim().slice(0, 80) : '';
+ if (!text || !/^[\p{L}\p{N} _.:#/-]+$/u.test(text) || /\d{7,}/.test(text.replace(/[ .-]/g, ''))) {
+ return null;
+ }
+ return text;
+}
+
+/** An order's id at the shop ("1001", "#1001", "gid://shopify/Order/1"), or null. */
+export function orderRef(value) {
+ const text = typeof value === 'string' ? value.trim() : '';
+ return /^[A-Za-z0-9_#:/.-]{1,128}$/.test(text) ? text : null;
+}
+
+/**
+ * An order's id as stored: a hash of the brand and the shop's id. It is only
+ * ever compared (an order counts once, 00286's unique index), never read
+ * back, so the id itself is never kept. Null without a valid id.
+ */
+export function orderKey(brandId, value) {
+ const ref = orderRef(value);
+ return ref ? createHash('sha256').update(`${brandId}:${ref}`).digest('hex') : null;
+}
+
+/**
+ * The row an event makes for the brand, or null when it makes none: not a
+ * journey event, malformed, or not creditable to an AI visit (see above).
+ * A conversion without its origin is the site's own call.
+ */
+export function eventRow(brandId, body, now = Date.now()) {
+ const parsed = eventBody.safeParse(body);
+ if (!parsed.success) return null;
+ const event = parsed.data;
+ const firstSource = classifyAiSource(event.f.s);
+ if (!firstSource) return null;
+ if (event.f.at > now + CLOCK_SKEW_MS) return null;
+ if (event.f.at < now - ATTRIBUTION_DAYS * 24 * 60 * 60 * 1000) return null;
+ const url = eventUrl(event.u);
+ const firstUrl = eventUrl(event.f.u);
+ if (!url || !firstUrl) return null;
+ const row = {
+ brand_id: brandId,
+ visitor_id: event.id,
+ kind: event.k === 'cv' ? 'conversion' : 'page',
+ url,
+ first_source: firstSource,
+ first_url: firstUrl,
+ // Never after the event itself, with a browser clock running ahead.
+ first_at: new Date(Math.min(event.f.at, now)).toISOString(),
+ };
+ if (event.k === 'pv') return row;
+ if (!event.g) return null;
+ const currency = typeof event.cur === 'string' ? event.cur.trim().toUpperCase() : '';
+ return {
+ ...row,
+ goal: event.g,
+ label: eventLabel(event.n),
+ origin: event.o ?? 'call',
+ value: event.val ?? null,
+ // A currency only with an amount, and only a three-letter code.
+ currency: event.val !== undefined && /^[A-Z]{3}$/.test(currency) ? currency : null,
+ units: event.q ?? null,
+ order_ref: orderKey(brandId, event.ref),
+ };
+}
diff --git a/server/src/lib/traffic-events.test.js b/server/src/lib/traffic-events.test.js
new file mode 100644
index 000000000..fc5bc49fd
--- /dev/null
+++ b/server/src/lib/traffic-events.test.js
@@ -0,0 +1,244 @@
+import { createHash } from 'node:crypto';
+import { describe, expect, it } from 'vitest';
+import {
+ ATTRIBUTION_DAYS,
+ EMAIL_SEGMENT,
+ holdsEmail,
+ eventLabel,
+ eventRow,
+ eventUrl,
+ orderKey,
+ orderRef,
+} from './traffic-events.js';
+
+const NOW = Date.parse('2026-10-26T12:00:00Z');
+const DAY = 24 * 60 * 60 * 1000;
+const ID = 'Vq3xT0mZ8kLp2RwYbN5sQa';
+const first = {
+ s: 'chatgpt.com',
+ u: 'https://acme.example/blog/crm?utm_source=chatgpt.com',
+ at: NOW - DAY,
+};
+
+describe('holdsEmail', () => {
+ it('reads a path segment once, however long: any sender’s address is read with it', () => {
+ expect(holdsEmail('[email protected]')).toBe(true);
+ expect(holdsEmail('ann%40example.com')).toBe(true);
+ // A malformed escape elsewhere in it: the @ is still read.
+ expect(holdsEmail('ann%40example.com%E0')).toBe(true);
+ expect(holdsEmail('ann%40example%2Ecom%E0')).toBe(true);
+ expect(holdsEmail('@acme')).toBe(false);
+ expect(holdsEmail('pricing')).toBe(false);
+ const started = Date.now();
+ for (const segment of ['a'.repeat(16_384), 'a.'.repeat(8_192), `x${'%40a'.repeat(4_000)}`]) {
+ holdsEmail(segment);
+ }
+ expect(Date.now() - started).toBeLessThan(50);
+ });
+
+ it('reads a long address of many short segments with malformed escapes fast', () => {
+ const address = `https://acme.example/ann%40example%2Ecom/${'a%E0/'.repeat(3_200)}`;
+ const started = Date.now();
+ expect(eventUrl(address)).toMatch(/^https:\/\/acme\.example\/\[email\]\/a%E0\//);
+ // A decode per segment took about 11 ms here; reading the escapes as they are, under 1.
+ expect(Date.now() - started).toBeLessThan(8);
+ });
+});
+
+describe('eventUrl', () => {
+ it('keeps the origin and path, never the query string or the fragment', () => {
+ expect(eventUrl('https://acme.example/[email protected]#done')).toBe(
+ 'https://acme.example/thanks',
+ );
+ expect(eventUrl('javascript:alert(1)')).toBeNull();
+ expect(eventUrl('not a url')).toBeNull();
+ });
+
+ it('keeps no email in the path: its segment goes as a placeholder, raw or encoded', () => {
+ expect(eventUrl('https://acme.example/unsubscribe/[email protected]')).toBe(
+ `https://acme.example/unsubscribe/${EMAIL_SEGMENT}`,
+ );
+ expect(eventUrl('https://acme.example/account/ann%40example.com/orders')).toBe(
+ `https://acme.example/account/${EMAIL_SEGMENT}/orders`,
+ );
+ expect(eventUrl('https://acme.example/account/ANN%40EXAMPLE.COM')).toBe(
+ `https://acme.example/account/${EMAIL_SEGMENT}`,
+ );
+ // An encoded dot too.
+ expect(eventUrl('https://acme.example/account/ann%40example%2Ecom')).toBe(
+ `https://acme.example/account/${EMAIL_SEGMENT}`,
+ );
+ // A handle isn't an email: the page keeps its key.
+ expect(eventUrl('https://acme.example/@acme/post')).toBe('https://acme.example/@acme/post');
+ expect(eventUrl('https://acme.example/@acme.dev')).toBe('https://acme.example/@acme.dev');
+ // The first visit's landing page is read the same way.
+ expect(
+ eventRow(
+ 'b1',
+ {
+ k: 'pv',
+ id: ID,
+ u: 'https://acme.example/u/[email protected]',
+ f: { ...first, u: 'https://acme.example/invite/bob%40example.com' },
+ },
+ NOW,
+ ),
+ ).toMatchObject({
+ url: `https://acme.example/u/${EMAIL_SEGMENT}`,
+ first_url: `https://acme.example/invite/${EMAIL_SEGMENT}`,
+ });
+ });
+});
+
+describe('eventLabel and orderRef', () => {
+ it('keep a form’s or a call’s own name, never what looks like a person’s details', () => {
+ expect(eventLabel(' pricing-demo ')).toBe('pricing-demo');
+ expect(eventLabel('Sipariş #12')).toBe('Sipariş #12');
+ expect(eventLabel('[email protected]')).toBeNull();
+ expect(eventLabel('+90 555 123 45 67')).toBeNull();
+ expect(eventLabel('')).toBeNull();
+ });
+
+ it('keep an order’s id at the shop as it is, and nothing else', () => {
+ expect(orderRef('#1001')).toBe('#1001');
+ expect(orderRef('gid://shopify/Order/123456789')).toBe('gid://shopify/Order/123456789');
+ expect(orderRef('a b')).toBeNull();
+ expect(orderRef('x'.repeat(129))).toBeNull();
+ });
+
+ it('store an order’s id as a hash of the brand and the id: compared, never kept as it is', () => {
+ const key = orderKey('b1', ' #1001 ');
+ expect(key).toBe(createHash('sha256').update('b1:#1001').digest('hex'));
+ expect(key).toHaveLength(64);
+ // The same order counts once for its brand; another brand's is another.
+ expect(orderKey('b1', '#1001')).toBe(key);
+ expect(orderKey('b2', '#1001')).not.toBe(key);
+ expect(orderKey('b1', 'a b')).toBeNull();
+ expect(orderKey('b1', undefined)).toBeNull();
+ });
+});
+
+describe('eventRow', () => {
+ it('makes a journey pageview, credited to the first AI visit', () => {
+ expect(
+ eventRow(
+ 'b1',
+ { k: 'pv', id: ID, u: 'https://acme.example/pricing?plan=pro', f: first },
+ NOW,
+ ),
+ ).toEqual({
+ brand_id: 'b1',
+ visitor_id: ID,
+ kind: 'page',
+ url: 'https://acme.example/pricing',
+ first_source: 'chatgpt.com',
+ first_url: 'https://acme.example/blog/crm',
+ first_at: new Date(NOW - DAY).toISOString(),
+ });
+ });
+
+ it('makes a conversion with its goal, label, value and units; a call when it says no origin', () => {
+ const row = eventRow(
+ 'b1',
+ {
+ k: 'cv',
+ id: ID,
+ u: 'https://acme.example/thanks',
+ f: first,
+ g: 'order',
+ n: ' checkout ',
+ val: 129.5,
+ cur: 'eur',
+ q: 2,
+ ref: 'order-1001',
+ },
+ NOW,
+ );
+ expect(row).toMatchObject({
+ kind: 'conversion',
+ goal: 'order',
+ label: 'checkout',
+ origin: 'call',
+ value: 129.5,
+ currency: 'EUR',
+ units: 2,
+ order_ref: orderKey('b1', 'order-1001'),
+ });
+ expect(
+ eventRow(
+ 'b1',
+ { k: 'cv', id: ID, u: 'https://acme.example/', f: first, g: 'demo', o: 'form' },
+ NOW,
+ ),
+ ).toMatchObject({
+ goal: 'demo',
+ origin: 'form',
+ value: null,
+ currency: null,
+ units: null,
+ order_ref: null,
+ });
+ });
+
+ it('keeps a currency only with an amount, and only as a three-letter code', () => {
+ const cv = (extra) =>
+ eventRow(
+ 'b1',
+ { k: 'cv', id: ID, u: 'https://acme.example/', f: first, g: 'order', ...extra },
+ NOW,
+ );
+ expect(cv({ cur: 'USD' }).currency).toBeNull();
+ expect(cv({ val: 10, cur: 'EURO' }).currency).toBeNull();
+ expect(cv({ val: 10, cur: ' usd ' }).currency).toBe('USD');
+ });
+
+ it('never dates the first visit after the event, with a browser clock running ahead', () => {
+ expect(
+ eventRow(
+ 'b1',
+ { k: 'pv', id: ID, u: 'https://acme.example/', f: { ...first, at: NOW + 60 * 60 * 1000 } },
+ NOW,
+ ).first_at,
+ ).toBe(new Date(NOW).toISOString());
+ });
+
+ it('takes a page whose ad tags make its address long: they go with the query string', () => {
+ const long = `https://acme.example/pricing?${'utm_term=x&'.repeat(600)}`;
+ expect(eventRow('b1', { k: 'pv', id: ID, u: long, f: first }, NOW).url).toBe(
+ 'https://acme.example/pricing',
+ );
+ });
+
+ it('stores an older snippet’s host under the source list’s key', () => {
+ expect(
+ eventRow(
+ 'b1',
+ { k: 'pv', id: ID, u: 'https://acme.example/', f: { ...first, s: 'chat.openai.com' } },
+ NOW,
+ ).first_source,
+ ).toBe('chatgpt.com');
+ });
+
+ it('drops what it can’t credit to an AI visit, or that isn’t one', () => {
+ const pv = (extra) =>
+ eventRow('b1', { k: 'pv', id: ID, u: 'https://acme.example/', f: first, ...extra }, NOW);
+ // Not a source on the list.
+ expect(pv({ f: { ...first, s: 'google.com' } })).toBeNull();
+ // A first visit past the window, or in the future.
+ expect(pv({ f: { ...first, at: NOW - (ATTRIBUTION_DAYS + 1) * DAY } })).toBeNull();
+ expect(pv({ f: { ...first, at: NOW + 2 * DAY } })).toBeNull();
+ // A malformed visitor id, page, or a conversion without a goal.
+ expect(pv({ id: 'short' })).toBeNull();
+ expect(pv({ u: 'ftp://acme.example/' })).toBeNull();
+ expect(
+ eventRow('b1', { k: 'cv', id: ID, u: 'https://acme.example/', f: first }, NOW),
+ ).toBeNull();
+ expect(
+ eventRow('b1', { k: 'cv', id: ID, u: 'https://acme.example/', f: first, g: 'refund' }, NOW),
+ ).toBeNull();
+ // Not a journey event at all.
+ expect(
+ eventRow('b1', { k: 'nf', id: ID, u: 'https://acme.example/', f: first }, NOW),
+ ).toBeNull();
+ });
+});
diff --git a/server/src/middleware/rate-limiter.js b/server/src/middleware/rate-limiter.js
index 0f86de8c1..8296b6867 100644
--- a/server/src/middleware/rate-limiter.js
+++ b/server/src/middleware/rate-limiter.js
@@ -179,6 +179,40 @@ export const trafficLimiter = rateLimit({
handler: (_req, res) => res.status(204).end(),
});
+/** Whether a beacon is a journey's pageview or conversion (CE-017), as the route reads its body. */
+export function isJourneyBeacon(body) {
+ let read = body;
+ if (typeof read === 'string') {
+ try {
+ read = JSON.parse(read);
+ } catch {
+ return false;
+ }
+ }
+ return read?.k === 'pv' || read?.k === 'cv';
+}
+
+/** A journey event's bucket: its tracking code, whatever address sends it. */
+export function journeyEventKey(req) {
+ return 'journey:' + req.params.trackingCode;
+}
+
+// A brand's journey events (CE-017): 3,000 a minute per tracking code, beside
+// the 60 per address, so a sender cycling addresses can't fill the table. A
+// big shop's AI visitors send a few pageviews each, far below it. Visits
+// aren't counted here; past it, the events are dropped as the address
+// limiter drops beacons.
+export const journeyEventLimiter = rateLimit({
+ windowMs: 60 * 1000,
+ max: 3000,
+ keyGenerator: journeyEventKey,
+ skip: (req) => !isJourneyBeacon(req.body),
+ standardHeaders: false,
+ legacyHeaders: false,
+ message: '',
+ handler: (_req, res) => res.status(204).end(),
+});
+
/**
* AI-bot ingest key (AT-010): the connection, since a Cloudflare Worker's
* requests leave from addresses Cloudflare shares across its customers. Only
diff --git a/server/src/middleware/rate-limiter.test.js b/server/src/middleware/rate-limiter.test.js
index c4dae646f..648c506a9 100644
--- a/server/src/middleware/rate-limiter.test.js
+++ b/server/src/middleware/rate-limiter.test.js
@@ -1,5 +1,11 @@
import { describe, it, expect, beforeEach } from 'vitest';
-import { apiLimiterKey, botIngestKey, webhookCallbackKey } from './rate-limiter.js';
+import {
+ apiLimiterKey,
+ botIngestKey,
+ isJourneyBeacon,
+ journeyEventKey,
+ webhookCallbackKey,
+} from './rate-limiter.js';
import {
markBotSenderVerified,
markTokenVerified,
@@ -116,3 +122,23 @@ describe('rate-limiter – webhookCallbackKey', () => {
expect(webhookCallbackKey({ headers: {}, ip: '1.1.1.1' })).toBe('ip:1.1.1.1');
});
});
+
+describe('rate-limiter – journey events (CE-017)', () => {
+ it('buckets a journey event by its tracking code, whatever the address', () => {
+ expect(journeyEventKey({ params: { trackingCode: 'abc123' }, ip: '203.0.113.7' })).toBe(
+ 'journey:abc123',
+ );
+ expect(journeyEventKey({ params: { trackingCode: 'abc123' }, ip: '198.51.100.9' })).toBe(
+ 'journey:abc123',
+ );
+ });
+
+ it('counts only pageviews and conversions, as a string or a parsed body: never a visit', () => {
+ expect(isJourneyBeacon('{"k":"pv","u":"https://acme.example/"}')).toBe(true);
+ expect(isJourneyBeacon({ k: 'cv' })).toBe(true);
+ expect(isJourneyBeacon('{"u":"https://acme.example/","s":"chatgpt.com"}')).toBe(false);
+ expect(isJourneyBeacon({ k: 'nf' })).toBe(false);
+ expect(isJourneyBeacon('not json')).toBe(false);
+ expect(isJourneyBeacon(undefined)).toBe(false);
+ });
+});
diff --git a/server/src/routes/traffic-track.test.js b/server/src/routes/traffic-track.test.js
index 42253dd34..8796de660 100644
--- a/server/src/routes/traffic-track.test.js
+++ b/server/src/routes/traffic-track.test.js
@@ -16,26 +16,49 @@ let insertError = null;
const filters = [];
let latestPageview = null;
const updated = [];
+// The brand's conversions setting as its own read answers it (CE-017,
+// 00286): the visit's read asks for the id only, and never sees it.
+let conversionsRead = { data: { conversions_enabled: false }, error: null };
+let eventInsertError = null;
+// The brands reads' filters, to tell which brand the setting was read for.
+const brandFilters = [];
+// A visitor's earlier events as the limit read finds them, and its filters.
+let earlierEvents = 0;
+const eventFilters = [];
vi.mock('../config/supabase.js', () => ({
default: {
from: (table) => {
const filter = (op) => (col, val) => {
if (table === 'ai_traffic_logs') filters.push([op, col, val]);
+ if (table === 'brands') brandFilters.push([chain.columns, op, col, val]);
+ if (table === 'ai_traffic_events') eventFilters.push([op, col, val]);
return chain;
};
const chain = {
- select: () => chain,
+ select: (columns) => {
+ chain.columns = columns;
+ return chain;
+ },
eq: filter('eq'),
is: filter('is'),
gte: filter('gte'),
order: () => chain,
- limit: () => chain,
- single: async () => ({ data: { id: 'b1' }, error: null }),
+ limit: (n) =>
+ table === 'ai_traffic_events'
+ ? Promise.resolve({
+ data: Array.from({ length: Math.min(earlierEvents, n) }, (_, i) => ({ id: i })),
+ error: null,
+ })
+ : chain,
+ single: async () =>
+ chain.columns === 'conversions_enabled'
+ ? conversionsRead
+ : { data: { id: 'b1' }, error: null },
maybeSingle: async () => ({ data: latestPageview, error: null }),
insert: async (row) => {
inserted.push({ table, row });
- return { error: insertError };
+ return { error: table === 'ai_traffic_events' ? eventInsertError : insertError };
},
update: (fields) => ({
eq: async (col, val) => {
@@ -48,7 +71,10 @@ vi.mock('../config/supabase.js', () => ({
},
},
}));
-vi.mock('../middleware/rate-limiter.js', () => ({ trafficLimiter: (req, res, next) => next() }));
+vi.mock('../middleware/rate-limiter.js', () => ({
+ trafficLimiter: (req, res, next) => next(),
+ journeyEventLimiter: (req, res, next) => next(),
+}));
vi.mock('../lib/logger.js', () => {
const logger = { info: vi.fn(), warn: vi.fn(), error: vi.fn() };
return { default: logger, logger };
@@ -82,6 +108,11 @@ beforeEach(() => {
filters.length = 0;
latestPageview = null;
updated.length = 0;
+ conversionsRead = { data: { conversions_enabled: false }, error: null };
+ eventInsertError = null;
+ brandFilters.length = 0;
+ earlierEvents = 0;
+ eventFilters.length = 0;
});
const beacon = (body = {}) =>
@@ -338,3 +369,115 @@ describe('POST /track/:trackingCode', () => {
expect(inserted).toEqual([]);
});
});
+
+describe('POST /track/:trackingCode: a journey’s events (CE-017)', () => {
+ const ID = 'Vq3xT0mZ8kLp2RwYbN5sQa';
+ const event = (body) =>
+ fetch(baseUrl, {
+ method: 'POST',
+ headers: { 'Content-Type': 'text/plain;charset=UTF-8', 'X-Forwarded-For': '203.0.113.7' },
+ body: JSON.stringify({
+ id: ID,
+ u: 'https://acme.example/[email protected]',
+ f: { s: 'chatgpt.com', u: 'https://acme.example/blog/crm', at: Date.now() - 60_000 },
+ ...body,
+ }),
+ });
+
+ it('stores a pageview and a conversion for a brand with conversions on, never as a visit', async () => {
+ conversionsRead = { data: { conversions_enabled: true }, error: null };
+ expect((await event({ k: 'pv' })).status).toBe(204);
+ expect((await event({ k: 'cv', g: 'demo', o: 'form', n: 'pricing-demo' })).status).toBe(204);
+ expect(inserted.map(({ table }) => table)).toEqual(['ai_traffic_events', 'ai_traffic_events']);
+ expect(inserted[0].row).toMatchObject({
+ brand_id: 'b1',
+ visitor_id: ID,
+ kind: 'page',
+ // No query string: the email never gets in.
+ url: 'https://acme.example/thanks',
+ first_source: 'chatgpt.com',
+ });
+ expect(inserted[1].row).toMatchObject({
+ kind: 'conversion',
+ goal: 'demo',
+ origin: 'form',
+ label: 'pricing-demo',
+ });
+ // Neither is an AI landing: no visit lookup, no visitor key.
+ expect(findVisitOf).not.toHaveBeenCalled();
+ expect(visitorKey).not.toHaveBeenCalled();
+ // The setting is read for this brand, by its own read.
+ expect(brandFilters).toContainEqual(['conversions_enabled', 'eq', 'id', 'b1']);
+ });
+
+ it('fails only the events when the setting can’t be read (a server before 00286), never the visits', async () => {
+ conversionsRead = {
+ data: null,
+ error: { message: 'column brands.conversions_enabled does not exist' },
+ };
+ const { default: logger } = await import('../lib/logger.js');
+ vi.mocked(logger.error).mockClear();
+ expect((await event({ k: 'cv', g: 'signup' })).status).toBe(204);
+ expect(inserted).toEqual([]);
+ expect(logger.error).toHaveBeenCalledTimes(1);
+ // A visit reads the brand's id only, and is stored as ever.
+ expect((await beacon()).status).toBe(204);
+ expect(inserted.map(({ table }) => table)).toEqual(['ai_traffic_logs']);
+ });
+
+ it('stores nothing while the brand has conversions off', async () => {
+ expect((await event({ k: 'cv', g: 'signup' })).status).toBe(204);
+ expect(inserted).toEqual([]);
+ });
+
+ it('stores nothing it can’t credit to an AI visit, and doesn’t read the brand for it', async () => {
+ conversionsRead = { data: { conversions_enabled: true }, error: null };
+ await event({ k: 'pv', f: { s: 'google.com', u: 'https://acme.example/', at: Date.now() } });
+ await event({ k: 'cv' });
+ expect(inserted).toEqual([]);
+ expect(brandFilters.filter(([columns]) => columns === 'conversions_enabled')).toEqual([]);
+ });
+
+ it('stores no more for a visitor id past its limits: pageviews in the window, conversions a day', async () => {
+ conversionsRead = { data: { conversions_enabled: true }, error: null };
+ earlierEvents = 299;
+ await event({ k: 'pv' });
+ expect(inserted).toHaveLength(1);
+ // Read for this brand, visitor and kind, over the attribution window.
+ expect(eventFilters).toEqual(
+ expect.arrayContaining([
+ ['eq', 'brand_id', 'b1'],
+ ['eq', 'visitor_id', ID],
+ ['eq', 'kind', 'page'],
+ ]),
+ );
+ const since = Date.parse(eventFilters.find(([op]) => op === 'gte')[2]);
+ expect(Date.now() - since).toBeGreaterThan(29 * 24 * 60 * 60 * 1000);
+ earlierEvents = 300;
+ await event({ k: 'pv' });
+ expect(inserted).toHaveLength(1);
+
+ eventFilters.length = 0;
+ earlierEvents = 10;
+ await event({ k: 'cv', g: 'signup' });
+ expect(inserted).toHaveLength(1);
+ expect(eventFilters).toContainEqual(['eq', 'kind', 'conversion']);
+ earlierEvents = 9;
+ await event({ k: 'cv', g: 'signup' });
+ expect(inserted).toHaveLength(2);
+ });
+
+ it('counts an order sent again once, without logging it as a failure', async () => {
+ conversionsRead = { data: { conversions_enabled: true }, error: null };
+ const { default: logger } = await import('../lib/logger.js');
+ vi.mocked(logger.error).mockClear();
+ eventInsertError = { code: '23505', message: 'duplicate key value' };
+ expect(
+ (await event({ k: 'cv', g: 'order', o: 'shopify', ref: '1001', val: 59, cur: 'USD' })).status,
+ ).toBe(204);
+ expect(logger.error).not.toHaveBeenCalled();
+ eventInsertError = { code: '42501', message: 'permission denied' };
+ await event({ k: 'cv', g: 'order', o: 'shopify', ref: '1002' });
+ expect(logger.error).toHaveBeenCalledTimes(1);
+ });
+});
diff --git a/server/src/routes/traffic.js b/server/src/routes/traffic.js
index 0cb8798eb..291e035a6 100644
--- a/server/src/routes/traffic.js
+++ b/server/src/routes/traffic.js
@@ -1,6 +1,6 @@
import express, { Router } from 'express';
import supabaseAdmin from '../config/supabase.js';
-import { trafficLimiter } from '../middleware/rate-limiter.js';
+import { journeyEventLimiter, trafficLimiter } from '../middleware/rate-limiter.js';
import logger from '../lib/logger.js';
import { findVisitOf } from '../lib/traffic-visits.js';
import { visitorKey as anonymousVisitorKey } from '../lib/visitor-key.js';
@@ -11,6 +11,7 @@ import {
landingMedium,
} from '../lib/ai-sources.js';
import { buildTrackingScript } from '../lib/tracking-script.js';
+import { VISITOR_LIMITS, eventRow } from '../lib/traffic-events.js';
const router = Router();
@@ -21,6 +22,8 @@ const router = Router();
// API callers) and read the same way: these routes are mounted before the
// server's JSON parser, so a body is read here, up to 64 KB, whatever its type.
const beaconBody = express.text({ type: ['text/plain', 'application/json'], limit: '64kb' });
+// The body read, then a journey event's own limit by tracking code (CE-017).
+const trackBody = [beaconBody, journeyEventLimiter];
let cachedScript = null;
let cachedEtag = null;
@@ -72,7 +75,7 @@ router.options('/track/:trackingCode', (req, res) => {
/**
* POST /track/:trackingCode — receive beacon data from t.js
*/
-router.post('/track/:trackingCode', trafficLimiter, beaconBody, async (req, res) => {
+router.post('/track/:trackingCode', trafficLimiter, trackBody, async (req, res) => {
res.set('Access-Control-Allow-Origin', '*');
try {
const { trackingCode } = req.params;
@@ -105,6 +108,13 @@ router.post('/track/:trackingCode', trafficLimiter, beaconBody, async (req, res)
return res.status(400).json({ ok: false });
}
+ // A journey's pageview or conversion (CE-017), from a snippet installed
+ // with conversions on: stored only while the brand has them on.
+ if (body.k === 'pv' || body.k === 'cv') {
+ await storeEvent(brand.id, body);
+ return res.status(204).end();
+ }
+
const ip =
req.headers['cf-connecting-ip'] ||
req.headers['x-forwarded-for']?.split(',')[0]?.trim() ||
@@ -214,6 +224,46 @@ router.post('/track/:trackingCode', trafficLimiter, beaconBody, async (req, res)
}
});
+/**
+ * Stores a journey event (traffic-events.js) for a brand with conversions
+ * on. Read here, not with the brand above: a server deployed before 00286
+ * then fails only these, never the visits. A visitor id past its limits
+ * (VISITOR_LIMITS) stores no more, so whatever a sender does, a journey's
+ * reads (00287, 00288) stay bounded. Best effort: an order sent again
+ * counts once (its unique ref), anything else that fails is logged.
+ */
+async function storeEvent(brandId, body) {
+ // What can't be stored needs no look at the brand.
+ const row = eventRow(brandId, body);
+ if (!row) return;
+ try {
+ const { data: brand, error } = await supabaseAdmin
+ .from('brands')
+ .select('conversions_enabled')
+ .eq('id', brandId)
+ .single();
+ if (error) throw new Error(error.message);
+ if (!brand?.conversions_enabled) return;
+ // Its pageviews in the attribution window, or its conversions in a day,
+ // read up to the limit through the journey index.
+ const limit = VISITOR_LIMITS[row.kind];
+ const { data: earlier, error: countError } = await supabaseAdmin
+ .from('ai_traffic_events')
+ .select('id')
+ .eq('brand_id', brandId)
+ .eq('visitor_id', row.visitor_id)
+ .eq('kind', row.kind)
+ .gte('occurred_at', new Date(Date.now() - limit.windowMs).toISOString())
+ .limit(limit.max);
+ if (countError) throw new Error(countError.message);
+ if ((earlier?.length ?? 0) >= limit.max) return;
+ const { error: insertError } = await supabaseAdmin.from('ai_traffic_events').insert(row);
+ if (insertError && insertError.code !== '23505') throw new Error(insertError.message);
+ } catch (err) {
+ logger.error({ err }, 'traffic event store failed');
+ }
+}
+
/** How long after a visit its pageview may still be marked as a soft 404. */
const NOT_FOUND_MARK_WINDOW_MS = 10 * 60 * 1000;
diff --git a/supabase/migrations/00286_ai_traffic_conversions.sql b/supabase/migrations/00286_ai_traffic_conversions.sql
new file mode 100644
index 000000000..4cf35a354
--- /dev/null
+++ b/supabase/migrations/00286_ai_traffic_conversions.sql
@@ -0,0 +1,114 @@
+-- ============================================================================
+-- 00286: conversions from AI visits, as the AI Traffic snippet sees them (CE-017)
+-- ============================================================================
+-- The snippet counts AI-referred landings (ai_traffic_logs). With conversions
+-- on, it also follows a visitor who came from an AI assistant through the
+-- site to a goal (a signup, a demo form, an order…), so the Content Editor
+-- can show what a page earns from AI visits, and the journeys that got there.
+--
+-- * Off by default, per brand (`conversions_enabled`). A brand that leaves
+-- it off sends nothing new: the snippet keeps nothing in the browser, as
+-- it always has.
+-- * `conversions_consent`: 'wait' (default) keeps the visitor's id only once
+-- the site says the visitor agreed (its consent banner, or Google Consent
+-- Mode); 'always' keeps it at once, for sites whose own banner covers it.
+-- The snippet reads it from its install tag; the server keeps it for the
+-- Settings page.
+-- * `ai_traffic_events`: a journey's pageviews and its conversions. One
+-- visitor id per browser, random and first-party: never the IP address,
+-- never a name or an email. Each row carries the first AI visit it is
+-- credited to (first touch, within 30 days), so a journey reads without
+-- joining the landing logs.
+-- * `url` is the page without its query string or fragment: a thank-you
+-- page's ?email=… never gets in, nor a path segment holding an email
+-- (stored as "[email]").
+-- * `order_ref`: a hash of the brand and an order's own id at the shop
+-- (Shopify's pixel, a site's call), once per visitor, so an order sent
+-- twice counts once; only ever compared, so the id itself isn't kept.
+-- Per visitor, not per brand: the tracking code is public, and an order
+-- id sent first by anyone else must not shut the real one out.
+-- * The server stores at most a few hundred pageviews per visitor id in
+-- the window and a few conversions a day, so a journey's reads stay
+-- bounded (routes/traffic.js).
+-- * `page_key` and `first_key`: the pages' keys, made once on insert, for
+-- the journeys' reads.
+-- * Members read their brands' events; only the server writes them.
+
+alter table public.brands
+ add column if not exists conversions_enabled boolean not null default false,
+ add column if not exists conversions_consent text not null default 'wait'
+ check (conversions_consent in ('wait', 'always'));
+
+comment on column public.brands.conversions_enabled is
+ 'CE-017 (00286): the AI Traffic snippet follows AI visitors to conversions (ai_traffic_events). Off by default.';
+comment on column public.brands.conversions_consent is
+ 'CE-017 (00286): ''wait'' keeps the snippet''s visitor id only after the site''s consent, ''always'' at once.';
+
+create table if not exists public.ai_traffic_events (
+ id uuid primary key default gen_random_uuid(),
+ brand_id uuid not null references public.brands(id) on delete cascade,
+ visitor_id text not null check (visitor_id ~ '^[A-Za-z0-9_-]{16,64}$'),
+ kind text not null check (kind in ('page', 'conversion')),
+ url text not null check (char_length(url) between 1 and 2048),
+ goal text check (goal in ('signup', 'demo', 'order', 'booking', 'newsletter', 'custom')),
+ -- The form's or the call's own name ("pricing-demo"), never what was typed in it.
+ label text check (label is null or char_length(label) between 1 and 80),
+ origin text check (origin in ('call', 'form', 'shopify')),
+ value numeric(14, 2) check (value is null or value >= 0),
+ currency text check (currency is null or currency ~ '^[A-Z]{3}$'),
+ units integer check (units is null or units between 0 and 100000),
+ order_ref text check (order_ref is null or char_length(order_ref) between 1 and 128),
+ first_source text not null check (char_length(first_source) between 1 and 64),
+ first_url text not null check (char_length(first_url) between 1 and 2048),
+ first_at timestamptz not null,
+ occurred_at timestamptz not null default now(),
+ -- The pages' keys as journeys read them (lower(normalize_page_path), as in
+ -- 00209 / 00210): made once when the row is stored, so a journey's reads
+ -- (00287, 00288) compare keys rather than work each address out again.
+ page_key text generated always as (lower(public.normalize_page_path(url))) stored,
+ first_key text generated always as (lower(public.normalize_page_path(first_url))) stored,
+ -- A conversion says what it was and where it came from; a pageview neither.
+ constraint ai_traffic_events_kind check (
+ (kind = 'conversion' and goal is not null and origin is not null)
+ or (kind = 'page' and goal is null and origin is null and order_ref is null)
+ )
+);
+
+-- A database that ran this file before the keys were added gets them, and
+-- loses the brand-time index nothing reads.
+alter table public.ai_traffic_events
+ add column if not exists page_key text
+ generated always as (lower(public.normalize_page_path(url))) stored,
+ add column if not exists first_key text
+ generated always as (lower(public.normalize_page_path(first_url))) stored;
+drop index if exists public.ai_traffic_events_brand_time;
+
+-- An order counts once however often its visitor's browser sends it.
+create unique index if not exists ai_traffic_events_order
+ on public.ai_traffic_events (brand_id, visitor_id, order_ref)
+ where order_ref is not null;
+
+create index if not exists ai_traffic_events_journey
+ on public.ai_traffic_events (brand_id, visitor_id, occurred_at);
+
+alter table public.ai_traffic_events enable row level security;
+revoke all on table public.ai_traffic_events from anon, authenticated;
+grant select on table public.ai_traffic_events to authenticated;
+grant all on table public.ai_traffic_events to service_role;
+
+drop policy if exists "ai_traffic_events: member select" on public.ai_traffic_events;
+create policy "ai_traffic_events: member select"
+ on public.ai_traffic_events
+ for select
+ using (
+ brand_id in (
+ select b.id from public.brands b
+ join public.profiles p on p.organization_id = b.organization_id
+ where p.id = auth.uid()
+ )
+ );
+
+comment on table public.ai_traffic_events is
+ 'CE-017 (00286): journeys of visitors who came from an AI assistant, as the AI Traffic snippet sends them with conversions on: pageviews and conversions (goal, value, units; an order once by the hash of its id, order_ref), each credited to the first AI visit (first_source, first_url, first_at). A random first-party visitor id; no IP, name or email. Members read; the server writes.';
+
+notify pgrst, 'reload schema';
diff --git a/supabase/schema.sql b/supabase/schema.sql
index e8ac349c2..7ce1d74f4 100644
--- a/supabase/schema.sql
+++ b/supabase/schema.sql
@@ -40216,6 +40216,124 @@ comment on column public.content_draft_suggestions.kept_chars is
comment on column public.content_draft_suggestions.after_chars is
'The characters a page rewrite came to (00285).';
+-- ─────────────────────────────────────────────────────────────────────────
+-- migrations/00286_ai_traffic_conversions.sql
+-- ─────────────────────────────────────────────────────────────────────────
+-- ============================================================================
+-- 00286: conversions from AI visits, as the AI Traffic snippet sees them (CE-017)
+-- ============================================================================
+-- The snippet counts AI-referred landings (ai_traffic_logs). With conversions
+-- on, it also follows a visitor who came from an AI assistant through the
+-- site to a goal (a signup, a demo form, an order…), so the Content Editor
+-- can show what a page earns from AI visits, and the journeys that got there.
+--
+-- * Off by default, per brand (`conversions_enabled`). A brand that leaves
+-- it off sends nothing new: the snippet keeps nothing in the browser, as
+-- it always has.
+-- * `conversions_consent`: 'wait' (default) keeps the visitor's id only once
+-- the site says the visitor agreed (its consent banner, or Google Consent
+-- Mode); 'always' keeps it at once, for sites whose own banner covers it.
+-- The snippet reads it from its install tag; the server keeps it for the
+-- Settings page.
+-- * `ai_traffic_events`: a journey's pageviews and its conversions. One
+-- visitor id per browser, random and first-party: never the IP address,
+-- never a name or an email. Each row carries the first AI visit it is
+-- credited to (first touch, within 30 days), so a journey reads without
+-- joining the landing logs.
+-- * `url` is the page without its query string or fragment: a thank-you
+-- page's ?email=… never gets in, nor a path segment holding an email
+-- (stored as "[email]").
+-- * `order_ref`: a hash of the brand and an order's own id at the shop
+-- (Shopify's pixel, a site's call), once per visitor, so an order sent
+-- twice counts once; only ever compared, so the id itself isn't kept.
+-- Per visitor, not per brand: the tracking code is public, and an order
+-- id sent first by anyone else must not shut the real one out.
+-- * The server stores at most a few hundred pageviews per visitor id in
+-- the window and a few conversions a day, so a journey's reads stay
+-- bounded (routes/traffic.js).
+-- * `page_key` and `first_key`: the pages' keys, made once on insert, for
+-- the journeys' reads.
+-- * Members read their brands' events; only the server writes them.
+
+alter table public.brands
+ add column if not exists conversions_enabled boolean not null default false,
+ add column if not exists conversions_consent text not null default 'wait'
+ check (conversions_consent in ('wait', 'always'));
+
+comment on column public.brands.conversions_enabled is
+ 'CE-017 (00286): the AI Traffic snippet follows AI visitors to conversions (ai_traffic_events). Off by default.';
+comment on column public.brands.conversions_consent is
+ 'CE-017 (00286): ''wait'' keeps the snippet''s visitor id only after the site''s consent, ''always'' at once.';
+
+create table if not exists public.ai_traffic_events (
+ id uuid primary key default gen_random_uuid(),
+ brand_id uuid not null references public.brands(id) on delete cascade,
+ visitor_id text not null check (visitor_id ~ '^[A-Za-z0-9_-]{16,64}$'),
+ kind text not null check (kind in ('page', 'conversion')),
+ url text not null check (char_length(url) between 1 and 2048),
+ goal text check (goal in ('signup', 'demo', 'order', 'booking', 'newsletter', 'custom')),
+ -- The form's or the call's own name ("pricing-demo"), never what was typed in it.
+ label text check (label is null or char_length(label) between 1 and 80),
+ origin text check (origin in ('call', 'form', 'shopify')),
+ value numeric(14, 2) check (value is null or value >= 0),
+ currency text check (currency is null or currency ~ '^[A-Z]{3}$'),
+ units integer check (units is null or units between 0 and 100000),
+ order_ref text check (order_ref is null or char_length(order_ref) between 1 and 128),
+ first_source text not null check (char_length(first_source) between 1 and 64),
+ first_url text not null check (char_length(first_url) between 1 and 2048),
+ first_at timestamptz not null,
+ occurred_at timestamptz not null default now(),
+ -- The pages' keys as journeys read them (lower(normalize_page_path), as in
+ -- 00209 / 00210): made once when the row is stored, so a journey's reads
+ -- (00287, 00288) compare keys rather than work each address out again.
+ page_key text generated always as (lower(public.normalize_page_path(url))) stored,
+ first_key text generated always as (lower(public.normalize_page_path(first_url))) stored,
+ -- A conversion says what it was and where it came from; a pageview neither.
+ constraint ai_traffic_events_kind check (
+ (kind = 'conversion' and goal is not null and origin is not null)
+ or (kind = 'page' and goal is null and origin is null and order_ref is null)
+ )
+);
+
+-- A database that ran this file before the keys were added gets them, and
+-- loses the brand-time index nothing reads.
+alter table public.ai_traffic_events
+ add column if not exists page_key text
+ generated always as (lower(public.normalize_page_path(url))) stored,
+ add column if not exists first_key text
+ generated always as (lower(public.normalize_page_path(first_url))) stored;
+drop index if exists public.ai_traffic_events_brand_time;
+
+-- An order counts once however often its visitor's browser sends it.
+create unique index if not exists ai_traffic_events_order
+ on public.ai_traffic_events (brand_id, visitor_id, order_ref)
+ where order_ref is not null;
+
+create index if not exists ai_traffic_events_journey
+ on public.ai_traffic_events (brand_id, visitor_id, occurred_at);
+
+alter table public.ai_traffic_events enable row level security;
+revoke all on table public.ai_traffic_events from anon, authenticated;
+grant select on table public.ai_traffic_events to authenticated;
+grant all on table public.ai_traffic_events to service_role;
+
+drop policy if exists "ai_traffic_events: member select" on public.ai_traffic_events;
+create policy "ai_traffic_events: member select"
+ on public.ai_traffic_events
+ for select
+ using (
+ brand_id in (
+ select b.id from public.brands b
+ join public.profiles p on p.organization_id = b.organization_id
+ where p.id = auth.uid()
+ )
+ );
+
+comment on table public.ai_traffic_events is
+ 'CE-017 (00286): journeys of visitors who came from an AI assistant, as the AI Traffic snippet sends them with conversions on: pageviews and conversions (goal, value, units; an order once by the hash of its id, order_ref), each credited to the first AI visit (first_source, first_url, first_at). A random first-party visitor id; no IP, name or email. Members read; the server writes.';
+
+notify pgrst, 'reload schema';
+
-- ─────────────────────────────────────────────────────────────────────────
-- migrations/00289_content_draft_restored_at.sql
-- ─────────────────────────────────────────────────────────────────────────