OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.
fix: ai referrers on seo page (percentage bug)
Carl-Gerhard Lindesvärdb104e6838d76
diff --git a/packages/trpc/src/routers/gsc.ts b/packages/trpc/src/routers/gsc.tsindex ba3c182fb..3af5248ec 100644--- a/packages/trpc/src/routers/gsc.ts+++ b/packages/trpc/src/routers/gsc.ts@@ -47,6 +47,80 @@ async function resolveDates(};}+/**+ * ClickHouse stores the same AI referrer under several spellings: the parsed+ * display name ('ChatGPT', 'Google Gemini'), the bare host ('chatgpt.com') and+ * the full origin ('https://kagi.com'). Normalize before matching so every+ * spelling lands on the same engine.+ */+const NORMALIZED_REFERRER_NAME =+ "lower(regexp_replace(referrer_name, '^https?://(www[.])?', ''))";++const AI_REFERRERS = [+ {+ canonical: 'chatgpt.com',+ aliases: ['chatgpt', 'chatgpt.com', 'chat.openai.com', 'openai', 'openai.com'],+ },+ {+ canonical: 'claude.ai',+ aliases: ['claude', 'claude.ai', 'anthropic', 'anthropic.com'],+ },+ {+ canonical: 'perplexity.ai',+ aliases: ['perplexity', 'perplexity.ai'],+ },+ {+ canonical: 'gemini.google.com',+ aliases: ['gemini', 'google gemini', 'gemini.google.com', 'bard.google.com'],+ },+ {+ canonical: 'copilot.com',+ aliases: ['copilot', 'copilot.com', 'copilot.microsoft.com', 'microsoft copilot'],+ },+ { canonical: 'grok.com', aliases: ['grok', 'grok.com'] },+ {+ canonical: 'mistral.ai',+ aliases: ['mistral', 'mistral.ai', 'chat.mistral.ai', 'le chat'],+ },+ { canonical: 'kagi.com', aliases: ['kagi', 'kagi.com', 'assistant.kagi.com'] },+] as const satisfies ReadonlyArray<{+ canonical: string;+ aliases: readonly string[];+}>;++const quoteList = (values: readonly string[]) =>+ values.map((value) => `'${value}'`).join(', ');++/** `norm` is the alias bound by AI_REFERRER_CTE below. */+const AI_REFERRER_CTE = `WITH ${NORMALIZED_REFERRER_NAME} AS norm`;++const AI_REFERRER_FILTER = `norm IN (${quoteList(+ AI_REFERRERS.flatMap((engine) => engine.aliases)+)})`;++/** Collapses every alias onto one row per engine. */+const AI_REFERRER_CANONICAL_NAME = `multiIf(${AI_REFERRERS.map(+ (engine) => `norm IN (${quoteList(engine.aliases)}), '${engine.canonical}'`+).join(', ')}, norm)`;++/**+ * Half-open windows so the last day of the range is included and the previous+ * window ends exactly where the current one starts (no gap, no overlap).+ */+function getComparisonWindows(startDate: string, endDate: string) {+ const start = new Date(`${startDate}T00:00:00Z`);+ const endExclusive = new Date(`${endDate}T00:00:00Z`);+ endExclusive.setUTCDate(endExclusive.getUTCDate() + 1);+ const previousStart = new Date(+ start.getTime() - (endExclusive.getTime() - start.getTime())+ );+ const fmt = (date: Date) => date.toISOString().slice(0, 19).replace('T', ' ');+ return {+ current: { start: fmt(start), end: fmt(endExclusive) },+ previous: { start: fmt(previousStart), end: fmt(start) },+ };+}+export const gscRouter = createTRPCRouter({getConnection: protectedProcedure.input(z.object({ projectId: z.string() }))@@ -256,40 +330,42 @@ export const gscRouter = createTRPCRouter({throw new TRPCForbiddenError('You do not have access to this project');}const { startDate, endDate } = await resolveDates(input.projectId, input);+ const windows = getComparisonWindows(startDate, endDate);- const startMs = new Date(startDate).getTime();- const duration = new Date(endDate).getTime() - startMs;- const prevEnd = new Date(startMs - 1);- const prevStart = new Date(prevEnd.getTime() - duration);- const fmt = (d: Date) => d.toISOString().slice(0, 10);+ const where = (window: { start: string; end: string }) =>+ `project_id = '${input.projectId}'+ AND referrer_type = 'search'+ AND created_at >= '${window.start}'+ AND created_at < '${window.end}'`;- const [engines, [prevResult]] = await Promise.all([+ const [engines, [currentResult], [prevResult]] = await Promise.all([chQuery<{ name: string; sessions: number }>(`SELECTreferrer_name as name,count(*) as sessionsFROM ${TABLE_NAMES.sessions}- WHERE project_id = '${input.projectId}'- AND referrer_type = 'search'- AND created_at >= '${startDate}'- AND created_at <= '${endDate}'- GROUP BY referrer_name+ WHERE ${where(windows.current)}+ GROUP BY nameORDER BY sessions DESCLIMIT 10`),chQuery<{ sessions: number }>(`SELECT count(*) as sessionsFROM ${TABLE_NAMES.sessions}- WHERE project_id = '${input.projectId}'- AND referrer_type = 'search'- AND created_at >= '${fmt(prevStart)}'- AND created_at <= '${fmt(prevEnd)}'`+ WHERE ${where(windows.current)}`+ ),+ chQuery<{ sessions: number }>(+ `SELECT count(*) as sessions+ FROM ${TABLE_NAMES.sessions}+ WHERE ${where(windows.previous)}`),]);return {engines,- total: engines.reduce((s, e) => s + e.sessions, 0),+ // Counted separately from `engines`, which is capped at the top 10, so+ // the total compares like-for-like with the previous period.+ total: currentResult?.sessions ?? 0,previousTotal: prevResult?.sessions ?? 0,};}),@@ -305,59 +381,37 @@ export const gscRouter = createTRPCRouter({throw new TRPCForbiddenError('You do not have access to this project');}const { startDate, endDate } = await resolveDates(input.projectId, input);+ const windows = getComparisonWindows(startDate, endDate);- const startMs = new Date(startDate).getTime();- const duration = new Date(endDate).getTime() - startMs;- const prevEnd = new Date(startMs - 1);- const prevStart = new Date(prevEnd.getTime() - duration);- const fmt = (d: Date) => d.toISOString().slice(0, 10);-- // Known AI referrer names — will switch to referrer_type = 'ai' once available- const aiNames = [- 'chatgpt.com',- 'openai.com',- 'claude.ai',- 'anthropic.com',- 'perplexity.ai',- 'gemini.google.com',- 'copilot.com',- 'grok.com',- 'mistral.ai',- 'kagi.com',- ]- .map((n) => `'${n}', '${n.replace(/\.[^.]+$/, '')}'`)- .join(', ');-- const where = (start: string, end: string) =>+ // Matched by name — will switch to referrer_type = 'ai' once available.+ const where = (window: { start: string; end: string }) =>`project_id = '${input.projectId}'- AND referrer_name IN (${aiNames})- AND created_at >= '${start}'- AND created_at <= '${end}'`;+ AND ${AI_REFERRER_FILTER}+ AND created_at >= '${window.start}'+ AND created_at < '${window.end}'`;const [engines, [prevResult]] = await Promise.all([- chQuery<{ referrer_name: string; sessions: number }>(- `SELECT lower(- regexp_replace(referrer_name, '^https?://', '')- ) as referrer_name, count(*) as sessions+ chQuery<{ name: string; sessions: number }>(+ `${AI_REFERRER_CTE}+ SELECT ${AI_REFERRER_CANONICAL_NAME} as name, count(*) as sessionsFROM ${TABLE_NAMES.sessions}- WHERE ${where(startDate, endDate)}- GROUP BY referrer_name- ORDER BY sessions DESC- LIMIT 10`+ WHERE ${where(windows.current)}+ GROUP BY name+ ORDER BY sessions DESC`),chQuery<{ sessions: number }>(- `SELECT count(*) as sessions+ `${AI_REFERRER_CTE}+ SELECT count(*) as sessionsFROM ${TABLE_NAMES.sessions}- WHERE ${where(fmt(prevStart), fmt(prevEnd))}`+ WHERE ${where(windows.previous)}`),]);return {- engines: engines.map((e) => ({- name: e.referrer_name,- sessions: e.sessions,- })),- total: engines.reduce((s, e) => s + e.sessions, 0),+ engines,+ // The filter is a fixed alias set, so the grouped rows are the whole+ // population — no LIMIT, and the total matches the previous period.+ total: engines.reduce((sum, engine) => sum + engine.sessions, 0),previousTotal: prevResult?.sessions ?? 0,};}),