Skip to content

OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.

fix: ai referrers on seo page (percentage bug)

Carl-Gerhard Lindesvärdb104e6838d76
diff --git a/packages/trpc/src/routers/gsc.ts b/packages/trpc/src/routers/gsc.ts
index ba3c182fb..3af5248ec 100644
--- a/packages/trpc/src/routers/gsc.ts
+++ b/packages/trpc/src/routers/gsc.ts
@@ -47,6 +47,80 @@ async function resolveDates(
};
}
+/**
+ * ClickHouse stores the same AI referrer under several spellings: the parsed
+ * display name ('ChatGPT', 'Google Gemini'), the bare host ('chatgpt.com') and
+ * the full origin ('https://kagi.com'). Normalize before matching so every
+ * spelling lands on the same engine.
+ */
+const NORMALIZED_REFERRER_NAME =
+ "lower(regexp_replace(referrer_name, '^https?://(www[.])?', ''))";
+
+const AI_REFERRERS = [
+ {
+ canonical: 'chatgpt.com',
+ aliases: ['chatgpt', 'chatgpt.com', 'chat.openai.com', 'openai', 'openai.com'],
+ },
+ {
+ canonical: 'claude.ai',
+ aliases: ['claude', 'claude.ai', 'anthropic', 'anthropic.com'],
+ },
+ {
+ canonical: 'perplexity.ai',
+ aliases: ['perplexity', 'perplexity.ai'],
+ },
+ {
+ canonical: 'gemini.google.com',
+ aliases: ['gemini', 'google gemini', 'gemini.google.com', 'bard.google.com'],
+ },
+ {
+ canonical: 'copilot.com',
+ aliases: ['copilot', 'copilot.com', 'copilot.microsoft.com', 'microsoft copilot'],
+ },
+ { canonical: 'grok.com', aliases: ['grok', 'grok.com'] },
+ {
+ canonical: 'mistral.ai',
+ aliases: ['mistral', 'mistral.ai', 'chat.mistral.ai', 'le chat'],
+ },
+ { canonical: 'kagi.com', aliases: ['kagi', 'kagi.com', 'assistant.kagi.com'] },
+] as const satisfies ReadonlyArray<{
+ canonical: string;
+ aliases: readonly string[];
+}>;
+
+const quoteList = (values: readonly string[]) =>
+ values.map((value) => `'${value}'`).join(', ');
+
+/** `norm` is the alias bound by AI_REFERRER_CTE below. */
+const AI_REFERRER_CTE = `WITH ${NORMALIZED_REFERRER_NAME} AS norm`;
+
+const AI_REFERRER_FILTER = `norm IN (${quoteList(
+ AI_REFERRERS.flatMap((engine) => engine.aliases)
+)})`;
+
+/** Collapses every alias onto one row per engine. */
+const AI_REFERRER_CANONICAL_NAME = `multiIf(${AI_REFERRERS.map(
+ (engine) => `norm IN (${quoteList(engine.aliases)}), '${engine.canonical}'`
+).join(', ')}, norm)`;
+
+/**
+ * Half-open windows so the last day of the range is included and the previous
+ * window ends exactly where the current one starts (no gap, no overlap).
+ */
+function getComparisonWindows(startDate: string, endDate: string) {
+ const start = new Date(`${startDate}T00:00:00Z`);
+ const endExclusive = new Date(`${endDate}T00:00:00Z`);
+ endExclusive.setUTCDate(endExclusive.getUTCDate() + 1);
+ const previousStart = new Date(
+ start.getTime() - (endExclusive.getTime() - start.getTime())
+ );
+ const fmt = (date: Date) => date.toISOString().slice(0, 19).replace('T', ' ');
+ return {
+ current: { start: fmt(start), end: fmt(endExclusive) },
+ previous: { start: fmt(previousStart), end: fmt(start) },
+ };
+}
+
export const gscRouter = createTRPCRouter({
getConnection: protectedProcedure
.input(z.object({ projectId: z.string() }))
@@ -256,40 +330,42 @@ export const gscRouter = createTRPCRouter({
throw new TRPCForbiddenError('You do not have access to this project');
}
const { startDate, endDate } = await resolveDates(input.projectId, input);
+ const windows = getComparisonWindows(startDate, endDate);
- const startMs = new Date(startDate).getTime();
- const duration = new Date(endDate).getTime() - startMs;
- const prevEnd = new Date(startMs - 1);
- const prevStart = new Date(prevEnd.getTime() - duration);
- const fmt = (d: Date) => d.toISOString().slice(0, 10);
+ const where = (window: { start: string; end: string }) =>
+ `project_id = '${input.projectId}'
+ AND referrer_type = 'search'
+ AND created_at >= '${window.start}'
+ AND created_at < '${window.end}'`;
- const [engines, [prevResult]] = await Promise.all([
+ const [engines, [currentResult], [prevResult]] = await Promise.all([
chQuery<{ name: string; sessions: number }>(
`SELECT
referrer_name as name,
count(*) as sessions
FROM ${TABLE_NAMES.sessions}
- WHERE project_id = '${input.projectId}'
- AND referrer_type = 'search'
- AND created_at >= '${startDate}'
- AND created_at <= '${endDate}'
- GROUP BY referrer_name
+ WHERE ${where(windows.current)}
+ GROUP BY name
ORDER BY sessions DESC
LIMIT 10`
),
chQuery<{ sessions: number }>(
`SELECT count(*) as sessions
FROM ${TABLE_NAMES.sessions}
- WHERE project_id = '${input.projectId}'
- AND referrer_type = 'search'
- AND created_at >= '${fmt(prevStart)}'
- AND created_at <= '${fmt(prevEnd)}'`
+ WHERE ${where(windows.current)}`
+ ),
+ chQuery<{ sessions: number }>(
+ `SELECT count(*) as sessions
+ FROM ${TABLE_NAMES.sessions}
+ WHERE ${where(windows.previous)}`
),
]);
return {
engines,
- total: engines.reduce((s, e) => s + e.sessions, 0),
+ // Counted separately from `engines`, which is capped at the top 10, so
+ // the total compares like-for-like with the previous period.
+ total: currentResult?.sessions ?? 0,
previousTotal: prevResult?.sessions ?? 0,
};
}),
@@ -305,59 +381,37 @@ export const gscRouter = createTRPCRouter({
throw new TRPCForbiddenError('You do not have access to this project');
}
const { startDate, endDate } = await resolveDates(input.projectId, input);
+ const windows = getComparisonWindows(startDate, endDate);
- const startMs = new Date(startDate).getTime();
- const duration = new Date(endDate).getTime() - startMs;
- const prevEnd = new Date(startMs - 1);
- const prevStart = new Date(prevEnd.getTime() - duration);
- const fmt = (d: Date) => d.toISOString().slice(0, 10);
-
- // Known AI referrer names — will switch to referrer_type = 'ai' once available
- const aiNames = [
- 'chatgpt.com',
- 'openai.com',
- 'claude.ai',
- 'anthropic.com',
- 'perplexity.ai',
- 'gemini.google.com',
- 'copilot.com',
- 'grok.com',
- 'mistral.ai',
- 'kagi.com',
- ]
- .map((n) => `'${n}', '${n.replace(/\.[^.]+$/, '')}'`)
- .join(', ');
-
- const where = (start: string, end: string) =>
+ // Matched by name — will switch to referrer_type = 'ai' once available.
+ const where = (window: { start: string; end: string }) =>
`project_id = '${input.projectId}'
- AND referrer_name IN (${aiNames})
- AND created_at >= '${start}'
- AND created_at <= '${end}'`;
+ AND ${AI_REFERRER_FILTER}
+ AND created_at >= '${window.start}'
+ AND created_at < '${window.end}'`;
const [engines, [prevResult]] = await Promise.all([
- chQuery<{ referrer_name: string; sessions: number }>(
- `SELECT lower(
- regexp_replace(referrer_name, '^https?://', '')
- ) as referrer_name, count(*) as sessions
+ chQuery<{ name: string; sessions: number }>(
+ `${AI_REFERRER_CTE}
+ SELECT ${AI_REFERRER_CANONICAL_NAME} as name, count(*) as sessions
FROM ${TABLE_NAMES.sessions}
- WHERE ${where(startDate, endDate)}
- GROUP BY referrer_name
- ORDER BY sessions DESC
- LIMIT 10`
+ WHERE ${where(windows.current)}
+ GROUP BY name
+ ORDER BY sessions DESC`
),
chQuery<{ sessions: number }>(
- `SELECT count(*) as sessions
+ `${AI_REFERRER_CTE}
+ SELECT count(*) as sessions
FROM ${TABLE_NAMES.sessions}
- WHERE ${where(fmt(prevStart), fmt(prevEnd))}`
+ WHERE ${where(windows.previous)}`
),
]);
return {
- engines: engines.map((e) => ({
- name: e.referrer_name,
- sessions: e.sessions,
- })),
- total: engines.reduce((s, e) => s + e.sessions, 0),
+ engines,
+ // The filter is a fixed alias set, so the grouped rows are the whole
+ // population — no LIMIT, and the total matches the previous period.
+ total: engines.reduce((sum, engine) => sum + engine.sessions, 0),
previousTotal: prevResult?.sessions ?? 0,
};
}),