Skip to content

OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.

perf(chart.values): read filter-value dropdowns from sessions, cache, clamp (#469)

ayushjhanwar-png6eb39464a638
diff --git a/apps/start/src/hooks/use-property-values.ts b/apps/start/src/hooks/use-property-values.ts
index 4b9b3589f..c0c2f40f5 100644
--- a/apps/start/src/hooks/use-property-values.ts
+++ b/apps/start/src/hooks/use-property-values.ts
@@ -7,6 +7,12 @@ export function usePropertyValues(params: any) {
const query = useQuery(
trpc.chart.values.queryOptions(input, {
enabled: enabled !== false && !!input.projectId,
+ // A filter-value dropdown doesn't need real-time data. Without a
+ // staleTime, React Query refetches on every window focus/remount — a tab
+ // left open on a filter re-fires the (slow, un-MV'd) values query and
+ // hammers ClickHouse. Cache 1h + no focus refetch.
+ staleTime: 60 * 60 * 1000,
+ refetchOnWindowFocus: false,
}),
);
return query.data?.values ?? [];
diff --git a/packages/trpc/src/routers/chart.ts b/packages/trpc/src/routers/chart.ts
index aea2980cc..0c0288570 100644
--- a/packages/trpc/src/routers/chart.ts
+++ b/packages/trpc/src/routers/chart.ts
@@ -53,6 +53,46 @@ import {
const cacher = cacheMiddleware(60);
+/**
+ * `chart.values` for a top-level event column (country, os, city, device…) has
+ * no MV, so it ran `SELECT DISTINCT <col> FROM events WHERE created_at >
+ * now() - 6 MONTH` — a full multi-billion-row scan on high-volume projects that
+ * hits max_execution_time and returns nothing, then a dashboard tab left open
+ * on such a filter re-fires it on every window focus (React Query staleTime=0).
+ *
+ * These columns are session-level (geo/device/referrer denormalised onto every
+ * event), so for an all-events (`*`) value dropdown we read the distinct set
+ * from the far smaller `sessions` table (one row per session) instead — same
+ * values, orders of magnitude fewer rows. `path`/`origin` are per-pageview (not
+ * on `sessions`) and keep using `events`. A specific-event query also stays on
+ * `events` since `sessions` has no `name` column (the name filter bounds it).
+ */
+const SESSION_LEVEL_VALUE_COLUMNS = new Set([
+ 'country',
+ 'region',
+ 'city',
+ 'os',
+ 'os_version',
+ 'browser',
+ 'browser_version',
+ 'device',
+ 'brand',
+ 'model',
+ 'referrer',
+ 'referrer_name',
+ 'referrer_type',
+]);
+
+/**
+ * Lookback for the `events` fallback of the filter-value dropdown. A value
+ * picker only needs recently-seen values, so clamp the scan (default 30d)
+ * instead of the old 6-month full scan. Env-tunable.
+ */
+const VALUES_LOOKBACK_DAYS = Number.parseInt(
+ process.env.CHART_VALUES_LOOKBACK_DAYS || '30',
+ 10,
+);
+
/**
* Cap on distinct event property keys returned to the picker. Projects in the
* 8k range exist, so the previous 10k was reachable in normal use.
@@ -422,17 +462,26 @@ export const chartRouter = createTRPCRouter({
if (!isKnownEventField(resolvedProperty)) {
return { values: [] };
}
+ // Session-level columns (geo/device/referrer) come from the small
+ // `sessions` table, not a billions-row `events` scan. Only for
+ // all-events (`*`): `sessions` has no `name` column to filter on.
+ const useSessions =
+ event === '*' && SESSION_LEVEL_VALUE_COLUMNS.has(resolvedProperty);
const query = clix(ch)
.select<{ values: string[] }>([
`distinct ${getSelectPropertyKey(resolvedProperty)} as values`,
])
- .from(TABLE_NAMES.events)
+ .from(useSessions ? TABLE_NAMES.sessions : TABLE_NAMES.events)
.where('project_id', '=', projectId)
- .where('created_at', '>', clix.exp('now() - INTERVAL 6 MONTH'))
+ .where(
+ 'created_at',
+ '>',
+ clix.exp(`now() - INTERVAL ${VALUES_LOOKBACK_DAYS} DAY`),
+ )
.orderBy('created_at', 'DESC')
.limit(100_000);
- if (event !== '*') {
+ if (!useSessions && event !== '*') {
query.where('name', '=', event);
}