OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.
fix(chart): list all profile property keys instead of sampling 10k unordered rows (#433)
Carl-Gerhard Lindesvärdd6a7bc005e0b
diff --git a/packages/db/src/services/profile.service.ts b/packages/db/src/services/profile.service.tsindex 8bdfdfc12..d32f38abb 100644--- a/packages/db/src/services/profile.service.ts+++ b/packages/db/src/services/profile.service.ts@@ -539,3 +539,37 @@ export async function getProfileMetricsCore(input: {revenue: raw.revenue,};}++/**+ * Every distinct key present in any external profile's `properties` map.+ *+ * Aggregated across all profiles rather than sampled — a `LIMIT n` sample with+ * no `ORDER BY` returns whichever rows ClickHouse's scheduler happens to+ * produce, so the key set varied between requests and properties set on a small+ * fraction of profiles were usually missing from the picker entirely.+ *+ * `FINAL` isn't needed: older row versions can only contribute keys that+ * genuinely existed at some point.+ */+export async function getProfilePropertyKeys(+ projectId: string,+): Promise<string[]> {+ const rows = await clix(ch)+ .select<{ key: string }>(['DISTINCT arrayJoin(mapKeys(properties)) as key'])+ .from(TABLE_NAMES.profiles)+ .where('project_id', '=', projectId)+ .where('is_external', '=', true)+ .execute();+ return rows.map((r) => r.key).sort();+}++/**+ * Cached by projectId only. The picker's tRPC-level cache keys on the whole+ * input, which includes `event` — so without this the full profile scan would+ * repeat once per event within the same window, even though the profile keys+ * don't depend on the event at all.+ */+export const getProfilePropertyKeysCached = cacheable(+ getProfilePropertyKeys,+ 60,+);diff --git a/packages/trpc/src/routers/chart.ts b/packages/trpc/src/routers/chart.tsindex 0c641e79e..20240a33e 100644--- a/packages/trpc/src/routers/chart.ts+++ b/packages/trpc/src/routers/chart.ts@@ -13,6 +13,7 @@ import {getEventFiltersWhereClause,getEventMetasCached,getGroupPropertySelect,+ getProfilePropertyKeysCached,getProfilePropertySelect,getProfilesCached,getReportById,@@ -51,6 +52,12 @@ import {const cacher = cacheMiddleware(60);+/**+ * Cap on distinct event property keys returned to the picker. Projects in the+ * 8k range exist, so the previous 10k was reachable in normal use.+ */+const EVENT_PROPERTY_KEY_LIMIT = 50_000;+const chartProcedure = publicProcedure.use(async ({ ctx, next, getRawInput }) => {const rawInput = (await getRawInput()) as {@@ -225,6 +232,10 @@ export const chartRouter = createTRPCRouter({}),properties: protectedProcedure+ // Aggregating every profile's property keys costs more than the old+ // 10k-row sample. It's still sub-second on millions of profiles, and this+ // list barely moves, so a short cache absorbs it.+ .use(cacheMiddleware(60)).input(z.object({event: z.string().optional(),@@ -232,21 +243,9 @@ export const chartRouter = createTRPCRouter({})).query(async ({ input: { projectId, event } }) => {- const profiles = await clix(ch, 'UTC')- .select<Pick<IServiceProfile, 'properties'>>(['properties'])- .from(TABLE_NAMES.profiles)- .where('project_id', '=', projectId)- .where('is_external', '=', true)- .limit(10_000)- .execute();-- const profileProperties = [- ...new Set(- profiles.flatMap((p) =>- Object.keys(p.properties).map((k) => `profile.properties.${k}`)- )- ),- ];+ const profileProperties = (+ await getProfilePropertyKeysCached(projectId)+ ).map((key) => `profile.properties.${key}`);const query = clix(ch).select<{ property_key: string; created_at: string }>([@@ -256,9 +255,14 @@ export const chartRouter = createTRPCRouter({.from(TABLE_NAMES.event_property_values_mv).where('project_id', '=', projectId).groupBy(['property_key'])- .orderBy('length(property_key)', 'ASC')+ // Order by recency, not by key length. The cap has to drop *something*+ // on projects with very many distinct keys, and dropping the longest+ // keys first meant losing the most descriptive ones. `property_key`+ // breaks ties so the cap can't cut an arbitrary side of a tied group —+ // an unstable list is the bug this whole change is about..orderBy('created_at', 'DESC')- .limit(10_000);+ .orderBy('property_key', 'ASC')+ .limit(EVENT_PROPERTY_KEY_LIMIT);if (event && event !== '*') {query.where('name', '=', event);