Skip to content

OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.

fix(chart): list all profile property keys instead of sampling 10k unordered rows (#433)

Carl-Gerhard Lindesvärdd6a7bc005e0b
diff --git a/packages/db/src/services/profile.service.ts b/packages/db/src/services/profile.service.ts
index 8bdfdfc12..d32f38abb 100644
--- a/packages/db/src/services/profile.service.ts
+++ b/packages/db/src/services/profile.service.ts
@@ -539,3 +539,37 @@ export async function getProfileMetricsCore(input: {
revenue: raw.revenue,
};
}
+
+/**
+ * Every distinct key present in any external profile's `properties` map.
+ *
+ * Aggregated across all profiles rather than sampled — a `LIMIT n` sample with
+ * no `ORDER BY` returns whichever rows ClickHouse's scheduler happens to
+ * produce, so the key set varied between requests and properties set on a small
+ * fraction of profiles were usually missing from the picker entirely.
+ *
+ * `FINAL` isn't needed: older row versions can only contribute keys that
+ * genuinely existed at some point.
+ */
+export async function getProfilePropertyKeys(
+ projectId: string,
+): Promise<string[]> {
+ const rows = await clix(ch)
+ .select<{ key: string }>(['DISTINCT arrayJoin(mapKeys(properties)) as key'])
+ .from(TABLE_NAMES.profiles)
+ .where('project_id', '=', projectId)
+ .where('is_external', '=', true)
+ .execute();
+ return rows.map((r) => r.key).sort();
+}
+
+/**
+ * Cached by projectId only. The picker's tRPC-level cache keys on the whole
+ * input, which includes `event` — so without this the full profile scan would
+ * repeat once per event within the same window, even though the profile keys
+ * don't depend on the event at all.
+ */
+export const getProfilePropertyKeysCached = cacheable(
+ getProfilePropertyKeys,
+ 60,
+);
diff --git a/packages/trpc/src/routers/chart.ts b/packages/trpc/src/routers/chart.ts
index 0c641e79e..20240a33e 100644
--- a/packages/trpc/src/routers/chart.ts
+++ b/packages/trpc/src/routers/chart.ts
@@ -13,6 +13,7 @@ import {
getEventFiltersWhereClause,
getEventMetasCached,
getGroupPropertySelect,
+ getProfilePropertyKeysCached,
getProfilePropertySelect,
getProfilesCached,
getReportById,
@@ -51,6 +52,12 @@ import {
const cacher = cacheMiddleware(60);
+/**
+ * Cap on distinct event property keys returned to the picker. Projects in the
+ * 8k range exist, so the previous 10k was reachable in normal use.
+ */
+const EVENT_PROPERTY_KEY_LIMIT = 50_000;
+
const chartProcedure = publicProcedure.use(
async ({ ctx, next, getRawInput }) => {
const rawInput = (await getRawInput()) as {
@@ -225,6 +232,10 @@ export const chartRouter = createTRPCRouter({
}),
properties: protectedProcedure
+ // Aggregating every profile's property keys costs more than the old
+ // 10k-row sample. It's still sub-second on millions of profiles, and this
+ // list barely moves, so a short cache absorbs it.
+ .use(cacheMiddleware(60))
.input(
z.object({
event: z.string().optional(),
@@ -232,21 +243,9 @@ export const chartRouter = createTRPCRouter({
})
)
.query(async ({ input: { projectId, event } }) => {
- const profiles = await clix(ch, 'UTC')
- .select<Pick<IServiceProfile, 'properties'>>(['properties'])
- .from(TABLE_NAMES.profiles)
- .where('project_id', '=', projectId)
- .where('is_external', '=', true)
- .limit(10_000)
- .execute();
-
- const profileProperties = [
- ...new Set(
- profiles.flatMap((p) =>
- Object.keys(p.properties).map((k) => `profile.properties.${k}`)
- )
- ),
- ];
+ const profileProperties = (
+ await getProfilePropertyKeysCached(projectId)
+ ).map((key) => `profile.properties.${key}`);
const query = clix(ch)
.select<{ property_key: string; created_at: string }>([
@@ -256,9 +255,14 @@ export const chartRouter = createTRPCRouter({
.from(TABLE_NAMES.event_property_values_mv)
.where('project_id', '=', projectId)
.groupBy(['property_key'])
- .orderBy('length(property_key)', 'ASC')
+ // Order by recency, not by key length. The cap has to drop *something*
+ // on projects with very many distinct keys, and dropping the longest
+ // keys first meant losing the most descriptive ones. `property_key`
+ // breaks ties so the cap can't cut an arbitrary side of a tied group —
+ // an unstable list is the bug this whole change is about.
.orderBy('created_at', 'DESC')
- .limit(10_000);
+ .orderBy('property_key', 'ASC')
+ .limit(EVENT_PROPERTY_KEY_LIMIT);
if (event && event !== '*') {
query.where('name', '=', event);