OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.
perf(api): chart, funnel, conversion and event-list profile joins drop FINAL
Carl-Gerhard Lindesvärddcf07875ad9a
diff --git a/packages/core/src/modules/chart/src/conversion.sql.ts b/packages/core/src/modules/chart/src/conversion.sql.tsindex efa959d3a..e91213cef 100644--- a/packages/core/src/modules/chart/src/conversion.sql.ts+++ b/packages/core/src/modules/chart/src/conversion.sql.ts@@ -10,6 +10,7 @@ import { compiledText } from './compiled';import {buildGroupsQuery,CHART_TABLE,+ latestProfileValue,profileIdsInWindow,} from './field-resolution';@@ -74,11 +75,17 @@ function profileJoin(if (fields.length === 0) {return sql.empty;}- // Simple column names (not aliased) so profile.properties works directly.+ // Aliased back to the column name so profile.properties works directly.+ const columns = fields.map((field) =>+ field === 'id'+ ? sql.id(field)+ : sql`${latestProfileValue(field, sql.id(field))} AS ${sql.id(field)}`+ );return sql`LEFT ANY JOIN (- SELECT ${sql.join(fields.map((field) => sql.id(field)))}- FROM ${sql.id(CHART_TABLE.profiles)} FINAL+ SELECT ${sql.join(columns)}+ FROM ${sql.id(CHART_TABLE.profiles)}WHERE project_id = ${sql.string(projectId)} AND ${activeProfiles}+ GROUP BY id) as profile ON profile.id = profile_id`;}diff --git a/packages/core/src/modules/chart/src/field-resolution.ts b/packages/core/src/modules/chart/src/field-resolution.tsindex d8b5f74bb..1ec1e5617 100644--- a/packages/core/src/modules/chart/src/field-resolution.ts+++ b/packages/core/src/modules/chart/src/field-resolution.ts@@ -480,6 +480,24 @@ export function getSelectPropertyKey(return sql`${map}[${sql.string(property.replace(new RegExp(`^${match}.`), ''))}]`;}+/**+ * `column` as `profiles FINAL` would return it, for a `GROUP BY id` over the+ * raw rows: `last_seen_at` is the ReplacingMergeTree version column, so the+ * value from the row with the greatest one is the surviving version's.+ */+export function latestProfileValue(+ name: string,+ column: SqlFragment+): SqlFragment {+ if (name === 'id') {+ return column;+ }+ if (name === 'last_seen_at') {+ return sql`max(${column})`;+ }+ return sql`argMax(${column}, last_seen_at)`;+}+// profile.properties.<key> refs render as Map lookups `profile.properties['<key>']`.// Pulling the whole `properties` Map into the profile CTE makes the LEFT ANY// JOIN hash carry the full Map per profile — roughly a kilobyte each on real@@ -530,10 +548,10 @@ export function profilePropertiesCteSelect(// could break out of the backticks.const cols = keys.map((k) =>- sql`properties[${sql.string(k)}] as ${compiledText(`\`profile.properties.${k}\``)}`+ sql`argMax(properties[${sql.string(k)}], last_seen_at) as ${compiledText(`\`profile.properties.${k}\``)}`);if (needsFullMap || cols.length === 0) {- cols.push(sql`properties as "profile.properties"`);+ cols.push(sql`argMax(properties, last_seen_at) as "profile.properties"`);}return sql.join(cols, ', ');}diff --git a/packages/core/src/modules/chart/src/funnel.sql.test.ts b/packages/core/src/modules/chart/src/funnel.sql.test.tsindex 431b74047..a9fc6623d 100644--- a/packages/core/src/modules/chart/src/funnel.sql.test.ts+++ b/packages/core/src/modules/chart/src/funnel.sql.test.ts@@ -175,7 +175,9 @@ describe('funnel.sql / buildFunnelBase — profile breakdowns', () => {// The breakdown is narrowed to a scalar alias selected by the profile// join, so both the join and the aliased column must exist.- expect(text).toContain("properties['plan'] as `profile.properties.plan`");+ expect(text).toContain(+ "argMax(properties['plan'], last_seen_at) as `profile.properties.plan`"+ );expect(sql).toMatch(/as profile/);expect(sql).toContain('profile.id = events.profile_id');await explain(statement);@@ -186,7 +188,7 @@ describe('funnel.sql / buildFunnelBase — profile breakdowns', () => {breakdowns: [breakdown('profile.properties.plan')],});expect(render(statement).sql).toMatch(- /SELECT [^)]*properties[^)]*FROM profiles/+ /SELECT .*properties.*FROM profiles/);await explain(statement);});@@ -419,9 +421,11 @@ describe('funnel.sql / profile-property narrowing', () => {});const { text } = render(statement);- expect(text).toContain("properties['plan'] as `profile.properties.plan`");expect(text).toContain(- "properties['experiment'] as `profile.properties.experiment`"+ "argMax(properties['plan'], last_seen_at) as `profile.properties.plan`"+ );+ expect(text).toContain(+ "argMax(properties['experiment'], last_seen_at) as `profile.properties.experiment`");expect(text).not.toContain('properties as "profile.properties"');// Conditions (windowFunnel + pre-filter) and the breakdown expression arediff --git a/packages/core/src/modules/chart/src/funnel.sql.ts b/packages/core/src/modules/chart/src/funnel.sql.tsindex 8ded7c2ef..fa87cb289 100644--- a/packages/core/src/modules/chart/src/funnel.sql.ts+++ b/packages/core/src/modules/chart/src/funnel.sql.ts@@ -27,6 +27,7 @@ import {getSelectPropertyKey,isAllCohortsBreakdown,isKnownEventField,+ latestProfileValue,profileIdsInWindow,profilePropertiesCteSelect,} from './field-resolution';@@ -217,7 +218,9 @@ function profileCteColumns(// The allowlist is the backstop that makes `sql.id` throw rather than inline// if either loop above lets something else through (GHSA-pc3q-gw7f-p2x2).const columns: SqlFragment[] = Array.from(fields).map((field) =>- sql.id(field, FUNNEL_JOIN_COLUMNS)+ field === 'id'+ ? sql.id(field, FUNNEL_JOIN_COLUMNS)+ : sql`${latestProfileValue(field, sql.id(field, FUNNEL_JOIN_COLUMNS))} AS ${sql.id(field, FUNNEL_JOIN_COLUMNS)}`);const referencesProperties =profileFilters.some((filter) => filter.startsWith('properties')) ||@@ -303,8 +306,8 @@ export function funnelBase(input: FunnelBaseInput): FunnelBase {});const profileJoin =columns.length > 0- ? sql` LEFT JOIN (SELECT ${sql.join(columns, ', ')} FROM ${sql.id(CHART_TABLE.profiles)} FINAL- WHERE project_id = ${sql.string(projectId)} AND ${activeProfiles}) as profile ON profile.id = events.profile_id`+ ? sql` LEFT JOIN (SELECT ${sql.join(columns, ', ')} FROM ${sql.id(CHART_TABLE.profiles)}+ WHERE project_id = ${sql.string(projectId)} AND ${activeProfiles} GROUP BY id) as profile ON profile.id = events.profile_id`: sql.empty;const groupJoin = needsGroupArrayJoin? sql` ARRAY JOIN groups AS _group_id LEFT ANY JOIN _g ON _g.id = _group_id`diff --git a/packages/core/src/modules/chart/src/property-key-escaping.test.ts b/packages/core/src/modules/chart/src/property-key-escaping.test.tsindex f2e367280..67652cc65 100644--- a/packages/core/src/modules/chart/src/property-key-escaping.test.ts+++ b/packages/core/src/modules/chart/src/property-key-escaping.test.ts@@ -277,7 +277,7 @@ describe('profile-property narrowing with a quoted key', () => {fieldResolution.profilePropertiesCteSelect(keys, needsFullMap));expect(cteSelect.query).toBe(- "properties[{p1:String}] as `profile.properties.pl'an`"+ "argMax(properties[{p1:String}], last_seen_at) as `profile.properties.pl'an`");expect(cteSelect.query_params).toEqual({ p1: key });@@ -302,7 +302,9 @@ describe('profile-property narrowing with a quoted key', () => {expect(statement(fieldResolution.profilePropertiesCteSelect(keys, needsFullMap)).query- ).toBe('properties as "profile.properties"');+ ).toBe(+ 'argMax(properties, last_seen_at) as "profile.properties"'+ );});it('leaves ordinary keys narrowing exactly as before', () => {@@ -314,7 +316,9 @@ describe('profile-property narrowing with a quoted key', () => {expect(statement(fieldResolution.profilePropertiesCteSelect(keys, needsFullMap)).query- ).toBe('properties[{p1:String}] as `profile.properties.plan`');+ ).toBe(+ 'argMax(properties[{p1:String}], last_seen_at) as `profile.properties.plan`'+ );expect(statement(compiled.fragmentWithProfileRefs(diff --git a/packages/core/src/modules/chart/src/sql.test.ts b/packages/core/src/modules/chart/src/sql.test.tsindex 6a2ce6f7d..feb3d6222 100644--- a/packages/core/src/modules/chart/src/sql.test.ts+++ b/packages/core/src/modules/chart/src/sql.test.ts@@ -456,12 +456,14 @@ describe('profile-property narrowing', () => {});// The CTE selects one scalar column per referenced key...- expect(text).toContain("properties['plan'] as `profile.properties.plan`");expect(text).toContain(- "properties['experiment'] as `profile.properties.experiment`"+ "argMax(properties['plan'], last_seen_at) as `profile.properties.plan`"+ );+ expect(text).toContain(+ "argMax(properties['experiment'], last_seen_at) as `profile.properties.experiment`");// ...instead of every profile's whole Map...- expect(text).not.toContain('properties as "profile.properties"');+ expect(text).not.toContain('as "profile.properties"');// ...and every ref in the query is rewritten to the scalar alias.expect(text).not.toContain("profile.properties['plan']");expect(text).not.toContain("profile.properties['experiment']");@@ -484,7 +486,9 @@ describe('profile-property narrowing', () => {});// mapExtractKeyLike needs the whole Map, so it must stay selected.- expect(text).toContain('properties as "profile.properties"');+ expect(text).toContain(+ 'argMax(properties, last_seen_at) as "profile.properties"'+ );});it('never narrows identifier-unsafe keys (backtick falls back to the Map)', async () => {@@ -505,7 +509,9 @@ describe('profile-property narrowing', () => {// The unsafe key keeps its original Map access against the full Map —// it must never be embedded in a backtick-quoted alias.- expect(text).toContain('properties as "profile.properties"');+ expect(text).toContain(+ 'argMax(properties, last_seen_at) as "profile.properties"'+ );expect(text).not.toContain('as `profile.properties.plan`tier`');});@@ -522,7 +528,9 @@ describe('profile-property narrowing', () => {// The metric's key must be narrowed alongside the filter's — otherwise// the metric keeps reading the Map that narrowing just removed.- expect(text).toContain("properties['age'] as `profile.properties.age`");+ expect(text).toContain(+ "argMax(properties['age'], last_seen_at) as `profile.properties.age`"+ );expect(text).not.toContain("profile.properties['age']");});@@ -540,7 +548,9 @@ describe('profile-property narrowing', () => {});expect(text).toContain('LEFT ANY JOIN profile ON profile.id = profile_id');- expect(text).toContain("properties['age'] as `profile.properties.age`");+ expect(text).toContain(+ "argMax(properties['age'], last_seen_at) as `profile.properties.age`"+ );});it('metric-only profile property parses and resolves', async () => {diff --git a/packages/core/src/modules/chart/src/sql.ts b/packages/core/src/modules/chart/src/sql.tsindex 604a97ef4..0f665bee6 100644--- a/packages/core/src/modules/chart/src/sql.ts+++ b/packages/core/src/modules/chart/src/sql.ts@@ -37,6 +37,7 @@ import {isAllCohortsBreakdown,isKnownEventField,isNumericColumn,+ latestProfileValue,PROFILE_CTE_FIELDS,profileIdsInWindow,profilePropertiesCteSelect,@@ -209,17 +210,18 @@ function profileCteSelectField(}// The alias is `"profile.<field>"` — two dots, so text, not `sql.id`.// `collectProfileCteFields` only ever yields a name from its own closed set.- return sql`${sql.id(field, PROFILE_CTE_FIELDS)} as ${compiledText(`"profile.${field}"`)}`;+ return sql`${latestProfileValue(field, sql.id(field, PROFILE_CTE_FIELDS))} as ${compiledText(`"profile.${field}"`)}`;}-function profileCte(+export function profileCte(selectFields: SqlFragment[],projectId: string,activeProfiles: SqlFragment): SqlFragment {return sql`SELECT ${sql.join(selectFields, ', ')}- FROM ${sql.id(CHART_TABLE.profiles)} FINAL- WHERE project_id = ${sql.string(projectId)} AND ${activeProfiles}`;+ FROM ${sql.id(CHART_TABLE.profiles)}+ WHERE project_id = ${sql.string(projectId)} AND ${activeProfiles}+ GROUP BY id`;}function anyRefOn(prefix: string, refs: { name: string }[]): boolean {@@ -633,7 +635,7 @@ export function profilePropertyValuesQuery(projectId: string,expression: SqlFragment): SqlFragment {- return sql`SELECT distinct ${expression} as values FROM ${sql.id(CHART_TABLE.profiles)} FINAL WHERE project_id = ${sql.string(projectId)} AND ${expression} != '' AND ${expression} IS NOT NULL ORDER BY created_at DESC LIMIT ${sql.uint64(PROPERTY_VALUES_LIMIT)}`;+ return sql`SELECT values FROM (SELECT argMax(${expression}, last_seen_at) as values, argMax(created_at, last_seen_at) as created_at FROM ${sql.id(CHART_TABLE.profiles)} WHERE project_id = ${sql.string(projectId)} GROUP BY id) WHERE values != '' AND values IS NOT NULL GROUP BY values ORDER BY max(created_at) DESC LIMIT ${sql.uint64(PROPERTY_VALUES_LIMIT)}`;}/** Distinct values of a `group.*` field straight off the groups table. */@@ -673,17 +675,20 @@ function bucketEquals(interval: IInterval, bucketDate: string): SqlFragment {}}-/** Top-level profile columns a `profile.*` reference reads, in first-seen order. */+/** Top-level profile columns a `profile.*` reference reads, in first-seen order, besides `id`. */function profileFieldsToSelect(refs: string[]): SqlFragment[] {const fields: string[] = [];for (const ref of refs) {const field = ref.replace('profile.', '').split('.')[0];- if (field && !fields.includes(field)) {+ if (field && field !== 'id' && !fields.includes(field)) {fields.push(field);}}// `sql.id` refuses any field not in PROFILE_COLUMNS rather than splicing it.- return fields.map((field) => sql.id(field, PROFILE_COLUMNS));+ return fields.map(+ (field) =>+ sql`${latestProfileValue(field, sql.id(field, PROFILE_COLUMNS))} AS ${sql.id(field, PROFILE_COLUMNS)}`+ );}/** Distinct profile ids behind one chart data point. */@@ -718,7 +723,7 @@ export function chartBucketProfilesQuery(eventNames: [event.name],window: [dateRange],});- joins.profiles = sql`LEFT ANY JOIN (SELECT id, ${fields} FROM ${sql.id(CHART_TABLE.profiles)} FINAL WHERE project_id = ${sql.string(projectId)} AND ${activeProfiles}) as profile on profile.id = profile_id`;+ joins.profiles = sql`LEFT ANY JOIN (SELECT id, ${fields} FROM ${sql.id(CHART_TABLE.profiles)} WHERE project_id = ${sql.string(projectId)} AND ${activeProfiles} GROUP BY id) as profile on profile.id = profile_id`;}const needsGroupJoin = [diff --git a/packages/core/src/modules/chart/src/sql.versions.test.ts b/packages/core/src/modules/chart/src/sql.versions.test.tsnew file mode 100644index 000000000..685020d36--- /dev/null+++ b/packages/core/src/modules/chart/src/sql.versions.test.ts@@ -0,0 +1,278 @@+// The profile joins no longer read `profiles FINAL`: every statement groups+// the raw rows by id and takes each column from the newest version. These+// run against profiles with several unmerged versions (test/profile-versions)+// and assert the latest version is what every chart shape sees — the stale+// version with plan=pro must never match, the other project must never leak.++import { afterAll, beforeAll, describe, expect, it } from 'bun:test';+import { sql } from '@openpanel/db/src/clickhouse/sql';+import { timeAgo } from '../../../../../../test/fixtures';+import {+ setupProfileVersions,+ teardownProfileVersions,+ VERSIONED,+} from '../../../../../../test/profile-versions';+import { expectSameRows, fetchRows } from '../../../../test/clickhouse-rows';+import type {+ IChartBreakdown,+ IChartEvent,+} from '../../report/report.constants';+import { compiledText } from './compiled';+import { conversionQuery } from './conversion.sql';+import {+ getProfilePropertySelect,+ getSelectPropertyKey,+ profileIdsInWindow,+} from './field-resolution';+import { funnelBase, funnelChartQuery } from './funnel.sql';+import {+ aggregateChartQuery,+ chartBucketProfilesQuery,+ chartSeriesQuery,+ profileCte,+ profilePropertyValuesQuery,+} from './sql';++const PROJECT_ID = 'chart-profile-versions';+const DAY_MS = 24 * 60 * 60 * 1000;+const PLAN = 'profile.properties.plan';+const planFilter = (value: string[]) => ({+ id: 'f',+ name: PLAN,+ operator: 'is' as const,+ value,+});++let ch: typeof import('@openpanel/db/src/clickhouse/client').ch;+let now: Date;+let window: { startDate: string; endDate: string };++const event = (overrides: Partial<IChartEvent> = {}): IChartEvent => ({+ id: 'A',+ name: 'screen_view',+ segment: 'event',+ filters: [],+ ...overrides,+});+const breakdown = (name: string): IChartBreakdown => ({ id: name, name });+const resolved = (breakdowns: IChartBreakdown[]) => ({+ breakdowns,+ allCohorts: [],+ cohortMetadata: new Map(),+ projectId: PROJECT_ID,+});++type Row = Record<string, string | number | string[]>;+async function rows(fragment: Parameters<typeof fetchRows>[1]): Promise<Row[]> {+ return (await fetchRows(ch, fragment)).map((row) => JSON.parse(row));+}+function countsBy(result: Row[], label: string): Record<string, number> {+ const counts: Record<string, number> = {};+ // WITH FILL pads a series with zero rows that carry no label.+ for (const row of result.filter((row) => Number(row.count) > 0)) {+ const key = String(row[label]);+ counts[key] = (counts[key] ?? 0) + Number(row.count);+ }+ return counts;+}++beforeAll(async () => {+ ({ ch } = await import('@openpanel/db/src/clickhouse/client'));+ const { bootstrapTestDatabases } = await import(+ '../../../../../../test/bootstrap-databases'+ );+ await bootstrapTestDatabases();+ now = await setupProfileVersions(PROJECT_ID);+ window = { startDate: timeAgo(now, 7), endDate: timeAgo(now, 0) };+}, 30_000);++afterAll(() => teardownProfileVersions(PROJECT_ID));++describe('the chart profile CTE', () => {+ it('matches the FINAL form it replaced, row for row', async () => {+ const activeProfiles = () =>+ profileIdsInWindow({+ projectId: PROJECT_ID,+ eventNames: ['screen_view'],+ window: [+ sql`created_at >= toDateTime(${sql.string(window.startDate)})`,+ ],+ });+ const reference = sql`SELECT id as "profile.id", properties['plan'] as ${compiledText('`profile.properties.plan`')}, email as "profile.email", last_seen_at as "profile.last_seen_at"+ FROM profiles FINAL WHERE project_id = ${sql.string(PROJECT_ID)} AND ${activeProfiles()}`;+ const candidate = profileCte(+ [+ sql`id as "profile.id"`,+ sql`argMax(properties['plan'], last_seen_at) as ${compiledText('`profile.properties.plan`')}`,+ sql`argMax(email, last_seen_at) as "profile.email"`,+ sql`max(last_seen_at) as "profile.last_seen_at"`,+ ],+ PROJECT_ID,+ activeProfiles()+ );+ const result = await expectSameRows(ch, reference, candidate, 3);+ const multi = result+ .map((row) => JSON.parse(row))+ .find((row) => row['profile.id'] === VERSIONED.multi);+ expect(multi).toMatchObject({+ 'profile.properties.plan': 'team',+ 'profile.email': '[email protected]',+ });+ });+});++describe('aggregate and series charts', () => {+ it('breaks events down by the latest plan', async () => {+ const result = await rows(+ aggregateChartQuery({+ ...resolved([breakdown(PLAN)]),+ event: event(),+ ...window,+ })+ );+ expect(countsBy(result, 'label_1')).toEqual({ team: 3, free: 1, pro: 1 });+ });++ it('filters on the latest plan, not a stale version', async () => {+ const result = await rows(+ aggregateChartQuery({+ ...resolved([]),+ event: event({ filters: [planFilter(['pro'])] }),+ ...window,+ })+ );+ expect(countsBy(result, 'label_0')).toEqual({ screen_view: 1 });+ });++ it('reads top-level columns from the latest version', async () => {+ const result = await rows(+ aggregateChartQuery({+ ...resolved([breakdown('profile.email')]),+ event: event(),+ ...window,+ })+ );+ expect(countsBy(result, 'label_1')).toEqual({+ '[email protected]': 3,+ '[email protected]': 1,+ '[email protected]': 1,+ });+ });++ it('selects the whole map from the latest version for wildcard refs', async () => {+ const statement = aggregateChartQuery({+ ...resolved([breakdown('profile.properties.experiments.*.name')]),+ event: event(),+ ...window,+ });+ expect(statement.toStatement().query).toContain(+ 'argMax(properties, last_seen_at) as "profile.properties"'+ );+ const result = await rows(statement);+ expect(countsBy(result, 'label_0')).toEqual({ screen_view: 5 });+ });++ it('series totals agree with the aggregate', async () => {+ const result = await rows(+ chartSeriesQuery({+ ...resolved([breakdown(PLAN)]),+ event: event({ segment: 'user' }),+ interval: 'day',+ timezone: 'UTC',+ ...window,+ })+ );+ expect(countsBy(result, 'label_1')).toEqual({ team: 3, free: 1, pro: 1 });+ });+});++describe('drill-down, funnel and conversion', () => {+ it('drill-down resolves the bucket against the latest plan', async () => {+ const bucket = (daysAgo: number, plan: string) =>+ rows(+ chartBucketProfilesQuery({+ projectId: PROJECT_ID,+ bucketDate: timeAgo(now, daysAgo),+ interval: 'day',+ event: { name: 'screen_view', filters: [] },+ breakdowns: { [PLAN]: plan },+ limit: 10,+ })+ );+ expect((await bucket(1, 'team')).map((row) => row.profile_id)).toEqual([+ VERSIONED.multi,+ ]);+ expect((await bucket(5, 'pro')).map((row) => row.profile_id)).toEqual([+ VERSIONED.single,+ ]);+ expect(await bucket(4, 'pro')).toEqual([]);+ });++ it('funnel breaks down by the latest plan', async () => {+ const base = funnelBase({+ projectId: PROJECT_ID,+ ...window,+ eventSeries: [event(), event({ id: 'B', name: 'signup' })],+ breakdowns: [breakdown(PLAN)],+ funnelWindowMilliseconds: 7 * DAY_MS,+ group: 'profile_id',+ cohortMetadata: new Map(),+ timezone: 'UTC',+ });+ const result = await rows(funnelChartQuery(base));+ expect(+ result.map((row) => `${row.b_0}:${row.level}:${row.count}`).sort()+ ).toEqual(['free:1:1', 'pro:2:1', 'team:2:1']);+ });++ it('conversion breaks down by the latest plan', async () => {+ const expression = getSelectPropertyKey(PLAN, PROJECT_ID);+ const result = await rows(+ conversionQuery({+ projectId: PROJECT_ID,+ ...window,+ interval: 'day',+ group: 'profile_id',+ funnelWindowSeconds: 7 * 24 * 3600,+ firstEventName: 'screen_view',+ secondEventName: 'signup',+ firstEventWhere: null,+ secondEventWhere: null,+ breakdownSelects: [sql`${expression} as b_0`],+ breakdownExpressions: [expression],+ profileJoinFields: ['id', 'properties'],+ needsGroupArrayJoin: false,+ cohortJoins: [],+ })+ );+ const byPlan: Record<string, [number, number]> = {};+ for (const row of result) {+ const plan = String(row.b_0);+ const [first, converted] = byPlan[plan] ?? [0, 0];+ byPlan[plan] = [+ first + Number(row.total_first),+ converted + Number(row.conversions),+ ];+ }+ expect(byPlan).toEqual({ team: [1, 1], pro: [1, 1], free: [1, 0] });+ });+});++describe('profile property autocomplete', () => {+ it('lists the latest values, newest profile first, like FINAL did', async () => {+ const expression = getProfilePropertySelect(PLAN);+ const reference = sql`SELECT distinct ${expression} as values FROM profiles FINAL WHERE project_id = ${sql.string(PROJECT_ID)} AND ${expression} != '' AND ${expression} IS NOT NULL ORDER BY created_at DESC LIMIT 500`;+ const expected = await rows(reference);+ const actual = await rows(+ profilePropertyValuesQuery(PROJECT_ID, expression)+ );+ expect(actual.map((row) => row.values)).toEqual(+ expected.map((row) => row.values)+ );+ expect(actual.map((row) => row.values).sort()).toEqual([+ 'free',+ 'pro',+ 'team',+ ]);+ });+});diff --git a/packages/core/src/modules/event/src/sql.test.ts b/packages/core/src/modules/event/src/sql.test.tsindex 6ae4f75b9..0c3b7738f 100644--- a/packages/core/src/modules/event/src/sql.test.ts+++ b/packages/core/src/modules/event/src/sql.test.ts@@ -178,7 +178,7 @@ describe('eventListQuery', () => {}).toStatement();expect(query).toContain(- 'FROM events e LEFT ANY JOIN (SELECT id, email, properties FROM profiles FINAL WHERE project_id = {p1:String}) as profile on profile.id = profile_id ARRAY JOIN groups AS _group_id LEFT ANY JOIN (SELECT id, name, type, properties FROM groups FINAL WHERE project_id = {p2:String}) AS _g ON _g.id = _group_id WHERE project_id = {p3:String}'+ 'FROM events e LEFT ANY JOIN (SELECT id, argMax(email, last_seen_at) AS email, argMax(properties, last_seen_at) AS properties FROM profiles WHERE project_id = {p1:String} GROUP BY id) as profile on profile.id = profile_id ARRAY JOIN groups AS _group_id LEFT ANY JOIN (SELECT id, name, type, properties FROM groups FINAL WHERE project_id = {p2:String}) AS _g ON _g.id = _group_id WHERE project_id = {p3:String}');expect(query_params).toMatchObject({ p1: PROJECT_ID, p2: PROJECT_ID });@@ -220,7 +220,7 @@ describe('eventsCountQuery', () => {}).toStatement();expect(query).toContain(- 'FROM events e LEFT ANY JOIN (SELECT id, email FROM profiles FINAL WHERE project_id = {p1:String}) as profile on profile.id = profile_id WHERE'+ 'FROM events e LEFT ANY JOIN (SELECT id, argMax(email, last_seen_at) AS email FROM profiles WHERE project_id = {p1:String} GROUP BY id) as profile on profile.id = profile_id WHERE');});});diff --git a/packages/core/src/modules/event/src/sql.ts b/packages/core/src/modules/event/src/sql.tsindex e8e523211..a6c660523 100644--- a/packages/core/src/modules/event/src/sql.ts+++ b/packages/core/src/modules/event/src/sql.ts@@ -83,18 +83,27 @@ export const NO_FILTER_JOINS: EventFilterJoins = {groups: false,};+/** A column as `profiles FINAL` returns it, over a `GROUP BY id`: `last_seen_at` is the version column. */+function latestProfileColumn(column: string): SqlFragment {+ const id = sql.id(column, PROFILE_JOIN_COLUMNS);+ if (column === 'last_seen_at') {+ return sql`max(${id}) AS ${id}`;+ }+ return sql`argMax(${id}, last_seen_at) AS ${id}`;+}+/** The joins the compiled filters need; empty = none. */function filterJoins(projectId: string, joins: EventFilterJoins): SqlFragment {const parts: SqlFragment[] = [];if (joins.profileColumns.length > 0) {const columns = sql.join(- joins.profileColumns.map((column) =>- sql.id(column, PROFILE_JOIN_COLUMNS)- ),+ joins.profileColumns+ .filter((column) => column !== 'id')+ .map(latestProfileColumn),', ');parts.push(- sql`LEFT ANY JOIN (SELECT id, ${columns} FROM ${sql.id(TABLE.profiles)} FINAL WHERE project_id = ${sql.string(projectId)}) as profile on profile.id = profile_id`+ sql`LEFT ANY JOIN (SELECT id, ${columns} FROM ${sql.id(TABLE.profiles)} WHERE project_id = ${sql.string(projectId)} GROUP BY id) as profile on profile.id = profile_id`);}if (joins.groups) {diff --git a/packages/core/src/modules/event/src/sql.versions.test.ts b/packages/core/src/modules/event/src/sql.versions.test.tsnew file mode 100644index 000000000..4e2f4608e--- /dev/null+++ b/packages/core/src/modules/event/src/sql.versions.test.ts@@ -0,0 +1,98 @@+// The events list's profile-filter join no longer reads `profiles FINAL`; it+// groups the raw rows by id and filters on the newest version. Runs against+// profiles with several unmerged versions (test/profile-versions).++import { afterAll, beforeAll, describe, expect, it } from 'bun:test';+import { sql } from '@openpanel/db/src/clickhouse/sql';+import { timeAgo } from '../../../../../../test/fixtures';+import {+ setupProfileVersions,+ teardownProfileVersions,+ VERSIONED,+} from '../../../../../../test/profile-versions';+import { expectSameRows, fetchRows } from '../../../../test/clickhouse-rows';+import { getEventFiltersWhereClause } from '../../chart/src/filter-where';+import type { IChartEventFilter } from '../../report/report.constants';+import { compiledFilterFragments } from './filter-clauses';+import { eventListQuery } from './sql';++const PROJECT_ID = 'event-profile-versions';+const COLUMNS = ['id', 'name', 'profile_id'] as const;++let ch: typeof import('@openpanel/db/src/clickhouse/client').ch;+let now: Date;++function listQuery(filters: IChartEventFilter[], profileColumns: string[]) {+ return eventListQuery({+ projectId: PROJECT_ID,+ columns: COLUMNS,+ take: 50,+ startDate: new Date(now.getTime() - 7 * 24 * 60 * 60 * 1000),+ endDate: now,+ filterClauses: getEventFiltersWhereClause(filters, PROJECT_ID, 'e'),+ joins: { profileColumns, groups: false },+ });+}++async function profileIds(+ filters: IChartEventFilter[],+ profileColumns: string[]+): Promise<string[]> {+ const rows = await fetchRows(ch, listQuery(filters, profileColumns));+ return [...new Set(rows.map((row) => JSON.parse(row).profile_id))].sort();+}++const planIs = (value: string): IChartEventFilter => ({+ id: 'f',+ name: 'profile.properties.plan',+ operator: 'is',+ value: [value],+});+const emailIs = (value: string): IChartEventFilter => ({+ id: 'f',+ name: 'profile.email',+ operator: 'is',+ value: [value],+});++beforeAll(async () => {+ ({ ch } = await import('@openpanel/db/src/clickhouse/client'));+ const { bootstrapTestDatabases } = await import(+ '../../../../../../test/bootstrap-databases'+ );+ await bootstrapTestDatabases();+ now = await setupProfileVersions(PROJECT_ID);+}, 30_000);++afterAll(() => teardownProfileVersions(PROJECT_ID));++describe('eventListQuery with profile filters', () => {+ it('filters on the latest plan, not a stale version', async () => {+ expect(await profileIds([planIs('pro')], ['properties'])).toEqual([+ VERSIONED.single,+ ]);+ expect(await profileIds([planIs('team')], ['properties'])).toEqual([+ VERSIONED.multi,+ ]);+ expect(await profileIds([planIs('free')], ['properties'])).toEqual([+ VERSIONED.staleMatch,+ ]);+ });++ it('filters top-level columns on the latest version', async () => {+ expect(await profileIds([emailIs('[email protected]')], ['email'])).toEqual([+ VERSIONED.multi,+ ]);+ expect(await profileIds([emailIs('[email protected]')], ['email'])).toEqual(+ []+ );+ });++ it('matches the FINAL form it replaced, row for row', async () => {+ const filters = [planIs('team')];+ const clauses = getEventFiltersWhereClause(filters, PROJECT_ID, 'e');+ const window = sql`created_at >= toDateTime(${sql.string(timeAgo(now, 7))}) AND created_at <= toDateTime(${sql.string(timeAgo(now, 0))})`;+ const reference = sql`SELECT id, name, profile_id FROM events e LEFT ANY JOIN (SELECT id, properties FROM profiles FINAL WHERE project_id = ${sql.string(PROJECT_ID)}) as profile on profile.id = profile_id WHERE project_id = ${sql.string(PROJECT_ID)} AND ${window} AND ${sql.join(compiledFilterFragments(clauses), ' AND ')}`;+ await expectSameRows(ch, reference, listQuery(filters, ['properties']), 4);+ });+});diff --git a/test/profile-versions.ts b/test/profile-versions.tsindex 908d62dde..414351fd5 100644--- a/test/profile-versions.ts+++ b/test/profile-versions.ts@@ -56,7 +56,12 @@ function profileVersions(projectId: string, now: Date): Version[][] {last_name: 'Kim-Lee',email: '[email protected]',is_external: true,- properties: { plan: 'team', country: 'SE', tier: 'gold' },+ properties: {+ plan: 'team',+ country: 'SE',+ tier: 'gold',+ 'experiments.checkout.name': 'one-page',+ },groups: [VERSIONED.groups.g1, VERSIONED.groups.g2],last_seen_at: timeAgo(now, 2),},