Skip to content

OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.

perf(api): chart, funnel, conversion and event-list profile joins drop FINAL

Carl-Gerhard Lindesvärddcf07875ad9a
diff --git a/packages/core/src/modules/chart/src/conversion.sql.ts b/packages/core/src/modules/chart/src/conversion.sql.ts
index efa959d3a..e91213cef 100644
--- a/packages/core/src/modules/chart/src/conversion.sql.ts
+++ b/packages/core/src/modules/chart/src/conversion.sql.ts
@@ -10,6 +10,7 @@ import { compiledText } from './compiled';
import {
buildGroupsQuery,
CHART_TABLE,
+ latestProfileValue,
profileIdsInWindow,
} from './field-resolution';
@@ -74,11 +75,17 @@ function profileJoin(
if (fields.length === 0) {
return sql.empty;
}
- // Simple column names (not aliased) so profile.properties works directly.
+ // Aliased back to the column name so profile.properties works directly.
+ const columns = fields.map((field) =>
+ field === 'id'
+ ? sql.id(field)
+ : sql`${latestProfileValue(field, sql.id(field))} AS ${sql.id(field)}`
+ );
return sql`LEFT ANY JOIN (
- SELECT ${sql.join(fields.map((field) => sql.id(field)))}
- FROM ${sql.id(CHART_TABLE.profiles)} FINAL
+ SELECT ${sql.join(columns)}
+ FROM ${sql.id(CHART_TABLE.profiles)}
WHERE project_id = ${sql.string(projectId)} AND ${activeProfiles}
+ GROUP BY id
) as profile ON profile.id = profile_id`;
}
diff --git a/packages/core/src/modules/chart/src/field-resolution.ts b/packages/core/src/modules/chart/src/field-resolution.ts
index d8b5f74bb..1ec1e5617 100644
--- a/packages/core/src/modules/chart/src/field-resolution.ts
+++ b/packages/core/src/modules/chart/src/field-resolution.ts
@@ -480,6 +480,24 @@ export function getSelectPropertyKey(
return sql`${map}[${sql.string(property.replace(new RegExp(`^${match}.`), ''))}]`;
}
+/**
+ * `column` as `profiles FINAL` would return it, for a `GROUP BY id` over the
+ * raw rows: `last_seen_at` is the ReplacingMergeTree version column, so the
+ * value from the row with the greatest one is the surviving version's.
+ */
+export function latestProfileValue(
+ name: string,
+ column: SqlFragment
+): SqlFragment {
+ if (name === 'id') {
+ return column;
+ }
+ if (name === 'last_seen_at') {
+ return sql`max(${column})`;
+ }
+ return sql`argMax(${column}, last_seen_at)`;
+}
+
// profile.properties.<key> refs render as Map lookups `profile.properties['<key>']`.
// Pulling the whole `properties` Map into the profile CTE makes the LEFT ANY
// JOIN hash carry the full Map per profile — roughly a kilobyte each on real
@@ -530,10 +548,10 @@ export function profilePropertiesCteSelect(
// could break out of the backticks.
const cols = keys.map(
(k) =>
- sql`properties[${sql.string(k)}] as ${compiledText(`\`profile.properties.${k}\``)}`
+ sql`argMax(properties[${sql.string(k)}], last_seen_at) as ${compiledText(`\`profile.properties.${k}\``)}`
);
if (needsFullMap || cols.length === 0) {
- cols.push(sql`properties as "profile.properties"`);
+ cols.push(sql`argMax(properties, last_seen_at) as "profile.properties"`);
}
return sql.join(cols, ', ');
}
diff --git a/packages/core/src/modules/chart/src/funnel.sql.test.ts b/packages/core/src/modules/chart/src/funnel.sql.test.ts
index 431b74047..a9fc6623d 100644
--- a/packages/core/src/modules/chart/src/funnel.sql.test.ts
+++ b/packages/core/src/modules/chart/src/funnel.sql.test.ts
@@ -175,7 +175,9 @@ describe('funnel.sql / buildFunnelBase — profile breakdowns', () => {
// The breakdown is narrowed to a scalar alias selected by the profile
// join, so both the join and the aliased column must exist.
- expect(text).toContain("properties['plan'] as `profile.properties.plan`");
+ expect(text).toContain(
+ "argMax(properties['plan'], last_seen_at) as `profile.properties.plan`"
+ );
expect(sql).toMatch(/as profile/);
expect(sql).toContain('profile.id = events.profile_id');
await explain(statement);
@@ -186,7 +188,7 @@ describe('funnel.sql / buildFunnelBase — profile breakdowns', () => {
breakdowns: [breakdown('profile.properties.plan')],
});
expect(render(statement).sql).toMatch(
- /SELECT [^)]*properties[^)]*FROM profiles/
+ /SELECT .*properties.*FROM profiles/
);
await explain(statement);
});
@@ -419,9 +421,11 @@ describe('funnel.sql / profile-property narrowing', () => {
});
const { text } = render(statement);
- expect(text).toContain("properties['plan'] as `profile.properties.plan`");
expect(text).toContain(
- "properties['experiment'] as `profile.properties.experiment`"
+ "argMax(properties['plan'], last_seen_at) as `profile.properties.plan`"
+ );
+ expect(text).toContain(
+ "argMax(properties['experiment'], last_seen_at) as `profile.properties.experiment`"
);
expect(text).not.toContain('properties as "profile.properties"');
// Conditions (windowFunnel + pre-filter) and the breakdown expression are
diff --git a/packages/core/src/modules/chart/src/funnel.sql.ts b/packages/core/src/modules/chart/src/funnel.sql.ts
index 8ded7c2ef..fa87cb289 100644
--- a/packages/core/src/modules/chart/src/funnel.sql.ts
+++ b/packages/core/src/modules/chart/src/funnel.sql.ts
@@ -27,6 +27,7 @@ import {
getSelectPropertyKey,
isAllCohortsBreakdown,
isKnownEventField,
+ latestProfileValue,
profileIdsInWindow,
profilePropertiesCteSelect,
} from './field-resolution';
@@ -217,7 +218,9 @@ function profileCteColumns(
// The allowlist is the backstop that makes `sql.id` throw rather than inline
// if either loop above lets something else through (GHSA-pc3q-gw7f-p2x2).
const columns: SqlFragment[] = Array.from(fields).map((field) =>
- sql.id(field, FUNNEL_JOIN_COLUMNS)
+ field === 'id'
+ ? sql.id(field, FUNNEL_JOIN_COLUMNS)
+ : sql`${latestProfileValue(field, sql.id(field, FUNNEL_JOIN_COLUMNS))} AS ${sql.id(field, FUNNEL_JOIN_COLUMNS)}`
);
const referencesProperties =
profileFilters.some((filter) => filter.startsWith('properties')) ||
@@ -303,8 +306,8 @@ export function funnelBase(input: FunnelBaseInput): FunnelBase {
});
const profileJoin =
columns.length > 0
- ? sql` LEFT JOIN (SELECT ${sql.join(columns, ', ')} FROM ${sql.id(CHART_TABLE.profiles)} FINAL
- WHERE project_id = ${sql.string(projectId)} AND ${activeProfiles}) as profile ON profile.id = events.profile_id`
+ ? sql` LEFT JOIN (SELECT ${sql.join(columns, ', ')} FROM ${sql.id(CHART_TABLE.profiles)}
+ WHERE project_id = ${sql.string(projectId)} AND ${activeProfiles} GROUP BY id) as profile ON profile.id = events.profile_id`
: sql.empty;
const groupJoin = needsGroupArrayJoin
? sql` ARRAY JOIN groups AS _group_id LEFT ANY JOIN _g ON _g.id = _group_id`
diff --git a/packages/core/src/modules/chart/src/property-key-escaping.test.ts b/packages/core/src/modules/chart/src/property-key-escaping.test.ts
index f2e367280..67652cc65 100644
--- a/packages/core/src/modules/chart/src/property-key-escaping.test.ts
+++ b/packages/core/src/modules/chart/src/property-key-escaping.test.ts
@@ -277,7 +277,7 @@ describe('profile-property narrowing with a quoted key', () => {
fieldResolution.profilePropertiesCteSelect(keys, needsFullMap)
);
expect(cteSelect.query).toBe(
- "properties[{p1:String}] as `profile.properties.pl'an`"
+ "argMax(properties[{p1:String}], last_seen_at) as `profile.properties.pl'an`"
);
expect(cteSelect.query_params).toEqual({ p1: key });
@@ -302,7 +302,9 @@ describe('profile-property narrowing with a quoted key', () => {
expect(
statement(fieldResolution.profilePropertiesCteSelect(keys, needsFullMap))
.query
- ).toBe('properties as "profile.properties"');
+ ).toBe(
+ 'argMax(properties, last_seen_at) as "profile.properties"'
+ );
});
it('leaves ordinary keys narrowing exactly as before', () => {
@@ -314,7 +316,9 @@ describe('profile-property narrowing with a quoted key', () => {
expect(
statement(fieldResolution.profilePropertiesCteSelect(keys, needsFullMap))
.query
- ).toBe('properties[{p1:String}] as `profile.properties.plan`');
+ ).toBe(
+ 'argMax(properties[{p1:String}], last_seen_at) as `profile.properties.plan`'
+ );
expect(
statement(
compiled.fragmentWithProfileRefs(
diff --git a/packages/core/src/modules/chart/src/sql.test.ts b/packages/core/src/modules/chart/src/sql.test.ts
index 6a2ce6f7d..feb3d6222 100644
--- a/packages/core/src/modules/chart/src/sql.test.ts
+++ b/packages/core/src/modules/chart/src/sql.test.ts
@@ -456,12 +456,14 @@ describe('profile-property narrowing', () => {
});
// The CTE selects one scalar column per referenced key...
- expect(text).toContain("properties['plan'] as `profile.properties.plan`");
expect(text).toContain(
- "properties['experiment'] as `profile.properties.experiment`"
+ "argMax(properties['plan'], last_seen_at) as `profile.properties.plan`"
+ );
+ expect(text).toContain(
+ "argMax(properties['experiment'], last_seen_at) as `profile.properties.experiment`"
);
// ...instead of every profile's whole Map...
- expect(text).not.toContain('properties as "profile.properties"');
+ expect(text).not.toContain('as "profile.properties"');
// ...and every ref in the query is rewritten to the scalar alias.
expect(text).not.toContain("profile.properties['plan']");
expect(text).not.toContain("profile.properties['experiment']");
@@ -484,7 +486,9 @@ describe('profile-property narrowing', () => {
});
// mapExtractKeyLike needs the whole Map, so it must stay selected.
- expect(text).toContain('properties as "profile.properties"');
+ expect(text).toContain(
+ 'argMax(properties, last_seen_at) as "profile.properties"'
+ );
});
it('never narrows identifier-unsafe keys (backtick falls back to the Map)', async () => {
@@ -505,7 +509,9 @@ describe('profile-property narrowing', () => {
// The unsafe key keeps its original Map access against the full Map —
// it must never be embedded in a backtick-quoted alias.
- expect(text).toContain('properties as "profile.properties"');
+ expect(text).toContain(
+ 'argMax(properties, last_seen_at) as "profile.properties"'
+ );
expect(text).not.toContain('as `profile.properties.plan`tier`');
});
@@ -522,7 +528,9 @@ describe('profile-property narrowing', () => {
// The metric's key must be narrowed alongside the filter's — otherwise
// the metric keeps reading the Map that narrowing just removed.
- expect(text).toContain("properties['age'] as `profile.properties.age`");
+ expect(text).toContain(
+ "argMax(properties['age'], last_seen_at) as `profile.properties.age`"
+ );
expect(text).not.toContain("profile.properties['age']");
});
@@ -540,7 +548,9 @@ describe('profile-property narrowing', () => {
});
expect(text).toContain('LEFT ANY JOIN profile ON profile.id = profile_id');
- expect(text).toContain("properties['age'] as `profile.properties.age`");
+ expect(text).toContain(
+ "argMax(properties['age'], last_seen_at) as `profile.properties.age`"
+ );
});
it('metric-only profile property parses and resolves', async () => {
diff --git a/packages/core/src/modules/chart/src/sql.ts b/packages/core/src/modules/chart/src/sql.ts
index 604a97ef4..0f665bee6 100644
--- a/packages/core/src/modules/chart/src/sql.ts
+++ b/packages/core/src/modules/chart/src/sql.ts
@@ -37,6 +37,7 @@ import {
isAllCohortsBreakdown,
isKnownEventField,
isNumericColumn,
+ latestProfileValue,
PROFILE_CTE_FIELDS,
profileIdsInWindow,
profilePropertiesCteSelect,
@@ -209,17 +210,18 @@ function profileCteSelectField(
}
// The alias is `"profile.<field>"` — two dots, so text, not `sql.id`.
// `collectProfileCteFields` only ever yields a name from its own closed set.
- return sql`${sql.id(field, PROFILE_CTE_FIELDS)} as ${compiledText(`"profile.${field}"`)}`;
+ return sql`${latestProfileValue(field, sql.id(field, PROFILE_CTE_FIELDS))} as ${compiledText(`"profile.${field}"`)}`;
}
-function profileCte(
+export function profileCte(
selectFields: SqlFragment[],
projectId: string,
activeProfiles: SqlFragment
): SqlFragment {
return sql`SELECT ${sql.join(selectFields, ', ')}
- FROM ${sql.id(CHART_TABLE.profiles)} FINAL
- WHERE project_id = ${sql.string(projectId)} AND ${activeProfiles}`;
+ FROM ${sql.id(CHART_TABLE.profiles)}
+ WHERE project_id = ${sql.string(projectId)} AND ${activeProfiles}
+ GROUP BY id`;
}
function anyRefOn(prefix: string, refs: { name: string }[]): boolean {
@@ -633,7 +635,7 @@ export function profilePropertyValuesQuery(
projectId: string,
expression: SqlFragment
): SqlFragment {
- return sql`SELECT distinct ${expression} as values FROM ${sql.id(CHART_TABLE.profiles)} FINAL WHERE project_id = ${sql.string(projectId)} AND ${expression} != '' AND ${expression} IS NOT NULL ORDER BY created_at DESC LIMIT ${sql.uint64(PROPERTY_VALUES_LIMIT)}`;
+ return sql`SELECT values FROM (SELECT argMax(${expression}, last_seen_at) as values, argMax(created_at, last_seen_at) as created_at FROM ${sql.id(CHART_TABLE.profiles)} WHERE project_id = ${sql.string(projectId)} GROUP BY id) WHERE values != '' AND values IS NOT NULL GROUP BY values ORDER BY max(created_at) DESC LIMIT ${sql.uint64(PROPERTY_VALUES_LIMIT)}`;
}
/** Distinct values of a `group.*` field straight off the groups table. */
@@ -673,17 +675,20 @@ function bucketEquals(interval: IInterval, bucketDate: string): SqlFragment {
}
}
-/** Top-level profile columns a `profile.*` reference reads, in first-seen order. */
+/** Top-level profile columns a `profile.*` reference reads, in first-seen order, besides `id`. */
function profileFieldsToSelect(refs: string[]): SqlFragment[] {
const fields: string[] = [];
for (const ref of refs) {
const field = ref.replace('profile.', '').split('.')[0];
- if (field && !fields.includes(field)) {
+ if (field && field !== 'id' && !fields.includes(field)) {
fields.push(field);
}
}
// `sql.id` refuses any field not in PROFILE_COLUMNS rather than splicing it.
- return fields.map((field) => sql.id(field, PROFILE_COLUMNS));
+ return fields.map(
+ (field) =>
+ sql`${latestProfileValue(field, sql.id(field, PROFILE_COLUMNS))} AS ${sql.id(field, PROFILE_COLUMNS)}`
+ );
}
/** Distinct profile ids behind one chart data point. */
@@ -718,7 +723,7 @@ export function chartBucketProfilesQuery(
eventNames: [event.name],
window: [dateRange],
});
- joins.profiles = sql`LEFT ANY JOIN (SELECT id, ${fields} FROM ${sql.id(CHART_TABLE.profiles)} FINAL WHERE project_id = ${sql.string(projectId)} AND ${activeProfiles}) as profile on profile.id = profile_id`;
+ joins.profiles = sql`LEFT ANY JOIN (SELECT id, ${fields} FROM ${sql.id(CHART_TABLE.profiles)} WHERE project_id = ${sql.string(projectId)} AND ${activeProfiles} GROUP BY id) as profile on profile.id = profile_id`;
}
const needsGroupJoin = [
diff --git a/packages/core/src/modules/chart/src/sql.versions.test.ts b/packages/core/src/modules/chart/src/sql.versions.test.ts
new file mode 100644
index 000000000..685020d36
--- /dev/null
+++ b/packages/core/src/modules/chart/src/sql.versions.test.ts
@@ -0,0 +1,278 @@
+// The profile joins no longer read `profiles FINAL`: every statement groups
+// the raw rows by id and takes each column from the newest version. These
+// run against profiles with several unmerged versions (test/profile-versions)
+// and assert the latest version is what every chart shape sees — the stale
+// version with plan=pro must never match, the other project must never leak.
+
+import { afterAll, beforeAll, describe, expect, it } from 'bun:test';
+import { sql } from '@openpanel/db/src/clickhouse/sql';
+import { timeAgo } from '../../../../../../test/fixtures';
+import {
+ setupProfileVersions,
+ teardownProfileVersions,
+ VERSIONED,
+} from '../../../../../../test/profile-versions';
+import { expectSameRows, fetchRows } from '../../../../test/clickhouse-rows';
+import type {
+ IChartBreakdown,
+ IChartEvent,
+} from '../../report/report.constants';
+import { compiledText } from './compiled';
+import { conversionQuery } from './conversion.sql';
+import {
+ getProfilePropertySelect,
+ getSelectPropertyKey,
+ profileIdsInWindow,
+} from './field-resolution';
+import { funnelBase, funnelChartQuery } from './funnel.sql';
+import {
+ aggregateChartQuery,
+ chartBucketProfilesQuery,
+ chartSeriesQuery,
+ profileCte,
+ profilePropertyValuesQuery,
+} from './sql';
+
+const PROJECT_ID = 'chart-profile-versions';
+const DAY_MS = 24 * 60 * 60 * 1000;
+const PLAN = 'profile.properties.plan';
+const planFilter = (value: string[]) => ({
+ id: 'f',
+ name: PLAN,
+ operator: 'is' as const,
+ value,
+});
+
+let ch: typeof import('@openpanel/db/src/clickhouse/client').ch;
+let now: Date;
+let window: { startDate: string; endDate: string };
+
+const event = (overrides: Partial<IChartEvent> = {}): IChartEvent => ({
+ id: 'A',
+ name: 'screen_view',
+ segment: 'event',
+ filters: [],
+ ...overrides,
+});
+const breakdown = (name: string): IChartBreakdown => ({ id: name, name });
+const resolved = (breakdowns: IChartBreakdown[]) => ({
+ breakdowns,
+ allCohorts: [],
+ cohortMetadata: new Map(),
+ projectId: PROJECT_ID,
+});
+
+type Row = Record<string, string | number | string[]>;
+async function rows(fragment: Parameters<typeof fetchRows>[1]): Promise<Row[]> {
+ return (await fetchRows(ch, fragment)).map((row) => JSON.parse(row));
+}
+function countsBy(result: Row[], label: string): Record<string, number> {
+ const counts: Record<string, number> = {};
+ // WITH FILL pads a series with zero rows that carry no label.
+ for (const row of result.filter((row) => Number(row.count) > 0)) {
+ const key = String(row[label]);
+ counts[key] = (counts[key] ?? 0) + Number(row.count);
+ }
+ return counts;
+}
+
+beforeAll(async () => {
+ ({ ch } = await import('@openpanel/db/src/clickhouse/client'));
+ const { bootstrapTestDatabases } = await import(
+ '../../../../../../test/bootstrap-databases'
+ );
+ await bootstrapTestDatabases();
+ now = await setupProfileVersions(PROJECT_ID);
+ window = { startDate: timeAgo(now, 7), endDate: timeAgo(now, 0) };
+}, 30_000);
+
+afterAll(() => teardownProfileVersions(PROJECT_ID));
+
+describe('the chart profile CTE', () => {
+ it('matches the FINAL form it replaced, row for row', async () => {
+ const activeProfiles = () =>
+ profileIdsInWindow({
+ projectId: PROJECT_ID,
+ eventNames: ['screen_view'],
+ window: [
+ sql`created_at >= toDateTime(${sql.string(window.startDate)})`,
+ ],
+ });
+ const reference = sql`SELECT id as "profile.id", properties['plan'] as ${compiledText('`profile.properties.plan`')}, email as "profile.email", last_seen_at as "profile.last_seen_at"
+ FROM profiles FINAL WHERE project_id = ${sql.string(PROJECT_ID)} AND ${activeProfiles()}`;
+ const candidate = profileCte(
+ [
+ sql`id as "profile.id"`,
+ sql`argMax(properties['plan'], last_seen_at) as ${compiledText('`profile.properties.plan`')}`,
+ sql`argMax(email, last_seen_at) as "profile.email"`,
+ sql`max(last_seen_at) as "profile.last_seen_at"`,
+ ],
+ PROJECT_ID,
+ activeProfiles()
+ );
+ const result = await expectSameRows(ch, reference, candidate, 3);
+ const multi = result
+ .map((row) => JSON.parse(row))
+ .find((row) => row['profile.id'] === VERSIONED.multi);
+ expect(multi).toMatchObject({
+ 'profile.properties.plan': 'team',
+ 'profile.email': '[email protected]',
+ });
+ });
+});
+
+describe('aggregate and series charts', () => {
+ it('breaks events down by the latest plan', async () => {
+ const result = await rows(
+ aggregateChartQuery({
+ ...resolved([breakdown(PLAN)]),
+ event: event(),
+ ...window,
+ })
+ );
+ expect(countsBy(result, 'label_1')).toEqual({ team: 3, free: 1, pro: 1 });
+ });
+
+ it('filters on the latest plan, not a stale version', async () => {
+ const result = await rows(
+ aggregateChartQuery({
+ ...resolved([]),
+ event: event({ filters: [planFilter(['pro'])] }),
+ ...window,
+ })
+ );
+ expect(countsBy(result, 'label_0')).toEqual({ screen_view: 1 });
+ });
+
+ it('reads top-level columns from the latest version', async () => {
+ const result = await rows(
+ aggregateChartQuery({
+ ...resolved([breakdown('profile.email')]),
+ event: event(),
+ ...window,
+ })
+ );
+ expect(countsBy(result, 'label_1')).toEqual({
+ });
+ });
+
+ it('selects the whole map from the latest version for wildcard refs', async () => {
+ const statement = aggregateChartQuery({
+ ...resolved([breakdown('profile.properties.experiments.*.name')]),
+ event: event(),
+ ...window,
+ });
+ expect(statement.toStatement().query).toContain(
+ 'argMax(properties, last_seen_at) as "profile.properties"'
+ );
+ const result = await rows(statement);
+ expect(countsBy(result, 'label_0')).toEqual({ screen_view: 5 });
+ });
+
+ it('series totals agree with the aggregate', async () => {
+ const result = await rows(
+ chartSeriesQuery({
+ ...resolved([breakdown(PLAN)]),
+ event: event({ segment: 'user' }),
+ interval: 'day',
+ timezone: 'UTC',
+ ...window,
+ })
+ );
+ expect(countsBy(result, 'label_1')).toEqual({ team: 3, free: 1, pro: 1 });
+ });
+});
+
+describe('drill-down, funnel and conversion', () => {
+ it('drill-down resolves the bucket against the latest plan', async () => {
+ const bucket = (daysAgo: number, plan: string) =>
+ rows(
+ chartBucketProfilesQuery({
+ projectId: PROJECT_ID,
+ bucketDate: timeAgo(now, daysAgo),
+ interval: 'day',
+ event: { name: 'screen_view', filters: [] },
+ breakdowns: { [PLAN]: plan },
+ limit: 10,
+ })
+ );
+ expect((await bucket(1, 'team')).map((row) => row.profile_id)).toEqual([
+ VERSIONED.multi,
+ ]);
+ expect((await bucket(5, 'pro')).map((row) => row.profile_id)).toEqual([
+ VERSIONED.single,
+ ]);
+ expect(await bucket(4, 'pro')).toEqual([]);
+ });
+
+ it('funnel breaks down by the latest plan', async () => {
+ const base = funnelBase({
+ projectId: PROJECT_ID,
+ ...window,
+ eventSeries: [event(), event({ id: 'B', name: 'signup' })],
+ breakdowns: [breakdown(PLAN)],
+ funnelWindowMilliseconds: 7 * DAY_MS,
+ group: 'profile_id',
+ cohortMetadata: new Map(),
+ timezone: 'UTC',
+ });
+ const result = await rows(funnelChartQuery(base));
+ expect(
+ result.map((row) => `${row.b_0}:${row.level}:${row.count}`).sort()
+ ).toEqual(['free:1:1', 'pro:2:1', 'team:2:1']);
+ });
+
+ it('conversion breaks down by the latest plan', async () => {
+ const expression = getSelectPropertyKey(PLAN, PROJECT_ID);
+ const result = await rows(
+ conversionQuery({
+ projectId: PROJECT_ID,
+ ...window,
+ interval: 'day',
+ group: 'profile_id',
+ funnelWindowSeconds: 7 * 24 * 3600,
+ firstEventName: 'screen_view',
+ secondEventName: 'signup',
+ firstEventWhere: null,
+ secondEventWhere: null,
+ breakdownSelects: [sql`${expression} as b_0`],
+ breakdownExpressions: [expression],
+ profileJoinFields: ['id', 'properties'],
+ needsGroupArrayJoin: false,
+ cohortJoins: [],
+ })
+ );
+ const byPlan: Record<string, [number, number]> = {};
+ for (const row of result) {
+ const plan = String(row.b_0);
+ const [first, converted] = byPlan[plan] ?? [0, 0];
+ byPlan[plan] = [
+ first + Number(row.total_first),
+ converted + Number(row.conversions),
+ ];
+ }
+ expect(byPlan).toEqual({ team: [1, 1], pro: [1, 1], free: [1, 0] });
+ });
+});
+
+describe('profile property autocomplete', () => {
+ it('lists the latest values, newest profile first, like FINAL did', async () => {
+ const expression = getProfilePropertySelect(PLAN);
+ const reference = sql`SELECT distinct ${expression} as values FROM profiles FINAL WHERE project_id = ${sql.string(PROJECT_ID)} AND ${expression} != '' AND ${expression} IS NOT NULL ORDER BY created_at DESC LIMIT 500`;
+ const expected = await rows(reference);
+ const actual = await rows(
+ profilePropertyValuesQuery(PROJECT_ID, expression)
+ );
+ expect(actual.map((row) => row.values)).toEqual(
+ expected.map((row) => row.values)
+ );
+ expect(actual.map((row) => row.values).sort()).toEqual([
+ 'free',
+ 'pro',
+ 'team',
+ ]);
+ });
+});
diff --git a/packages/core/src/modules/event/src/sql.test.ts b/packages/core/src/modules/event/src/sql.test.ts
index 6ae4f75b9..0c3b7738f 100644
--- a/packages/core/src/modules/event/src/sql.test.ts
+++ b/packages/core/src/modules/event/src/sql.test.ts
@@ -178,7 +178,7 @@ describe('eventListQuery', () => {
}).toStatement();
expect(query).toContain(
- 'FROM events e LEFT ANY JOIN (SELECT id, email, properties FROM profiles FINAL WHERE project_id = {p1:String}) as profile on profile.id = profile_id ARRAY JOIN groups AS _group_id LEFT ANY JOIN (SELECT id, name, type, properties FROM groups FINAL WHERE project_id = {p2:String}) AS _g ON _g.id = _group_id WHERE project_id = {p3:String}'
+ 'FROM events e LEFT ANY JOIN (SELECT id, argMax(email, last_seen_at) AS email, argMax(properties, last_seen_at) AS properties FROM profiles WHERE project_id = {p1:String} GROUP BY id) as profile on profile.id = profile_id ARRAY JOIN groups AS _group_id LEFT ANY JOIN (SELECT id, name, type, properties FROM groups FINAL WHERE project_id = {p2:String}) AS _g ON _g.id = _group_id WHERE project_id = {p3:String}'
);
expect(query_params).toMatchObject({ p1: PROJECT_ID, p2: PROJECT_ID });
@@ -220,7 +220,7 @@ describe('eventsCountQuery', () => {
}).toStatement();
expect(query).toContain(
- 'FROM events e LEFT ANY JOIN (SELECT id, email FROM profiles FINAL WHERE project_id = {p1:String}) as profile on profile.id = profile_id WHERE'
+ 'FROM events e LEFT ANY JOIN (SELECT id, argMax(email, last_seen_at) AS email FROM profiles WHERE project_id = {p1:String} GROUP BY id) as profile on profile.id = profile_id WHERE'
);
});
});
diff --git a/packages/core/src/modules/event/src/sql.ts b/packages/core/src/modules/event/src/sql.ts
index e8e523211..a6c660523 100644
--- a/packages/core/src/modules/event/src/sql.ts
+++ b/packages/core/src/modules/event/src/sql.ts
@@ -83,18 +83,27 @@ export const NO_FILTER_JOINS: EventFilterJoins = {
groups: false,
};
+/** A column as `profiles FINAL` returns it, over a `GROUP BY id`: `last_seen_at` is the version column. */
+function latestProfileColumn(column: string): SqlFragment {
+ const id = sql.id(column, PROFILE_JOIN_COLUMNS);
+ if (column === 'last_seen_at') {
+ return sql`max(${id}) AS ${id}`;
+ }
+ return sql`argMax(${id}, last_seen_at) AS ${id}`;
+}
+
/** The joins the compiled filters need; empty = none. */
function filterJoins(projectId: string, joins: EventFilterJoins): SqlFragment {
const parts: SqlFragment[] = [];
if (joins.profileColumns.length > 0) {
const columns = sql.join(
- joins.profileColumns.map((column) =>
- sql.id(column, PROFILE_JOIN_COLUMNS)
- ),
+ joins.profileColumns
+ .filter((column) => column !== 'id')
+ .map(latestProfileColumn),
', '
);
parts.push(
- sql`LEFT ANY JOIN (SELECT id, ${columns} FROM ${sql.id(TABLE.profiles)} FINAL WHERE project_id = ${sql.string(projectId)}) as profile on profile.id = profile_id`
+ sql`LEFT ANY JOIN (SELECT id, ${columns} FROM ${sql.id(TABLE.profiles)} WHERE project_id = ${sql.string(projectId)} GROUP BY id) as profile on profile.id = profile_id`
);
}
if (joins.groups) {
diff --git a/packages/core/src/modules/event/src/sql.versions.test.ts b/packages/core/src/modules/event/src/sql.versions.test.ts
new file mode 100644
index 000000000..4e2f4608e
--- /dev/null
+++ b/packages/core/src/modules/event/src/sql.versions.test.ts
@@ -0,0 +1,98 @@
+// The events list's profile-filter join no longer reads `profiles FINAL`; it
+// groups the raw rows by id and filters on the newest version. Runs against
+// profiles with several unmerged versions (test/profile-versions).
+
+import { afterAll, beforeAll, describe, expect, it } from 'bun:test';
+import { sql } from '@openpanel/db/src/clickhouse/sql';
+import { timeAgo } from '../../../../../../test/fixtures';
+import {
+ setupProfileVersions,
+ teardownProfileVersions,
+ VERSIONED,
+} from '../../../../../../test/profile-versions';
+import { expectSameRows, fetchRows } from '../../../../test/clickhouse-rows';
+import { getEventFiltersWhereClause } from '../../chart/src/filter-where';
+import type { IChartEventFilter } from '../../report/report.constants';
+import { compiledFilterFragments } from './filter-clauses';
+import { eventListQuery } from './sql';
+
+const PROJECT_ID = 'event-profile-versions';
+const COLUMNS = ['id', 'name', 'profile_id'] as const;
+
+let ch: typeof import('@openpanel/db/src/clickhouse/client').ch;
+let now: Date;
+
+function listQuery(filters: IChartEventFilter[], profileColumns: string[]) {
+ return eventListQuery({
+ projectId: PROJECT_ID,
+ columns: COLUMNS,
+ take: 50,
+ startDate: new Date(now.getTime() - 7 * 24 * 60 * 60 * 1000),
+ endDate: now,
+ filterClauses: getEventFiltersWhereClause(filters, PROJECT_ID, 'e'),
+ joins: { profileColumns, groups: false },
+ });
+}
+
+async function profileIds(
+ filters: IChartEventFilter[],
+ profileColumns: string[]
+): Promise<string[]> {
+ const rows = await fetchRows(ch, listQuery(filters, profileColumns));
+ return [...new Set(rows.map((row) => JSON.parse(row).profile_id))].sort();
+}
+
+const planIs = (value: string): IChartEventFilter => ({
+ id: 'f',
+ name: 'profile.properties.plan',
+ operator: 'is',
+ value: [value],
+});
+const emailIs = (value: string): IChartEventFilter => ({
+ id: 'f',
+ name: 'profile.email',
+ operator: 'is',
+ value: [value],
+});
+
+beforeAll(async () => {
+ ({ ch } = await import('@openpanel/db/src/clickhouse/client'));
+ const { bootstrapTestDatabases } = await import(
+ '../../../../../../test/bootstrap-databases'
+ );
+ await bootstrapTestDatabases();
+ now = await setupProfileVersions(PROJECT_ID);
+}, 30_000);
+
+afterAll(() => teardownProfileVersions(PROJECT_ID));
+
+describe('eventListQuery with profile filters', () => {
+ it('filters on the latest plan, not a stale version', async () => {
+ expect(await profileIds([planIs('pro')], ['properties'])).toEqual([
+ VERSIONED.single,
+ ]);
+ expect(await profileIds([planIs('team')], ['properties'])).toEqual([
+ VERSIONED.multi,
+ ]);
+ expect(await profileIds([planIs('free')], ['properties'])).toEqual([
+ VERSIONED.staleMatch,
+ ]);
+ });
+
+ it('filters top-level columns on the latest version', async () => {
+ expect(await profileIds([emailIs('[email protected]')], ['email'])).toEqual([
+ VERSIONED.multi,
+ ]);
+ expect(await profileIds([emailIs('[email protected]')], ['email'])).toEqual(
+ []
+ );
+ });
+
+ it('matches the FINAL form it replaced, row for row', async () => {
+ const filters = [planIs('team')];
+ const clauses = getEventFiltersWhereClause(filters, PROJECT_ID, 'e');
+ const window = sql`created_at >= toDateTime(${sql.string(timeAgo(now, 7))}) AND created_at <= toDateTime(${sql.string(timeAgo(now, 0))})`;
+ const reference = sql`SELECT id, name, profile_id FROM events e LEFT ANY JOIN (SELECT id, properties FROM profiles FINAL WHERE project_id = ${sql.string(PROJECT_ID)}) as profile on profile.id = profile_id WHERE project_id = ${sql.string(PROJECT_ID)} AND ${window} AND ${sql.join(compiledFilterFragments(clauses), ' AND ')}`;
+ await expectSameRows(ch, reference, listQuery(filters, ['properties']), 4);
+ });
+});
diff --git a/test/profile-versions.ts b/test/profile-versions.ts
index 908d62dde..414351fd5 100644
--- a/test/profile-versions.ts
+++ b/test/profile-versions.ts
@@ -56,7 +56,12 @@ function profileVersions(projectId: string, now: Date): Version[][] {
last_name: 'Kim-Lee',
is_external: true,
- properties: { plan: 'team', country: 'SE', tier: 'gold' },
+ properties: {
+ plan: 'team',
+ country: 'SE',
+ tier: 'gold',
+ 'experiments.checkout.name': 'one-page',
+ },
groups: [VERSIONED.groups.g1, VERSIONED.groups.g2],
last_seen_at: timeAgo(now, 2),
},