Skip to content

OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.

perf(api): profile lists, counts, member pages and filters drop FINAL

Carl-Gerhard Lindesvärd0f226d003d29
diff --git a/packages/core/src/modules/chart/src/table-filter-where.test.ts b/packages/core/src/modules/chart/src/table-filter-where.test.ts
index 84369aa57..0f348fb73 100644
--- a/packages/core/src/modules/chart/src/table-filter-where.test.ts
+++ b/packages/core/src/modules/chart/src/table-filter-where.test.ts
@@ -98,11 +98,28 @@ function literal(value: SqlParamValue | undefined): string {
return String(value);
}
-/** What `buildProfileClause` wraps `inner` in for a given table. */
-const profileWrap = (table: keyof typeof CONTEXTS, inner: string): string =>
+/**
+ * What `buildProfileClause` wraps the predicate in for a given table: on the
+ * profiles table the predicate itself; elsewhere a semi-join that narrows the
+ * candidate ids with it on the raw rows and decides on the newest version.
+ */
+const profileWrap = (
+ table: keyof typeof CONTEXTS,
+ inner: string,
+ onLatest: string
+): string =>
table === 'profiles'
? `(${inner})`
- : `(profile_id IN (SELECT id FROM ${TABLE_NAMES.profiles} FINAL WHERE project_id = 'p' AND ${inner}))`;
+ : `(cityHash64(profile_id) IN (SELECT cityHash64(id) FROM (SELECT id FROM ${TABLE_NAMES.profiles} WHERE project_id = 'p' AND cityHash64(id) IN (SELECT cityHash64(id) FROM ${TABLE_NAMES.profiles} WHERE project_id = 'p' AND ${inner}) GROUP BY id HAVING ${onLatest})))`;
+
+const latestOf = (column: string): string => {
+ if (column === 'id') {
+ return 'id';
+ }
+ return column === 'last_seen_at'
+ ? 'max(last_seen_at)'
+ : `argMax(${column}, last_seen_at)`;
+};
const ALLOWED_STRING_COLUMNS = [
'id',
@@ -130,7 +147,9 @@ describe.each(TABLES)('buildFilterWhere on %s', (table) => {
const where = build(table, [
filter({ name: `profile.${column}`, value: ['a'] }),
]);
- expect(where.f0).toBe(profileWrap(table, `${column} = 'a'`));
+ expect(where.f0).toBe(
+ profileWrap(table, `${column} = 'a'`, `${latestOf(column)} = 'a'`)
+ );
}
});
@@ -140,7 +159,11 @@ describe.each(TABLES)('buildFilterWhere on %s', (table) => {
filter({ name: `profile.${column}`, operator: 'gt', value: ['123'] }),
]);
expect(where.f0).toBe(
- profileWrap(table, `(toFloat64(${column}) > toFloat64('123'))`)
+ profileWrap(
+ table,
+ `(toFloat64(${column}) > toFloat64('123'))`,
+ `(toFloat64(${latestOf(column)}) > toFloat64('123'))`
+ )
);
}
});
@@ -149,7 +172,13 @@ describe.each(TABLES)('buildFilterWhere on %s', (table) => {
const where = build(table, [
filter({ name: 'profile.properties.plan', value: ['pro'] }),
]);
- expect(where.f0).toBe(profileWrap(table, "properties['plan'] = 'pro'"));
+ expect(where.f0).toBe(
+ profileWrap(
+ table,
+ "properties['plan'] = 'pro'",
+ "argMax(properties['plan'], last_seen_at) = 'pro'"
+ )
+ );
});
it('escapes a quote in a properties key', () => {
@@ -157,7 +186,13 @@ describe.each(TABLES)('buildFilterWhere on %s', (table) => {
filter({ name: "profile.properties.pl'an", value: ['pro'] }),
];
const where = build(table, filters);
- expect(where.f0).toBe(profileWrap(table, "properties['pl\\'an'] = 'pro'"));
+ expect(where.f0).toBe(
+ profileWrap(
+ table,
+ "properties['pl\\'an'] = 'pro'",
+ "argMax(properties['pl\\'an'], last_seen_at) = 'pro'"
+ )
+ );
// The key is a bound param now, so the quote never reaches the SQL text.
expect(Object.values(paramsOf(table, filters)[0]!)).toContain("pl'an");
});
@@ -177,7 +212,13 @@ describe.each(TABLES)('buildFilterWhere on %s', (table) => {
filter({ name: 'profile.not a column', value: ['a'] }),
]);
expect(Object.keys(where)).toEqual(['f0']);
- expect(where.f0).toBe(profileWrap(table, "email = '[email protected]'"));
+ expect(where.f0).toBe(
+ profileWrap(
+ table,
+ "email = '[email protected]'",
+ "argMax(email, last_seen_at) = '[email protected]'"
+ )
+ );
});
it('keys surviving filters by their original index', () => {
diff --git a/packages/core/src/modules/chart/src/table-filter-where.ts b/packages/core/src/modules/chart/src/table-filter-where.ts
index 2d37bc6b4..3e495c2d3 100644
--- a/packages/core/src/modules/chart/src/table-filter-where.ts
+++ b/packages/core/src/modules/chart/src/table-filter-where.ts
@@ -17,7 +17,11 @@ import {
} from '../../report/report.constants';
import { PROFILE_SELECT_COLUMNS } from '../chart.constants';
import { formatClickhouseDate } from './dates';
-import { buildCohortMembersSubselect, CHART_TABLE } from './field-resolution';
+import {
+ buildCohortMembersSubselect,
+ CHART_TABLE,
+ latestProfileValue,
+} from './field-resolution';
import {
buildTypedClauseFragment,
hasTypedCast,
@@ -27,6 +31,11 @@ import {
export interface FilterTableContext {
/** Outer query's primary table. */
selfTable: 'sessions' | 'profiles' | 'events';
+ /**
+ * The outer query groups `profiles` by id, so a `profile.*` predicate reads
+ * the newest version of its column — an aggregate, for a HAVING clause.
+ */
+ latestVersion?: boolean;
/**
* Expression on the outer row that yields the owning profile id. For the
* profiles table this is `id`; for events/sessions it's `profile_id`. Used
@@ -358,7 +367,10 @@ function buildGroupClause(
if (!inner) {
return null;
}
- return sql`arrayExists(g -> g IN (SELECT id FROM ${sql.id(CHART_TABLE.groups)} FINAL WHERE project_id = ${sql.string(projectId)} AND ${inner}), ${sql.id(ctx.groupsExpr)})`;
+ const groups = ctx.latestVersion
+ ? sql`argMax(${sql.id(ctx.groupsExpr)}, last_seen_at)`
+ : sql.id(ctx.groupsExpr);
+ return sql`arrayExists(g -> g IN (SELECT id FROM ${sql.id(CHART_TABLE.groups)} FINAL WHERE project_id = ${sql.string(projectId)} AND ${inner}), ${groups})`;
}
function buildProfileClause(
@@ -372,17 +384,37 @@ function buildProfileClause(
return null;
}
const numeric = name === 'created_at' || name === 'last_seen_at';
- const inner = compileScalarClause(column, filter.operator, filter.value, {
- numeric,
- type: filter.type,
- });
- if (!inner) {
- return null;
- }
+ const compile = (expression: SqlFragment) =>
+ compileScalarClause(expression, filter.operator, filter.value, {
+ numeric,
+ type: filter.type,
+ });
+ const latest = latestProfileValue(name, column);
if (ctx.selfTable === 'profiles') {
- return inner;
+ return compile(ctx.latestVersion ? latest : column);
+ }
+ const onAnyVersion = compile(column);
+ const onLatestVersion = compile(latest);
+ if (!(onAnyVersion && onLatestVersion)) {
+ return null;
}
- return sql`${sql.id(ctx.profileIdExpr)} IN (SELECT id FROM ${sql.id(CHART_TABLE.profiles)} FINAL WHERE project_id = ${sql.string(projectId)} AND ${inner})`;
+ return sql`cityHash64(${sql.id(ctx.profileIdExpr)}) IN (SELECT cityHash64(id) FROM ${latestProfilesMatching(projectId, onAnyVersion, onLatestVersion)})`;
+}
+
+/**
+ * Ids whose newest version satisfies `onLatestVersion`, without `FINAL`. The
+ * raw-row prefilter is a superset — the surviving version is one of the rows
+ * — so the GROUP BY only sees candidates, not the whole project. The set
+ * holds hashes: a String set costs ~300 bytes per id, and a collision only
+ * adds a candidate the HAVING clause rejects.
+ */
+export function latestProfilesMatching(
+ projectId: string,
+ onAnyVersion: SqlFragment,
+ onLatestVersion: SqlFragment
+): SqlFragment {
+ const project = sql`project_id = ${sql.string(projectId)}`;
+ return sql`(SELECT id FROM ${sql.id(CHART_TABLE.profiles)} WHERE ${project} AND cityHash64(id) IN (SELECT cityHash64(id) FROM ${sql.id(CHART_TABLE.profiles)} WHERE ${project} AND ${onAnyVersion}) GROUP BY id HAVING ${onLatestVersion})`;
}
function buildSessionClause(
diff --git a/packages/core/src/modules/chart/src/table-filter-where.versions.test.ts b/packages/core/src/modules/chart/src/table-filter-where.versions.test.ts
new file mode 100644
index 000000000..604ecbcc2
--- /dev/null
+++ b/packages/core/src/modules/chart/src/table-filter-where.versions.test.ts
@@ -0,0 +1,95 @@
+// A `profile.*` filter on another table compiles to a semi-join that no
+// longer reads `profiles FINAL`: it decides on the newest version of the
+// column. Runs against profiles with several unmerged versions.
+
+import { afterAll, beforeAll, describe, expect, it } from 'bun:test';
+import { type SqlFragment, sql } from '@openpanel/db/src/clickhouse/sql';
+import {
+ setupProfileVersions,
+ teardownProfileVersions,
+ VERSIONED,
+} from '../../../../../../test/profile-versions';
+import { expectSameRows } from '../../../../test/clickhouse-rows';
+import type { IChartEventFilter } from '../../report/report.constants';
+import { buildFilterWhere } from './table-filter-where';
+
+const PROJECT_ID = 'semi-join-profile-versions';
+const ON_EVENTS = {
+ selfTable: 'events',
+ profileIdExpr: 'profile_id',
+ groupsExpr: 'groups',
+} as const;
+
+let ch: typeof import('@openpanel/db/src/clickhouse/client').ch;
+
+beforeAll(async () => {
+ ({ ch } = await import('@openpanel/db/src/clickhouse/client'));
+ const { bootstrapTestDatabases } = await import(
+ '../../../../../../test/bootstrap-databases'
+ );
+ await bootstrapTestDatabases();
+ await setupProfileVersions(PROJECT_ID);
+}, 30_000);
+
+afterAll(() => teardownProfileVersions(PROJECT_ID));
+
+function eventProfiles(clause: SqlFragment) {
+ return sql`SELECT DISTINCT profile_id FROM events WHERE project_id = ${sql.string(PROJECT_ID)} AND ${clause}`;
+}
+
+/** The semi-join as it was: `profile_id IN (SELECT id FROM profiles FINAL WHERE ... AND <predicate>)`. */
+function finalSemiJoin(predicate: SqlFragment) {
+ return sql`profile_id IN (SELECT id FROM profiles FINAL WHERE project_id = ${sql.string(PROJECT_ID)} AND ${predicate})`;
+}
+
+const filter = (
+ name: string,
+ operator: IChartEventFilter['operator'],
+ value: string
+): IChartEventFilter => ({ id: 'f', name, operator, value: [value] });
+
+describe('buildFilterWhere on events with a profile.* filter', () => {
+ const cases: [string, IChartEventFilter, SqlFragment, string[]][] = [
+ [
+ 'plan is pro: the stale pro version must not match',
+ filter('profile.properties.plan', 'is', 'pro'),
+ sql`properties['plan'] = 'pro'`,
+ [VERSIONED.single],
+ ],
+ [
+ 'plan isNot pro: decided on the newest version',
+ filter('profile.properties.plan', 'isNot', 'pro'),
+ sql`properties['plan'] != 'pro'`,
+ [VERSIONED.multi, VERSIONED.staleMatch],
+ ],
+ [
+ 'email is the old address: nothing matches',
+ filter('profile.email', 'is', '[email protected]'),
+ sql`email = '[email protected]'`,
+ [],
+ ],
+ [
+ 'last_name contains the new value',
+ filter('profile.last_name', 'contains', 'Lee'),
+ sql`last_name LIKE '%Lee%'`,
+ [VERSIONED.multi],
+ ],
+ ];
+
+ it.each(cases)('%s', async (_label, profileFilter, predicate, expected) => {
+ const [clause] = Object.values(
+ buildFilterWhere([profileFilter], PROJECT_ID, ON_EVENTS)
+ );
+ expect(clause).toBeDefined();
+ expect(clause?.toStatement().query).not.toContain('FINAL');
+ const rows = await expectSameRows(
+ ch,
+ eventProfiles(finalSemiJoin(predicate)),
+ eventProfiles(clause as SqlFragment),
+ expected.length
+ );
+ expect(rows.map((row) => JSON.parse(row).profile_id).sort()).toEqual(
+ [...expected].sort()
+ );
+ });
+});
diff --git a/packages/core/src/modules/cohort/cohort.service.ts b/packages/core/src/modules/cohort/cohort.service.ts
index 7e42655e4..0f880513c 100644
--- a/packages/core/src/modules/cohort/cohort.service.ts
+++ b/packages/core/src/modules/cohort/cohort.service.ts
@@ -865,40 +865,56 @@ export async function listCohortMemberProfiles(
}
): Promise<{ data: IServiceProfile[]; count: number }> {
const { buildFilterWhere } = await import('../chart/src/table-filter-where');
- const { getProfiles, profileSearchCondition } = await import(
- '../profile/profile.service'
- );
+ const { getProfiles, latestProfileColumn, profileSearchCondition } =
+ await import('../profile/profile.service');
const offset = Math.max(0, (cursor ?? 0) * take);
- const searchClause = profileSearchCondition(search);
- const searchCondition = searchClause ? sql`AND ${searchClause}` : sql.empty;
-
- const extraConditions = filters?.length
- ? Object.values(
- buildFilterWhere(filters, projectId, {
- selfTable: 'profiles',
- profileIdExpr: 'id',
- groupsExpr: 'groups',
- })
- )
- : [];
- const extraConditionSql = extraConditions.length
- ? sql`AND ${sql.join(extraConditions, ' AND ')}`
+ // The raw-row conditions only narrow the id set (a superset: the surviving
+ // version is one of the rows), never the rows the aggregate reads; the
+ // HAVING clause decides on the newest version.
+ const conditionsOn = (latestVersion: boolean): SqlFragment[] => {
+ const searchClause = profileSearchCondition(
+ search,
+ latestVersion ? latestProfileColumn : undefined
+ );
+ const filterClauses = filters?.length
+ ? Object.values(
+ buildFilterWhere(filters, projectId, {
+ selfTable: 'profiles',
+ profileIdExpr: 'id',
+ groupsExpr: 'groups',
+ latestVersion,
+ })
+ )
+ : [];
+ return searchClause ? [searchClause, ...filterClauses] : filterClauses;
+ };
+ const anyVersion = conditionsOn(false);
+ const latestVersion = conditionsOn(true);
+ const candidates = anyVersion.length
+ ? sql`AND cityHash64(id) IN (SELECT cityHash64(id) FROM ${sql.id(TABLE.profiles)} WHERE project_id = ${sql.string(projectId)} AND ${sql.join(anyVersion, ' AND ')})`
+ : sql.empty;
+ const having = latestVersion.length
+ ? sql`HAVING ${sql.join(latestVersion, ' AND ')}`
: sql.empty;
const rows = await chQuery<{ id: string; total_count: number }>(
deps,
sql`
SELECT id, count() OVER () AS total_count
- FROM ${sql.id(TABLE.profiles)} FINAL
- WHERE project_id = ${sql.string(projectId)}
- AND id IN (
- SELECT profile_id FROM ${sql.id(TABLE.cohortMembers)} FINAL
- WHERE cohort_id = ${sql.string(cohortId)}
- AND project_id = ${sql.string(projectId)}
- )
- ${searchCondition}
- ${extraConditionSql}
+ FROM (
+ SELECT id, argMax(created_at, last_seen_at) AS created_at
+ FROM ${sql.id(TABLE.profiles)}
+ WHERE project_id = ${sql.string(projectId)}
+ AND id IN (
+ SELECT profile_id FROM ${sql.id(TABLE.cohortMembers)} FINAL
+ WHERE cohort_id = ${sql.string(cohortId)}
+ AND project_id = ${sql.string(projectId)}
+ )
+ ${candidates}
+ GROUP BY id
+ ${having}
+ )
ORDER BY created_at DESC
LIMIT ${sql.uint64(take)} OFFSET ${sql.uint64(offset)}
`
diff --git a/packages/core/src/modules/cohort/cohort.versions.test.ts b/packages/core/src/modules/cohort/cohort.versions.test.ts
new file mode 100644
index 000000000..d7934c50b
--- /dev/null
+++ b/packages/core/src/modules/cohort/cohort.versions.test.ts
@@ -0,0 +1,116 @@
+// The cohort member page no longer reads `profiles FINAL`: search and
+// filters decide on the newest version. Runs against profiles with several
+// unmerged versions (test/profile-versions) enrolled in one cohort.
+
+import { afterAll, beforeAll, describe, expect, it } from 'bun:test';
+import { sql } from '@openpanel/db/src/clickhouse/sql';
+import {
+ otherProjectId,
+ setupProfileVersions,
+ teardownProfileVersions,
+ VERSIONED,
+} from '../../../../../test/profile-versions';
+import { fetchRows } from '../../../test/clickhouse-rows';
+import type { IChartEventFilter } from '../report/report.constants';
+
+const PROJECT_ID = 'cohort-profile-versions';
+const COHORT_ID = 'dddddddd-0000-4000-8000-00000000pv01';
+
+let ch: typeof import('@openpanel/db/src/clickhouse/client').ch;
+let deps: import('../../services').ServiceDeps;
+let listCohortMemberProfiles: typeof import('./cohort.service').listCohortMemberProfiles;
+
+beforeAll(async () => {
+ ({ ch } = await import('@openpanel/db/src/clickhouse/client'));
+ const { testServiceDeps } = await import('../../../test/service-deps');
+ deps = await testServiceDeps();
+ ({ listCohortMemberProfiles } = await import('./cohort.service'));
+ const { bootstrapTestDatabases } = await import(
+ '../../../../../test/bootstrap-databases'
+ );
+ await bootstrapTestDatabases();
+ await setupProfileVersions(PROJECT_ID);
+ await ch.insert({
+ table: 'cohort_members',
+ values: [VERSIONED.multi, VERSIONED.staleMatch, VERSIONED.single].map(
+ (profileId) => ({
+ project_id: PROJECT_ID,
+ cohort_id: COHORT_ID,
+ profile_id: profileId,
+ matching_properties: {},
+ version: 1,
+ })
+ ),
+ format: 'JSONEachRow',
+ });
+ await ch.insert({
+ table: 'cohort_members',
+ values: [
+ {
+ project_id: otherProjectId(PROJECT_ID),
+ cohort_id: COHORT_ID,
+ profile_id: VERSIONED.multi,
+ matching_properties: {},
+ version: 1,
+ },
+ ],
+ format: 'JSONEachRow',
+ });
+}, 30_000);
+
+afterAll(async () => {
+ await ch.command({
+ query: `DELETE FROM cohort_members WHERE cohort_id = '${COHORT_ID}'`,
+ });
+ await teardownProfileVersions(PROJECT_ID);
+});
+
+const planIs = (value: string): IChartEventFilter => ({
+ id: 'f',
+ name: 'profile.properties.plan',
+ operator: 'is',
+ value: [value],
+});
+
+async function page(
+ options: { search?: string; filters?: IChartEventFilter[] } = {}
+) {
+ const { data, count } = await listCohortMemberProfiles(deps, {
+ projectId: PROJECT_ID,
+ cohortId: COHORT_ID,
+ take: 10,
+ ...options,
+ });
+ return { ids: data.map((profile) => profile.id), count };
+}
+
+describe('listCohortMemberProfiles', () => {
+ it('pages the members newest first with the total, like FINAL did', async () => {
+ const reference = await fetchRows(
+ ch,
+ sql`SELECT id FROM profiles FINAL WHERE project_id = ${sql.string(PROJECT_ID)} AND id IN (SELECT profile_id FROM cohort_members FINAL WHERE cohort_id = ${sql.string(COHORT_ID)} AND project_id = ${sql.string(PROJECT_ID)}) ORDER BY created_at DESC LIMIT 10`
+ );
+ const { ids, count } = await page();
+ expect(ids).toEqual([
+ VERSIONED.single,
+ VERSIONED.staleMatch,
+ VERSIONED.multi,
+ ]);
+ expect([...ids].sort()).toEqual(
+ reference.map((row) => JSON.parse(row).id).sort()
+ );
+ expect(count).toBe(3);
+ });
+
+ it('search and filters decide on the newest version', async () => {
+ expect(await page({ filters: [planIs('pro')] })).toEqual({
+ ids: [VERSIONED.single],
+ count: 1,
+ });
+ expect(await page({ search: 'mia' })).toEqual({
+ ids: [VERSIONED.multi],
+ count: 1,
+ });
+ expect(await page({ search: 'Old' })).toEqual({ ids: [], count: 0 });
+ });
+});
diff --git a/packages/core/src/modules/group/src/sql.test.ts b/packages/core/src/modules/group/src/sql.test.ts
index 61a7040f4..d6254c711 100644
--- a/packages/core/src/modules/group/src/sql.test.ts
+++ b/packages/core/src/modules/group/src/sql.test.ts
@@ -167,18 +167,20 @@ describe('groupMemberProfilesQuery', () => {
}).toStatement();
const text = collapse(query);
+ // Membership is decided on the newest version of `groups`; the raw-row
+ // `has()` only narrows the candidate ids.
expect(text).toContain(
- 'count() OVER () AS total_count FROM profiles FINAL'
- );
- expect(text).toContain(
- 'WHERE project_id = {p1:String} AND has(groups, {p2:String}) ORDER BY created_at DESC LIMIT {p3:UInt64} OFFSET {p4:UInt64}'
+ 'count() OVER () AS total_count FROM ( SELECT id, argMax(created_at, last_seen_at) AS created_at FROM profiles WHERE project_id = {p1:String} AND cityHash64(id) IN (SELECT cityHash64(id) FROM profiles WHERE project_id = {p2:String} AND has(groups, {p3:String})) GROUP BY id HAVING has(argMax(groups, last_seen_at), {p4:String})) ORDER BY created_at DESC LIMIT {p5:UInt64} OFFSET {p6:UInt64}'
);
+ expect(text).not.toContain('FINAL');
expect(text).not.toContain('ILIKE');
expect(query_params).toEqual({
p1: PROJECT_ID,
- p2: GROUP_ID,
- p3: 50,
- p4: 0,
+ p2: PROJECT_ID,
+ p3: GROUP_ID,
+ p4: GROUP_ID,
+ p5: 50,
+ p6: 0,
});
});
@@ -192,23 +194,21 @@ describe('groupMemberProfilesQuery', () => {
}).toStatement();
expect(collapse(query)).toContain(
- 'AND (email ILIKE {p3:String} OR first_name ILIKE {p4:String} OR last_name ILIKE {p5:String})'
+ 'AND (argMax(email, last_seen_at) ILIKE {p5:String} OR argMax(first_name, last_seen_at) ILIKE {p6:String} OR argMax(last_name, last_seen_at) ILIKE {p7:String})'
);
expect(query).not.toContain(HOSTILE);
- expect(query_params.p3).toBe(`%${HOSTILE}%`);
expect(query_params.p5).toBe(`%${HOSTILE}%`);
+ expect(query_params.p7).toBe(`%${HOSTILE}%`);
});
});
describe('the trpc router queries', () => {
const ref = { projectId: PROJECT_ID, groupId: HOSTILE };
- test('every per-group query binds project then group and interpolates nothing', () => {
+ test('every per-group event query binds project then group and interpolates nothing', () => {
const builders = [
groupEventMetricsQuery,
- groupUniqueProfilesQuery,
groupActivityQuery,
- groupMemberGrowthQuery,
groupMostEventsQuery,
groupPopularRoutesQuery,
];
@@ -222,14 +222,31 @@ describe('the trpc router queries', () => {
}
});
+ test('every per-group profile query decides membership on the newest version', () => {
+ for (const build of [groupUniqueProfilesQuery, groupMemberGrowthQuery]) {
+ const { query, query_params } = build(ref).toStatement();
+ expect(collapse(query)).toContain(
+ 'FROM profiles WHERE project_id = {p1:String} AND cityHash64(id) IN (SELECT cityHash64(id) FROM profiles WHERE project_id = {p2:String} AND has(groups, {p3:String})) GROUP BY id HAVING has(argMax(groups, last_seen_at), {p4:String})'
+ );
+ expect(query).not.toContain('FINAL');
+ expect(query).not.toContain(HOSTILE);
+ expect(query_params).toMatchObject({
+ p1: PROJECT_ID,
+ p2: PROJECT_ID,
+ p3: HOSTILE,
+ p4: HOSTILE,
+ });
+ }
+ });
+
test('memberGrowth keeps the 30-day window and the WITH FILL frame', () => {
const { query, query_params } = groupMemberGrowthQuery(ref).toStatement();
const text = collapse(query);
expect(text).toContain(
- 'AND created_at >= now() - INTERVAL {p3:UInt64} DAY GROUP BY date ORDER BY date ASC WITH FILL FROM toDate(now() - INTERVAL {p4:UInt64} DAY) TO toDate(now() + INTERVAL 1 DAY) STEP 1'
+ 'WHERE created_at >= now() - INTERVAL {p5:UInt64} DAY GROUP BY date ORDER BY date ASC WITH FILL FROM toDate(now() - INTERVAL {p6:UInt64} DAY) TO toDate(now() + INTERVAL 1 DAY) STEP 1'
);
- expect(query_params).toMatchObject({ p3: 30, p4: 29 });
+ expect(query_params).toMatchObject({ p5: 30, p6: 29 });
});
test('mostEvents excludes the lifecycle events; popularRoutes keeps only screen_view', () => {
diff --git a/packages/core/src/modules/group/src/sql.ts b/packages/core/src/modules/group/src/sql.ts
index 2d33ceaff..19f9c19c5 100644
--- a/packages/core/src/modules/group/src/sql.ts
+++ b/packages/core/src/modules/group/src/sql.ts
@@ -145,21 +145,45 @@ export interface GroupMemberProfilesQuery {
search?: string;
}
+/**
+ * The profiles whose newest version belongs to the group, with their first
+ * seen. The raw-row `has()` is a superset (the surviving version is one of
+ * the rows), so the GROUP BY only sees candidates, matched on a hash so the
+ * set stays small; `having` adds conditions on the newest version.
+ */
+function groupMembers(
+ projectId: string,
+ groupId: string,
+ having: SqlFragment[] = []
+): SqlFragment {
+ const project = sql`project_id = ${sql.string(projectId)}`;
+ const group = sql.string(groupId);
+ return sql`
+ SELECT id, argMax(created_at, last_seen_at) AS created_at
+ FROM ${sql.id(TABLE.profiles)}
+ WHERE ${project}
+ AND cityHash64(id) IN (SELECT cityHash64(id) FROM ${sql.id(TABLE.profiles)} WHERE ${project} AND has(groups, ${group}))
+ GROUP BY id
+ HAVING ${sql.join([sql`has(argMax(groups, last_seen_at), ${group})`, ...having], ' AND ')}`;
+}
+
/** Member ids plus the window total, so one round trip pages and counts. */
export function groupMemberProfilesQuery(
query: GroupMemberProfilesQuery
): SqlFragment {
+ const search = query.search
+ ? [
+ (() => {
+ const pattern = sql.string(`%${query.search}%`);
+ return sql`(argMax(email, last_seen_at) ILIKE ${pattern} OR argMax(first_name, last_seen_at) ILIKE ${pattern} OR argMax(last_name, last_seen_at) ILIKE ${pattern})`;
+ })(),
+ ]
+ : [];
return sql`
SELECT
id,
count() OVER () AS total_count
- FROM ${sql.id(TABLE.profiles)} FINAL
- WHERE project_id = ${sql.string(query.projectId)}
- AND has(groups, ${sql.string(query.groupId)})
- ${optional(query.search, () => {
- const pattern = sql.string(`%${query.search}%`);
- return sql`AND (email ILIKE ${pattern} OR first_name ILIKE ${pattern} OR last_name ILIKE ${pattern})`;
- })}
+ FROM (${groupMembers(query.projectId, query.groupId, search)})
ORDER BY created_at DESC
LIMIT ${sql.uint64(query.take)}
OFFSET ${sql.uint64(query.offset)}
@@ -187,9 +211,7 @@ export function groupUniqueProfilesQuery(query: {
}): SqlFragment {
return sql`
SELECT count() AS uniqueProfiles
- FROM ${sql.id(TABLE.profiles)} FINAL
- WHERE project_id = ${sql.string(query.projectId)}
- AND has(groups, ${sql.string(query.groupId)})
+ FROM (${groupMembers(query.projectId, query.groupId)})
`;
}
@@ -218,10 +240,8 @@ export function groupMemberGrowthQuery(query: {
SELECT
toDate(toStartOfDay(created_at)) AS date,
count() AS count
- FROM ${sql.id(TABLE.profiles)} FINAL
- WHERE project_id = ${sql.string(query.projectId)}
- AND has(groups, ${sql.string(query.groupId)})
- AND created_at >= now() - INTERVAL ${sql.uint64(MEMBER_GROWTH_WINDOW_DAYS)} DAY
+ FROM (${groupMembers(query.projectId, query.groupId)})
+ WHERE created_at >= now() - INTERVAL ${sql.uint64(MEMBER_GROWTH_WINDOW_DAYS)} DAY
GROUP BY date
ORDER BY date ASC WITH FILL
FROM toDate(now() - INTERVAL ${sql.uint64(MEMBER_GROWTH_FILL_FROM_DAYS_AGO)} DAY)
diff --git a/packages/core/src/modules/group/src/sql.versions.test.ts b/packages/core/src/modules/group/src/sql.versions.test.ts
new file mode 100644
index 000000000..35306ffc2
--- /dev/null
+++ b/packages/core/src/modules/group/src/sql.versions.test.ts
@@ -0,0 +1,91 @@
+// The group member queries no longer read `profiles FINAL`: membership is
+// decided on the newest version of `groups`. Runs against profiles with
+// several unmerged versions (test/profile-versions).
+
+import { afterAll, beforeAll, describe, expect, it } from 'bun:test';
+import { sql } from '@openpanel/db/src/clickhouse/sql';
+import {
+ setupProfileVersions,
+ teardownProfileVersions,
+ VERSIONED,
+} from '../../../../../../test/profile-versions';
+import { expectSameRows, fetchRows } from '../../../../test/clickhouse-rows';
+import {
+ groupMemberGrowthQuery,
+ groupMemberProfilesQuery,
+ groupUniqueProfilesQuery,
+} from './sql';
+
+const PROJECT_ID = 'group-profile-versions';
+const { g1, g2 } = VERSIONED.groups;
+
+let ch: typeof import('@openpanel/db/src/clickhouse/client').ch;
+
+beforeAll(async () => {
+ ({ ch } = await import('@openpanel/db/src/clickhouse/client'));
+ const { bootstrapTestDatabases } = await import(
+ '../../../../../../test/bootstrap-databases'
+ );
+ await bootstrapTestDatabases();
+ await setupProfileVersions(PROJECT_ID);
+}, 30_000);
+
+afterAll(() => teardownProfileVersions(PROJECT_ID));
+
+const parsed = async (fragment: Parameters<typeof fetchRows>[1]) =>
+ (await fetchRows(ch, fragment)).map((row) => JSON.parse(row));
+
+describe('group members', () => {
+ it('counts profiles whose newest version is in the group, like FINAL did', async () => {
+ for (const [groupId, expected] of [
+ [g1, 2],
+ [g2, 2],
+ ['pv-no-such-group', 0],
+ ] as const) {
+ const reference = sql`SELECT count() AS uniqueProfiles FROM profiles FINAL WHERE project_id = ${sql.string(PROJECT_ID)} AND has(groups, ${sql.string(groupId)})`;
+ const [row] = await expectSameRows(
+ ch,
+ reference,
+ groupUniqueProfilesQuery({ projectId: PROJECT_ID, groupId }),
+ 1
+ );
+ expect(Number(JSON.parse(row as string).uniqueProfiles)).toBe(expected);
+ }
+ });
+
+ it('pages the member ids with the total, and searches the newest version', async () => {
+ const page = (search?: string) =>
+ groupMemberProfilesQuery({
+ projectId: PROJECT_ID,
+ groupId: g1,
+ take: 10,
+ offset: 0,
+ search,
+ });
+ const reference = sql`SELECT id, count() OVER () AS total_count FROM profiles FINAL WHERE project_id = ${sql.string(PROJECT_ID)} AND has(groups, ${sql.string(g1)}) ORDER BY created_at DESC LIMIT 10 OFFSET 0`;
+ const members = await expectSameRows(ch, reference, page(), 2);
+ expect(members.map((row) => JSON.parse(row).id).sort()).toEqual(
+ [VERSIONED.multi, VERSIONED.staleMatch].sort()
+ );
+ expect(Number(JSON.parse(members[0] as string).total_count)).toBe(2);
+ expect((await parsed(page('sam'))).map((row) => row.id)).toEqual([
+ VERSIONED.staleMatch,
+ ]);
+ expect((await parsed(page('Old'))).map((row) => row.id)).toEqual([]);
+ });
+
+ it('member growth counts first-seen days of the newest version, like FINAL did', async () => {
+ const reference = sql`SELECT toDate(toStartOfDay(created_at)) AS date, count() AS count FROM profiles FINAL WHERE project_id = ${sql.string(PROJECT_ID)} AND has(groups, ${sql.string(g2)}) AND created_at >= now() - INTERVAL 30 DAY GROUP BY date ORDER BY date ASC WITH FILL FROM toDate(now() - INTERVAL 29 DAY) TO toDate(now() + INTERVAL 1 DAY) STEP 1`;
+ const rows = await expectSameRows(
+ ch,
+ reference,
+ groupMemberGrowthQuery({ projectId: PROJECT_ID, groupId: g2 }),
+ 30
+ );
+ const counted = rows
+ .map((row) => JSON.parse(row))
+ .filter((row) => Number(row.count) > 0);
+ expect(counted).toHaveLength(1);
+ expect(Number(counted[0].count)).toBe(1);
+ });
+});
diff --git a/packages/core/src/modules/profile/profile.service.ts b/packages/core/src/modules/profile/profile.service.ts
index 3c48d3bfd..4773cb692 100644
--- a/packages/core/src/modules/profile/profile.service.ts
+++ b/packages/core/src/modules/profile/profile.service.ts
@@ -36,7 +36,7 @@ import {
profileValuesQuery,
} from './src/sql';
-export { profileSearchCondition } from './src/sql';
+export { latestProfileColumn, profileSearchCondition } from './src/sql';
const PROFILES_CACHE_SECONDS = 60 * 5;
const PROPERTY_KEYS_CACHE_SECONDS = 60;
@@ -166,12 +166,16 @@ export function transformProfile({
async function compileProfileFilters(
filters: IChartEventFilter[] | undefined,
- projectId: string
+ projectId: string,
+ latestVersion = false
): Promise<CompiledFilterClauses> {
if (!filters?.length) {
return {};
}
- return buildFilterWhere(filters, projectId, PROFILE_FILTER_TARGET);
+ return buildFilterWhere(filters, projectId, {
+ ...PROFILE_FILTER_TARGET,
+ latestVersion,
+ });
}
/**
@@ -287,6 +291,11 @@ export async function getProfileList(
startDate,
endDate,
filterClauses: await compileProfileFilters(filters, projectId),
+ latestFilterClauses: await compileProfileFilters(
+ filters,
+ projectId,
+ true
+ ),
})
);
return data.map(transformProfile);
@@ -312,6 +321,11 @@ export async function getProfileListCount(
startDate,
endDate,
filterClauses: await compileProfileFilters(filters, projectId),
+ latestFilterClauses: await compileProfileFilters(
+ filters,
+ projectId,
+ true
+ ),
})
);
return data[0]?.count ?? 0;
@@ -390,6 +404,11 @@ export async function findProfilesCore(
input.filters,
input.projectId
),
+ latestFilterClauses: await compileProfileFilters(
+ input.filters,
+ input.projectId,
+ true
+ ),
sortOrder: input.sortOrder === 'asc' ? 'asc' : 'desc',
limit: Math.min(
input.limit ?? FIND_PROFILES_DEFAULT_LIMIT,
diff --git a/packages/core/src/modules/profile/src/sql.test.ts b/packages/core/src/modules/profile/src/sql.test.ts
index d03363e5a..c74addda3 100644
--- a/packages/core/src/modules/profile/src/sql.test.ts
+++ b/packages/core/src/modules/profile/src/sql.test.ts
@@ -28,12 +28,24 @@ const PROJECT_ID = 'proj-1';
const PROFILE_ID = 'prof-1';
const HOSTILE = "x' OR 1=1 --";
const NO_FILTERS = {};
+const NO_FILTER_CLAUSES = {
+ filterClauses: NO_FILTERS,
+ latestFilterClauses: NO_FILTERS,
+};
+/** The outer statement of a page: every column from the newest version of the ids inside. */
+const LATEST_COLUMNS_PAGE =
+ 'SELECT id, argMax(first_name, p.last_seen_at) AS first_name, argMax(last_name, p.last_seen_at) AS last_name, argMax(email, p.last_seen_at) AS email, argMax(avatar, p.last_seen_at) AS avatar, argMax(properties, p.last_seen_at) AS properties, project_id, argMax(is_external, p.last_seen_at) AS is_external, argMax(created_at, p.last_seen_at) AS created_at, max(p.last_seen_at) AS last_seen_at, argMax(groups, p.last_seen_at) AS groups FROM profiles AS p';
+/** The inner id query of a page or count up to `end`, with its placeholders unnumbered. */
+function innerIdsOf(text: string, end: number): string {
+ return text
+ .slice(text.indexOf('SELECT id FROM profiles WHERE'), end)
+ .replace(/\{p\d+:/g, '{p:');
+}
const WINDOW = {
startDate: '2026-06-16 00:00:00',
endDate: '2026-09-16 23:59:59',
};
const COLUMN_LIST = PROFILE_COLUMNS.join(', ');
-const PLAIN_WHERE = /\sWHERE\s/;
function collapse(text: string): string {
return text.replace(/\s+/g, ' ').trim();
@@ -141,24 +153,26 @@ describe('single-profile lookups', () => {
});
describe('profileListQuery', () => {
- test('plain page: project + window, no OFFSET at offset 0', () => {
+ test('plain page: newest version of the ids in the window, no OFFSET at offset 0', () => {
const { query, query_params } = profileListQuery({
projectId: PROJECT_ID,
take: 50,
offset: 0,
- filterClauses: NO_FILTERS,
+ ...NO_FILTER_CLAUSES,
...WINDOW,
}).toStatement();
const text = collapse(query);
expect(text).toBe(
- 'SELECT * FROM profiles FINAL WHERE project_id = {p1:String} AND created_at BETWEEN toDateTime({p2:String}) AND toDateTime({p3:String}) ORDER BY created_at DESC LIMIT {p4:UInt64}'
+ `${LATEST_COLUMNS_PAGE} WHERE p.project_id = {p1:String} AND p.id IN (SELECT id FROM profiles WHERE project_id = {p2:String} AND created_at BETWEEN toDateTime({p3:String}) AND toDateTime({p4:String}) GROUP BY id ORDER BY argMax(created_at, last_seen_at) DESC LIMIT {p5:UInt64}) GROUP BY p.id, p.project_id ORDER BY created_at DESC`
);
+ expect(text).not.toContain('FINAL');
expect(query_params).toEqual({
p1: PROJECT_ID,
- p2: '2026-06-16 00:00:00',
- p3: '2026-09-16 23:59:59',
- p4: 50,
+ p2: PROJECT_ID,
+ p3: '2026-06-16 00:00:00',
+ p4: '2026-09-16 23:59:59',
+ p5: 50,
});
});
@@ -170,52 +184,57 @@ describe('profileListQuery', () => {
search: 'ann',
isExternal: true,
filterClauses: { f0: sql`(properties['plan'] = 'pro')` },
+ latestFilterClauses: {
+ f0: sql`(argMax(properties['plan'], last_seen_at) = 'pro')`,
+ },
...WINDOW,
}).toStatement();
const text = collapse(query);
+ // The raw-row conditions only narrow the id set (a superset); the same
+ // conditions on the newest version decide, in HAVING.
expect(text).toContain(
- "WHERE project_id = {p1:String} AND created_at BETWEEN toDateTime({p2:String}) AND toDateTime({p3:String}) AND ((id ILIKE {p4:String} OR email ILIKE {p5:String} OR first_name ILIKE {p6:String} OR last_name ILIKE {p7:String} OR concat(first_name, ' ', last_name) ILIKE {p8:String})) AND is_external = {p9:Bool} AND (properties['plan'] = 'pro') ORDER BY created_at DESC LIMIT {p10:UInt64} OFFSET {p11:UInt64}"
+ "(SELECT id FROM profiles WHERE project_id = {p2:String} AND created_at BETWEEN toDateTime({p3:String}) AND toDateTime({p4:String}) AND cityHash64(id) IN (SELECT cityHash64(id) FROM profiles WHERE project_id = {p5:String} AND created_at BETWEEN toDateTime({p6:String}) AND toDateTime({p7:String}) AND ((id ILIKE {p8:String} OR email ILIKE {p9:String} OR first_name ILIKE {p10:String} OR last_name ILIKE {p11:String} OR concat(first_name, ' ', last_name) ILIKE {p12:String})) AND is_external = {p13:Bool} AND (properties['plan'] = 'pro')) GROUP BY id HAVING ((id ILIKE {p14:String} OR argMax(email, last_seen_at) ILIKE {p15:String} OR argMax(first_name, last_seen_at) ILIKE {p16:String} OR argMax(last_name, last_seen_at) ILIKE {p17:String} OR concat(argMax(first_name, last_seen_at), ' ', argMax(last_name, last_seen_at)) ILIKE {p18:String})) AND argMax(is_external, last_seen_at) = {p19:Bool} AND (argMax(properties['plan'], last_seen_at) = 'pro') ORDER BY argMax(created_at, last_seen_at) DESC LIMIT {p20:UInt64} OFFSET {p21:UInt64})"
);
expect(query_params).toMatchObject({
p1: PROJECT_ID,
- p4: '%ann%',
- p9: true,
- p10: 50,
- p11: 100,
+ p8: '%ann%',
+ p13: true,
+ p14: '%ann%',
+ p19: true,
+ p20: 50,
+ p21: 100,
});
});
- test('count shares the exact WHERE of the page (minus FINAL)', () => {
+ test('count counts exactly the ids the page selects from', () => {
const list = {
projectId: PROJECT_ID,
search: 'ann',
isExternal: false,
filterClauses: { f0: sql`(email != '')` },
+ latestFilterClauses: { f0: sql`(argMax(email, last_seen_at) != '')` },
...WINDOW,
};
const page = collapse(
profileListQuery({ ...list, take: 1, offset: 0 }).toStatement().query
);
const count = collapse(profileListCountQuery(list).toStatement().query);
- const whereOf = (text: string) =>
- text
- .slice(text.indexOf('WHERE'))
- .replace(/ ORDER BY.*$/, '')
- .replace(/ GROUP BY.*$/, '');
expect(count).toStartWith(
- 'SELECT uniqExact(id) as count FROM profiles WHERE'
+ 'SELECT count() as count FROM (SELECT id FROM profiles WHERE'
+ );
+ expect(count).not.toContain('FINAL');
+ expect(innerIdsOf(count, count.lastIndexOf(')'))).toBe(
+ innerIdsOf(page, page.indexOf(' ORDER BY'))
);
- expect(count).toEndWith('GROUP BY project_id');
- expect(whereOf(count)).toBe(whereOf(page));
});
});
describe('findProfilesQuery', () => {
const base = {
projectId: PROJECT_ID,
- filterClauses: NO_FILTERS,
+ ...NO_FILTER_CLAUSES,
sortOrder: 'desc' as const,
limit: 25,
};
@@ -224,9 +243,9 @@ describe('findProfilesQuery', () => {
const { query, query_params } = findProfilesQuery(base).toStatement();
expect(collapse(query)).toBe(
- `SELECT ${COLUMN_LIST} FROM profiles FINAL WHERE project_id = {p1:String} ORDER BY created_at DESC LIMIT {p2:UInt64}`
+ `${LATEST_COLUMNS_PAGE} WHERE p.project_id = {p1:String} AND p.id IN (SELECT id FROM profiles WHERE project_id = {p2:String} GROUP BY id ORDER BY argMax(created_at, last_seen_at) DESC LIMIT {p3:UInt64}) GROUP BY p.id, p.project_id ORDER BY created_at DESC`
);
- expect(query_params).toEqual({ p1: PROJECT_ID, p2: 25 });
+ expect(query_params).toEqual({ p1: PROJECT_ID, p2: PROJECT_ID, p3: 25 });
});
test('every optional condition binds and none interpolates', () => {
@@ -243,36 +262,45 @@ describe('findProfilesQuery', () => {
minSessions: 3,
performedEvent: HOSTILE,
filterClauses: { f0: sql`(1 = 1)` },
+ latestFilterClauses: { f0: sql`(2 = 2)` },
}).toStatement();
const text = collapse(query);
- expect(text).toContain('AND email ILIKE {p2:String}');
+ // The id-level conditions filter rows; the column conditions narrow the
+ // candidate ids on the raw rows and decide on the newest version in HAVING.
+ expect(text).toContain(
+ "AND id NOT IN ( SELECT DISTINCT profile_id FROM events WHERE project_id = {p3:String} AND profile_id != '' AND created_at >= now() - INTERVAL {p4:UInt64} DAY )"
+ );
+ expect(text).toContain(
+ "AND id IN ( SELECT profile_id FROM sessions WHERE project_id = {p5:String} AND sign = 1 AND profile_id != '' GROUP BY profile_id HAVING count() >= {p6:UInt64} )"
+ );
expect(text).toContain(
- 'AND properties[{p8:String}] = {p9:String} AND properties[{p10:String}] = {p11:String} AND properties[{p12:String}] = {p13:String} AND properties[{p14:String}] = {p15:String}'
+ 'AND id IN ( SELECT DISTINCT profile_id FROM events WHERE project_id = {p7:String} AND name = {p8:String} )'
);
expect(text).toContain(
- "AND id NOT IN ( SELECT DISTINCT profile_id FROM events WHERE project_id = {p16:String} AND profile_id != '' AND created_at >= now() - INTERVAL {p17:UInt64} DAY )"
+ 'AND cityHash64(id) IN (SELECT cityHash64(id) FROM profiles WHERE project_id = {p9:String} AND email ILIKE {p10:String} AND ((id ILIKE {p11:String}'
);
expect(text).toContain(
- "AND id IN ( SELECT profile_id FROM sessions WHERE project_id = {p18:String} AND sign = 1 AND profile_id != '' GROUP BY profile_id HAVING count() >= {p19:UInt64} )"
+ 'AND properties[{p16:String}] = {p17:String} AND properties[{p18:String}] = {p19:String} AND properties[{p20:String}] = {p21:String} AND properties[{p22:String}] = {p23:String} AND (1 = 1)) GROUP BY id HAVING argMax(email, last_seen_at) ILIKE {p24:String}'
);
expect(text).toContain(
- 'AND id IN ( SELECT DISTINCT profile_id FROM events WHERE project_id = {p20:String} AND name = {p21:String} )'
+ 'AND argMax(properties, last_seen_at)[{p30:String}] = {p31:String}'
);
expect(text).toEndWith(
- 'AND (1 = 1) ORDER BY created_at ASC LIMIT {p22:UInt64}'
+ 'AND (2 = 2) ORDER BY argMax(created_at, last_seen_at) ASC LIMIT {p38:UInt64}) GROUP BY p.id, p.project_id ORDER BY created_at ASC'
);
expect(query).not.toContain(HOSTILE);
expect(query_params).toMatchObject({
- p2: `%${HOSTILE}%`,
- p8: 'country',
- p9: 'SE',
- p10: 'city',
- p11: HOSTILE,
- p17: 7,
- p19: 3,
- p21: HOSTILE,
- p22: 25,
+ p4: 7,
+ p6: 3,
+ p8: HOSTILE,
+ p10: `%${HOSTILE}%`,
+ p16: 'country',
+ p17: 'SE',
+ p18: 'city',
+ p19: HOSTILE,
+ p24: `%${HOSTILE}%`,
+ p38: 25,
});
});
@@ -285,8 +313,8 @@ describe('findProfilesQuery', () => {
}).toStatement().query
);
expect(zero).toContain('NOT IN');
- expect(zero).toContain('INTERVAL {p3:UInt64} DAY');
- expect(zero).toContain('HAVING count() >= {p5:UInt64}');
+ expect(zero).toContain('INTERVAL {p4:UInt64} DAY');
+ expect(zero).toContain('HAVING count() >= {p6:UInt64}');
});
});
diff --git a/packages/core/src/modules/profile/src/sql.ts b/packages/core/src/modules/profile/src/sql.ts
index fd14d79ac..9b4cffa8a 100644
--- a/packages/core/src/modules/profile/src/sql.ts
+++ b/packages/core/src/modules/profile/src/sql.ts
@@ -7,7 +7,6 @@ import { toRangeBoundaryLiteral } from '../../../shared/ch-dates';
import {
type CompiledFilterClauses,
compiledFilterFragments,
- spliceCompiledFilters,
} from './filter-clauses';
const TABLE = {
@@ -37,6 +36,19 @@ const PROFILE_COLUMNS_FRAGMENT = sql.join(
const PROFILE_KEY_COLUMNS = ['id', 'project_id'] as const;
+type ProfileColumn = (typeof PROFILE_COLUMNS)[number];
+
+/** A column on the raw rows, or its newest version per id (for a `GROUP BY id`). */
+export type ProfileColumnAccess = (column: ProfileColumn) => SqlFragment;
+
+export const profileColumn: ProfileColumnAccess = (column) =>
+ sql.id(column, PROFILE_COLUMNS);
+
+export const latestProfileColumn: ProfileColumnAccess = (column) =>
+ column === 'id'
+ ? sql.id(column, PROFILE_COLUMNS)
+ : sql`argMax(${sql.id(column, PROFILE_COLUMNS)}, last_seen_at)`;
+
/**
* The newest version of every column, as `profiles FINAL` would return it,
* for a `GROUP BY id, project_id` over the raw rows. `last_seen_at` is the
@@ -86,7 +98,8 @@ export interface ProfileWindow {
* when the search is blank.
*/
export function profileSearchCondition(
- search: string | null | undefined
+ search: string | null | undefined,
+ column: ProfileColumnAccess = profileColumn
): SqlFragment | null {
const tokens = (search ?? '')
.trim()
@@ -98,7 +111,7 @@ export function profileSearchCondition(
}
const perToken = tokens.map((token) => {
const like = sql.string(`%${token}%`);
- return sql`(id ILIKE ${like} OR email ILIKE ${like} OR first_name ILIKE ${like} OR last_name ILIKE ${like} OR concat(first_name, ' ', last_name) ILIKE ${like})`;
+ return sql`(${column('id')} ILIKE ${like} OR ${column('email')} ILIKE ${like} OR ${column('first_name')} ILIKE ${like} OR ${column('last_name')} ILIKE ${like} OR concat(${column('first_name')}, ' ', ${column('last_name')}) ILIKE ${like})`;
});
return sql`(${sql.join(perToken, ' AND ')})`;
}
@@ -197,52 +210,147 @@ export interface ProfileListQuery extends ProfileWindow {
projectId: string;
search?: string;
isExternal?: boolean;
- /** Pre-compiled by `buildFilterWhere` (src/filter-clauses.ts). */
+ /** Pre-compiled by `buildFilterWhere` on the raw rows (src/filter-clauses.ts). */
filterClauses: CompiledFilterClauses;
+ /** The same filters compiled on the newest version (`latestVersion: true`). */
+ latestFilterClauses: CompiledFilterClauses;
+}
+
+/**
+ * Ids whose newest version satisfies `having`, ranked by first seen.
+ *
+ * `scope` (project, first-seen window) and `where` (id-level subqueries) are
+ * the same for every version of a profile, so they may filter rows.
+ * `candidates` holds the column conditions on the raw rows: they only narrow
+ * the id set, through a subquery, and never the rows the aggregate reads —
+ * otherwise argMax would see nothing but the matching stale version and the
+ * HAVING clause would pass on it. The set is a superset of the answer, since
+ * the surviving version is one of the rows, and holds hashes: a String set
+ * costs ~300 bytes per id, and a collision only adds a candidate.
+ */
+interface LatestProfileIdsPage {
+ scope: SqlFragment;
+ where?: SqlFragment[];
+ candidates: SqlFragment[];
+ having: SqlFragment[];
+}
+
+function latestProfileIdsWhere(page: LatestProfileIdsPage): SqlFragment {
+ const conditions = [page.scope, ...(page.where ?? [])];
+ if (page.candidates.length) {
+ conditions.push(
+ sql`cityHash64(id) IN (SELECT cityHash64(id) FROM ${sql.id(TABLE.profiles)} WHERE ${page.scope} AND ${sql.join(page.candidates, ' AND ')})`
+ );
+ }
+ const having = page.having.length
+ ? sql` HAVING ${sql.join(page.having, ' AND ')}`
+ : sql.empty;
+ return sql`WHERE ${sql.join(conditions, ' AND ')} GROUP BY id${having}`;
+}
+
+function latestProfileIdsQuery(
+ page: LatestProfileIdsPage & {
+ orderDirection: SqlFragment;
+ limit: number;
+ offset?: number;
+ }
+): SqlFragment {
+ return sql`SELECT id FROM ${sql.id(TABLE.profiles)} ${latestProfileIdsWhere(page)} ORDER BY ${latestProfileColumn('created_at')} ${page.orderDirection} LIMIT ${sql.uint64(page.limit)}${optional(
+ page.offset,
+ () => sql` OFFSET ${sql.uint64(page.offset as number)}`
+ )}`;
+}
+
+/** The page's rows, newest version of every column, in the ids' order. */
+function latestProfilesPage(
+ projectId: string,
+ ids: SqlFragment,
+ orderDirection: SqlFragment
+): SqlFragment {
+ return sql`
+ SELECT ${PROFILE_LATEST_COLUMNS_FRAGMENT}
+ FROM ${sql.id(TABLE.profiles)} AS p
+ WHERE p.project_id = ${sql.string(projectId)} AND p.id IN (${ids})
+ GROUP BY p.id, p.project_id
+ ORDER BY created_at ${orderDirection}
+ `;
}
/**
- * `WHERE project_id =.. [AND search] [AND is_external =..] [AND filters]`,
- * bounded to a window rather than reading every profile the project ever
- * had — otherwise the cost grows with the tenant's lifetime rather than with
- * anything the caller asked for. `created_at` is the column the list already
- * orders by, so the bound and the order agree.
+ * `project_id =.. AND created_at BETWEEN ..`: bounded to a window rather than
+ * reading every profile the project ever had — otherwise the cost grows with
+ * the tenant's lifetime rather than with anything the caller asked for.
+ * `created_at` is the column the list already orders by, so the bound and
+ * the order agree; the write path keeps it the same across versions.
*/
-function profileListCondition(query: ProfileListQuery): SqlFragment {
- const search = profileSearchCondition(query.search);
+function profileListScope(query: ProfileListQuery): SqlFragment {
return sql`project_id = ${sql.string(query.projectId)} AND ${dateRangeWhere(
'created_at',
query.startDate,
query.endDate
- )}${optional(search, () => sql` AND ${search as SqlFragment}`)}${optional(
- query.isExternal !== undefined,
- () => sql` AND is_external = ${sql.bool(query.isExternal as boolean)}`
- )} ${spliceCompiledFilters(query.filterClauses)}`;
+ )}`;
+}
+
+/**
+ * `[search] [is_external =..] [filters]` as conditions on one version:
+ * the raw rows (candidates) or the newest one per id (the answer).
+ */
+function profileListConditions(
+ query: ProfileListQuery,
+ column: ProfileColumnAccess,
+ filterClauses: CompiledFilterClauses
+): SqlFragment[] {
+ const conditions: SqlFragment[] = [];
+ const search = profileSearchCondition(query.search, column);
+ if (search) {
+ conditions.push(search);
+ }
+ if (query.isExternal !== undefined) {
+ conditions.push(
+ sql`${column('is_external')} = ${sql.bool(query.isExternal)}`
+ );
+ }
+ conditions.push(...compiledFilterFragments(filterClauses));
+ return conditions;
+}
+
+function profileListPage(query: ProfileListQuery): LatestProfileIdsPage {
+ return {
+ scope: profileListScope(query),
+ candidates: profileListConditions(
+ query,
+ profileColumn,
+ query.filterClauses
+ ),
+ having: profileListConditions(
+ query,
+ latestProfileColumn,
+ query.latestFilterClauses
+ ),
+ };
}
export function profileListQuery(
query: ProfileListQuery & { take: number; offset: number }
): SqlFragment {
- return sql`
- SELECT *
- FROM ${sql.id(TABLE.profiles)} FINAL
- WHERE ${profileListCondition(query)}
- ORDER BY created_at DESC
- LIMIT ${sql.uint64(query.take)}
- ${optional(query.offset, () => sql`OFFSET ${sql.uint64(query.offset)}`)}
- `;
+ const orderDirection = sql`DESC`;
+ return latestProfilesPage(
+ query.projectId,
+ latestProfileIdsQuery({
+ ...profileListPage(query),
+ orderDirection,
+ limit: query.take,
+ offset: query.offset,
+ }),
+ orderDirection
+ );
}
-// `uniqExact`, not `count`: the page reads `profiles FINAL` so it sees one row
-// per id, while the count deliberately skips FINAL (it is the expensive half
-// on a large tenant). Counting rows therefore counted every unmerged
-// ReplacingMergeTree version and the total ran ahead of the list.
+/** One per profile whose newest version matches: exactly what the page shows. */
export function profileListCountQuery(query: ProfileListQuery): SqlFragment {
return sql`
- SELECT uniqExact(id) as count
- FROM ${sql.id(TABLE.profiles)}
- WHERE ${profileListCondition(query)}
- GROUP BY project_id
+ SELECT count() as count
+ FROM (SELECT id FROM ${sql.id(TABLE.profiles)} ${latestProfileIdsWhere(profileListPage(query))})
`;
}
@@ -260,6 +368,7 @@ export interface FindProfilesQuery {
minSessions?: number;
performedEvent?: string;
filterClauses: CompiledFilterClauses;
+ latestFilterClauses: CompiledFilterClauses;
sortOrder: 'asc' | 'desc';
limit: number;
}
@@ -271,28 +380,52 @@ const PROFILE_PROPERTY_FILTERS = [
'browser',
] as const;
-function findProfilesConditions(query: FindProfilesQuery): SqlFragment[] {
+/**
+ * `where` holds the id-level conditions, the same for every version;
+ * `candidates` every column condition on the raw rows and `having` the same
+ * on the newest version (see latestProfileIdsQuery).
+ */
+function findProfilesConditions(query: FindProfilesQuery): {
+ where: SqlFragment[];
+ candidates: SqlFragment[];
+ having: SqlFragment[];
+} {
const projectId = sql.string(query.projectId);
- const conditions: SqlFragment[] = [sql`project_id = ${projectId}`];
+ const where: SqlFragment[] = [];
+ const candidates: SqlFragment[] = [];
+ const having: SqlFragment[] = [];
+ const onBothVersions = (
+ condition: (column: ProfileColumnAccess) => SqlFragment | null
+ ) => {
+ const anyVersion = condition(profileColumn);
+ const latestVersion = condition(latestProfileColumn);
+ if (anyVersion && latestVersion) {
+ candidates.push(anyVersion);
+ having.push(latestVersion);
+ }
+ };
if (query.email) {
- conditions.push(sql`email ILIKE ${sql.string(`%${query.email}%`)}`);
+ onBothVersions(
+ (column) =>
+ sql`${column('email')} ILIKE ${sql.string(`%${query.email}%`)}`
+ );
}
- const nameCondition = profileSearchCondition(query.name);
- if (query.name && nameCondition) {
- conditions.push(nameCondition);
+ if (query.name) {
+ onBothVersions((column) => profileSearchCondition(query.name, column));
}
for (const key of PROFILE_PROPERTY_FILTERS) {
const value = query[key];
if (value) {
- conditions.push(
- sql`properties[${sql.string(key)}] = ${sql.string(value)}`
+ onBothVersions(
+ (column) =>
+ sql`${column('properties')}[${sql.string(key)}] = ${sql.string(value)}`
);
}
}
if (query.inactiveDays !== undefined) {
- conditions.push(sql`id NOT IN (
+ where.push(sql`id NOT IN (
SELECT DISTINCT profile_id FROM ${sql.id(TABLE.events)}
WHERE project_id = ${projectId}
AND profile_id != ''
@@ -301,7 +434,7 @@ function findProfilesConditions(query: FindProfilesQuery): SqlFragment[] {
}
if (query.minSessions !== undefined) {
- conditions.push(sql`id IN (
+ where.push(sql`id IN (
SELECT profile_id FROM ${sql.id(TABLE.sessions)}
WHERE project_id = ${projectId}
AND sign = 1
@@ -312,27 +445,34 @@ function findProfilesConditions(query: FindProfilesQuery): SqlFragment[] {
}
if (query.performedEvent) {
- conditions.push(sql`id IN (
+ where.push(sql`id IN (
SELECT DISTINCT profile_id FROM ${sql.id(TABLE.events)}
WHERE project_id = ${projectId}
AND name = ${sql.string(query.performedEvent)}
)`);
}
- conditions.push(...compiledFilterFragments(query.filterClauses));
+ candidates.push(...compiledFilterFragments(query.filterClauses));
+ having.push(...compiledFilterFragments(query.latestFilterClauses));
- return conditions;
+ return { where, candidates, having };
}
export function findProfilesQuery(query: FindProfilesQuery): SqlFragment {
const orderDirection = query.sortOrder === 'asc' ? sql`ASC` : sql`DESC`;
- return sql`
- SELECT ${PROFILE_COLUMNS_FRAGMENT}
- FROM ${sql.id(TABLE.profiles)} FINAL
- WHERE ${sql.join(findProfilesConditions(query), ' AND ')}
- ORDER BY created_at ${orderDirection}
- LIMIT ${sql.uint64(query.limit)}
- `;
+ const { where, candidates, having } = findProfilesConditions(query);
+ return latestProfilesPage(
+ query.projectId,
+ latestProfileIdsQuery({
+ scope: sql`project_id = ${sql.string(query.projectId)}`,
+ where,
+ candidates,
+ having,
+ orderDirection,
+ limit: query.limit,
+ }),
+ orderDirection
+ );
}
export function profileRowQuery(query: {
diff --git a/packages/core/src/modules/profile/src/sql.versions.test.ts b/packages/core/src/modules/profile/src/sql.versions.test.ts
index 3e7c49b42..916eeb4f1 100644
--- a/packages/core/src/modules/profile/src/sql.versions.test.ts
+++ b/packages/core/src/modules/profile/src/sql.versions.test.ts
@@ -5,6 +5,7 @@
import { afterAll, beforeAll, describe, expect, it } from 'bun:test';
import { sql } from '@openpanel/db/src/clickhouse/sql';
+import { timeAgo } from '../../../../../../test/fixtures';
import {
otherProjectId,
setupProfileVersions,
@@ -12,9 +13,14 @@ import {
VERSIONED,
} from '../../../../../../test/profile-versions';
import { expectSameRows, fetchRows } from '../../../../test/clickhouse-rows';
+import { buildFilterWhere } from '../../chart/src/table-filter-where';
+import type { IChartEventFilter } from '../../report/report.constants';
import {
+ findProfilesQuery,
PROFILE_COLUMNS,
profileByIdQuery,
+ profileListCountQuery,
+ profileListQuery,
profileMetricsQuery,
profileRowQuery,
profilesByIdsQuery,
@@ -27,6 +33,7 @@ const columns = sql.join(
);
let ch: typeof import('@openpanel/db/src/clickhouse/client').ch;
+let now: Date;
beforeAll(async () => {
({ ch } = await import('@openpanel/db/src/clickhouse/client'));
@@ -34,7 +41,7 @@ beforeAll(async () => {
'../../../../../../test/bootstrap-databases'
);
await bootstrapTestDatabases();
- await setupProfileVersions(PROJECT_ID);
+ now = await setupProfileVersions(PROJECT_ID);
}, 30_000);
afterAll(() => teardownProfileVersions(PROJECT_ID));
@@ -141,3 +148,128 @@ describe('profileMetricsQuery', () => {
expect(JSON.parse(metrics as string)).toMatchObject(expected);
});
});
+
+const PROFILE_TARGET = {
+ selfTable: 'profiles',
+ profileIdExpr: 'id',
+ groupsExpr: 'groups',
+} as const;
+const planIs = (value: string): IChartEventFilter => ({
+ id: 'f',
+ name: 'profile.properties.plan',
+ operator: 'is',
+ value: [value],
+});
+function compiled(filters: IChartEventFilter[]) {
+ return {
+ filterClauses: buildFilterWhere(filters, PROJECT_ID, PROFILE_TARGET),
+ latestFilterClauses: buildFilterWhere(filters, PROJECT_ID, {
+ ...PROFILE_TARGET,
+ latestVersion: true,
+ }),
+ };
+}
+const ids = (rows: string[]) => rows.map((row) => JSON.parse(row).id);
+
+describe('profileListQuery / profileListCountQuery', () => {
+ const window = () => ({
+ startDate: timeAgo(now, 60),
+ endDate: timeAgo(now, 0),
+ });
+ const list = (
+ overrides: Partial<Parameters<typeof profileListQuery>[0]> = {},
+ filters: IChartEventFilter[] = []
+ ) =>
+ profileListQuery({
+ projectId: PROJECT_ID,
+ take: 50,
+ offset: 0,
+ ...window(),
+ ...compiled(filters),
+ ...overrides,
+ });
+ const count = (
+ overrides: Partial<Parameters<typeof profileListCountQuery>[0]> = {},
+ filters: IChartEventFilter[] = []
+ ) =>
+ profileListCountQuery({
+ projectId: PROJECT_ID,
+ ...window(),
+ ...compiled(filters),
+ ...overrides,
+ });
+ const countOf = async (fragment: ReturnType<typeof count>) =>
+ Number(JSON.parse((await fetchRows(ch, fragment))[0] as string).count);
+
+ it('pages the newest version of every profile in the window, like FINAL did', async () => {
+ const reference = sql`SELECT * FROM profiles FINAL WHERE project_id = ${sql.string(PROJECT_ID)} AND created_at BETWEEN toDateTime(${sql.string(timeAgo(now, 60))}) AND toDateTime(${sql.string(timeAgo(now, 0))}) ORDER BY created_at DESC LIMIT 50`;
+ const rows = await expectSameRows(ch, reference, list(), 3);
+ expect(ids(rows).sort()).toEqual(
+ [VERSIONED.multi, VERSIONED.staleMatch, VERSIONED.single].sort()
+ );
+ expect(await countOf(count())).toBe(3);
+ });
+
+ it('keeps the newest-first order and the offset', async () => {
+ const page = (await fetchRows(ch, list({ take: 2, offset: 1 }))).map(
+ (row) => JSON.parse(row)
+ );
+ const ordered = page.sort((a, b) => (a.created_at < b.created_at ? 1 : -1));
+ expect(ordered.map((row) => row.id)).toEqual([
+ VERSIONED.staleMatch,
+ VERSIONED.multi,
+ ]);
+ });
+
+ it('filters, searches and is_external read the newest version', async () => {
+ expect(ids(await fetchRows(ch, list({}, [planIs('pro')])))).toEqual([
+ VERSIONED.single,
+ ]);
+ expect(await countOf(count({}, [planIs('pro')]))).toBe(1);
+ expect(ids(await fetchRows(ch, list({ search: 'mia' })))).toEqual([
+ VERSIONED.multi,
+ ]);
+ expect(ids(await fetchRows(ch, list({ search: 'Old' })))).toEqual([]);
+ expect(await countOf(count({ isExternal: true }))).toBe(3);
+ expect(await countOf(count({ isExternal: false }))).toBe(0);
+ });
+});
+
+describe('findProfilesQuery', () => {
+ const find = (
+ overrides: Partial<Parameters<typeof findProfilesQuery>[0]> = {},
+ filters: IChartEventFilter[] = []
+ ) =>
+ findProfilesQuery({
+ projectId: PROJECT_ID,
+ sortOrder: 'desc',
+ limit: 20,
+ ...compiled(filters),
+ ...overrides,
+ });
+
+ it('matches the FINAL form it replaced, row for row', async () => {
+ const reference = sql`SELECT ${columns} FROM profiles FINAL WHERE project_id = ${sql.string(PROJECT_ID)} ORDER BY created_at DESC LIMIT 20`;
+ await expectSameRows(ch, reference, find(), 3);
+ });
+
+ it('every column condition reads the newest version', async () => {
+ expect(ids(await fetchRows(ch, find({ email: 'mia' })))).toEqual([
+ VERSIONED.multi,
+ ]);
+ expect(ids(await fetchRows(ch, find({ email: 'old.example' })))).toEqual(
+ []
+ );
+ expect(ids(await fetchRows(ch, find({ country: 'US' })))).toEqual([
+ VERSIONED.staleMatch,
+ ]);
+ expect(ids(await fetchRows(ch, find({ name: 'Kim-Lee' })))).toEqual([
+ VERSIONED.multi,
+ ]);
+ expect(
+ ids(
+ await fetchRows(ch, find({ performedEvent: 'signup' }, [planIs('pro')]))
+ )
+ ).toEqual([VERSIONED.single]);
+ });
+});