OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.
perf(api): profile lists, counts, member pages and filters drop FINAL
Carl-Gerhard Lindesvärd0f226d003d29
diff --git a/packages/core/src/modules/chart/src/table-filter-where.test.ts b/packages/core/src/modules/chart/src/table-filter-where.test.tsindex 84369aa57..0f348fb73 100644--- a/packages/core/src/modules/chart/src/table-filter-where.test.ts+++ b/packages/core/src/modules/chart/src/table-filter-where.test.ts@@ -98,11 +98,28 @@ function literal(value: SqlParamValue | undefined): string {return String(value);}-/** What `buildProfileClause` wraps `inner` in for a given table. */-const profileWrap = (table: keyof typeof CONTEXTS, inner: string): string =>+/**+ * What `buildProfileClause` wraps the predicate in for a given table: on the+ * profiles table the predicate itself; elsewhere a semi-join that narrows the+ * candidate ids with it on the raw rows and decides on the newest version.+ */+const profileWrap = (+ table: keyof typeof CONTEXTS,+ inner: string,+ onLatest: string+): string =>table === 'profiles'? `(${inner})`- : `(profile_id IN (SELECT id FROM ${TABLE_NAMES.profiles} FINAL WHERE project_id = 'p' AND ${inner}))`;+ : `(cityHash64(profile_id) IN (SELECT cityHash64(id) FROM (SELECT id FROM ${TABLE_NAMES.profiles} WHERE project_id = 'p' AND cityHash64(id) IN (SELECT cityHash64(id) FROM ${TABLE_NAMES.profiles} WHERE project_id = 'p' AND ${inner}) GROUP BY id HAVING ${onLatest})))`;++const latestOf = (column: string): string => {+ if (column === 'id') {+ return 'id';+ }+ return column === 'last_seen_at'+ ? 'max(last_seen_at)'+ : `argMax(${column}, last_seen_at)`;+};const ALLOWED_STRING_COLUMNS = ['id',@@ -130,7 +147,9 @@ describe.each(TABLES)('buildFilterWhere on %s', (table) => {const where = build(table, [filter({ name: `profile.${column}`, value: ['a'] }),]);- expect(where.f0).toBe(profileWrap(table, `${column} = 'a'`));+ expect(where.f0).toBe(+ profileWrap(table, `${column} = 'a'`, `${latestOf(column)} = 'a'`)+ );}});@@ -140,7 +159,11 @@ describe.each(TABLES)('buildFilterWhere on %s', (table) => {filter({ name: `profile.${column}`, operator: 'gt', value: ['123'] }),]);expect(where.f0).toBe(- profileWrap(table, `(toFloat64(${column}) > toFloat64('123'))`)+ profileWrap(+ table,+ `(toFloat64(${column}) > toFloat64('123'))`,+ `(toFloat64(${latestOf(column)}) > toFloat64('123'))`+ ));}});@@ -149,7 +172,13 @@ describe.each(TABLES)('buildFilterWhere on %s', (table) => {const where = build(table, [filter({ name: 'profile.properties.plan', value: ['pro'] }),]);- expect(where.f0).toBe(profileWrap(table, "properties['plan'] = 'pro'"));+ expect(where.f0).toBe(+ profileWrap(+ table,+ "properties['plan'] = 'pro'",+ "argMax(properties['plan'], last_seen_at) = 'pro'"+ )+ );});it('escapes a quote in a properties key', () => {@@ -157,7 +186,13 @@ describe.each(TABLES)('buildFilterWhere on %s', (table) => {filter({ name: "profile.properties.pl'an", value: ['pro'] }),];const where = build(table, filters);- expect(where.f0).toBe(profileWrap(table, "properties['pl\\'an'] = 'pro'"));+ expect(where.f0).toBe(+ profileWrap(+ table,+ "properties['pl\\'an'] = 'pro'",+ "argMax(properties['pl\\'an'], last_seen_at) = 'pro'"+ )+ );// The key is a bound param now, so the quote never reaches the SQL text.expect(Object.values(paramsOf(table, filters)[0]!)).toContain("pl'an");});@@ -177,7 +212,13 @@ describe.each(TABLES)('buildFilterWhere on %s', (table) => {filter({ name: 'profile.not a column', value: ['a'] }),]);expect(Object.keys(where)).toEqual(['f0']);- expect(where.f0).toBe(profileWrap(table, "email = '[email protected]'"));+ expect(where.f0).toBe(+ profileWrap(+ table,+ "email = '[email protected]'",+ "argMax(email, last_seen_at) = '[email protected]'"+ )+ );});it('keys surviving filters by their original index', () => {diff --git a/packages/core/src/modules/chart/src/table-filter-where.ts b/packages/core/src/modules/chart/src/table-filter-where.tsindex 2d37bc6b4..3e495c2d3 100644--- a/packages/core/src/modules/chart/src/table-filter-where.ts+++ b/packages/core/src/modules/chart/src/table-filter-where.ts@@ -17,7 +17,11 @@ import {} from '../../report/report.constants';import { PROFILE_SELECT_COLUMNS } from '../chart.constants';import { formatClickhouseDate } from './dates';-import { buildCohortMembersSubselect, CHART_TABLE } from './field-resolution';+import {+ buildCohortMembersSubselect,+ CHART_TABLE,+ latestProfileValue,+} from './field-resolution';import {buildTypedClauseFragment,hasTypedCast,@@ -27,6 +31,11 @@ import {export interface FilterTableContext {/** Outer query's primary table. */selfTable: 'sessions' | 'profiles' | 'events';+ /**+ * The outer query groups `profiles` by id, so a `profile.*` predicate reads+ * the newest version of its column — an aggregate, for a HAVING clause.+ */+ latestVersion?: boolean;/*** Expression on the outer row that yields the owning profile id. For the* profiles table this is `id`; for events/sessions it's `profile_id`. Used@@ -358,7 +367,10 @@ function buildGroupClause(if (!inner) {return null;}- return sql`arrayExists(g -> g IN (SELECT id FROM ${sql.id(CHART_TABLE.groups)} FINAL WHERE project_id = ${sql.string(projectId)} AND ${inner}), ${sql.id(ctx.groupsExpr)})`;+ const groups = ctx.latestVersion+ ? sql`argMax(${sql.id(ctx.groupsExpr)}, last_seen_at)`+ : sql.id(ctx.groupsExpr);+ return sql`arrayExists(g -> g IN (SELECT id FROM ${sql.id(CHART_TABLE.groups)} FINAL WHERE project_id = ${sql.string(projectId)} AND ${inner}), ${groups})`;}function buildProfileClause(@@ -372,17 +384,37 @@ function buildProfileClause(return null;}const numeric = name === 'created_at' || name === 'last_seen_at';- const inner = compileScalarClause(column, filter.operator, filter.value, {- numeric,- type: filter.type,- });- if (!inner) {- return null;- }+ const compile = (expression: SqlFragment) =>+ compileScalarClause(expression, filter.operator, filter.value, {+ numeric,+ type: filter.type,+ });+ const latest = latestProfileValue(name, column);if (ctx.selfTable === 'profiles') {- return inner;+ return compile(ctx.latestVersion ? latest : column);+ }+ const onAnyVersion = compile(column);+ const onLatestVersion = compile(latest);+ if (!(onAnyVersion && onLatestVersion)) {+ return null;}- return sql`${sql.id(ctx.profileIdExpr)} IN (SELECT id FROM ${sql.id(CHART_TABLE.profiles)} FINAL WHERE project_id = ${sql.string(projectId)} AND ${inner})`;+ return sql`cityHash64(${sql.id(ctx.profileIdExpr)}) IN (SELECT cityHash64(id) FROM ${latestProfilesMatching(projectId, onAnyVersion, onLatestVersion)})`;+}++/**+ * Ids whose newest version satisfies `onLatestVersion`, without `FINAL`. The+ * raw-row prefilter is a superset — the surviving version is one of the rows+ * — so the GROUP BY only sees candidates, not the whole project. The set+ * holds hashes: a String set costs ~300 bytes per id, and a collision only+ * adds a candidate the HAVING clause rejects.+ */+export function latestProfilesMatching(+ projectId: string,+ onAnyVersion: SqlFragment,+ onLatestVersion: SqlFragment+): SqlFragment {+ const project = sql`project_id = ${sql.string(projectId)}`;+ return sql`(SELECT id FROM ${sql.id(CHART_TABLE.profiles)} WHERE ${project} AND cityHash64(id) IN (SELECT cityHash64(id) FROM ${sql.id(CHART_TABLE.profiles)} WHERE ${project} AND ${onAnyVersion}) GROUP BY id HAVING ${onLatestVersion})`;}function buildSessionClause(diff --git a/packages/core/src/modules/chart/src/table-filter-where.versions.test.ts b/packages/core/src/modules/chart/src/table-filter-where.versions.test.tsnew file mode 100644index 000000000..604ecbcc2--- /dev/null+++ b/packages/core/src/modules/chart/src/table-filter-where.versions.test.ts@@ -0,0 +1,95 @@+// A `profile.*` filter on another table compiles to a semi-join that no+// longer reads `profiles FINAL`: it decides on the newest version of the+// column. Runs against profiles with several unmerged versions.++import { afterAll, beforeAll, describe, expect, it } from 'bun:test';+import { type SqlFragment, sql } from '@openpanel/db/src/clickhouse/sql';+import {+ setupProfileVersions,+ teardownProfileVersions,+ VERSIONED,+} from '../../../../../../test/profile-versions';+import { expectSameRows } from '../../../../test/clickhouse-rows';+import type { IChartEventFilter } from '../../report/report.constants';+import { buildFilterWhere } from './table-filter-where';++const PROJECT_ID = 'semi-join-profile-versions';+const ON_EVENTS = {+ selfTable: 'events',+ profileIdExpr: 'profile_id',+ groupsExpr: 'groups',+} as const;++let ch: typeof import('@openpanel/db/src/clickhouse/client').ch;++beforeAll(async () => {+ ({ ch } = await import('@openpanel/db/src/clickhouse/client'));+ const { bootstrapTestDatabases } = await import(+ '../../../../../../test/bootstrap-databases'+ );+ await bootstrapTestDatabases();+ await setupProfileVersions(PROJECT_ID);+}, 30_000);++afterAll(() => teardownProfileVersions(PROJECT_ID));++function eventProfiles(clause: SqlFragment) {+ return sql`SELECT DISTINCT profile_id FROM events WHERE project_id = ${sql.string(PROJECT_ID)} AND ${clause}`;+}++/** The semi-join as it was: `profile_id IN (SELECT id FROM profiles FINAL WHERE ... AND <predicate>)`. */+function finalSemiJoin(predicate: SqlFragment) {+ return sql`profile_id IN (SELECT id FROM profiles FINAL WHERE project_id = ${sql.string(PROJECT_ID)} AND ${predicate})`;+}++const filter = (+ name: string,+ operator: IChartEventFilter['operator'],+ value: string+): IChartEventFilter => ({ id: 'f', name, operator, value: [value] });++describe('buildFilterWhere on events with a profile.* filter', () => {+ const cases: [string, IChartEventFilter, SqlFragment, string[]][] = [+ [+ 'plan is pro: the stale pro version must not match',+ filter('profile.properties.plan', 'is', 'pro'),+ sql`properties['plan'] = 'pro'`,+ [VERSIONED.single],+ ],+ [+ 'plan isNot pro: decided on the newest version',+ filter('profile.properties.plan', 'isNot', 'pro'),+ sql`properties['plan'] != 'pro'`,+ [VERSIONED.multi, VERSIONED.staleMatch],+ ],+ [+ 'email is the old address: nothing matches',+ filter('profile.email', 'is', '[email protected]'),+ sql`email = '[email protected]'`,+ [],+ ],+ [+ 'last_name contains the new value',+ filter('profile.last_name', 'contains', 'Lee'),+ sql`last_name LIKE '%Lee%'`,+ [VERSIONED.multi],+ ],+ ];++ it.each(cases)('%s', async (_label, profileFilter, predicate, expected) => {+ const [clause] = Object.values(+ buildFilterWhere([profileFilter], PROJECT_ID, ON_EVENTS)+ );+ expect(clause).toBeDefined();+ expect(clause?.toStatement().query).not.toContain('FINAL');+ const rows = await expectSameRows(+ ch,+ eventProfiles(finalSemiJoin(predicate)),+ eventProfiles(clause as SqlFragment),+ expected.length+ );+ expect(rows.map((row) => JSON.parse(row).profile_id).sort()).toEqual(+ [...expected].sort()+ );+ });+});diff --git a/packages/core/src/modules/cohort/cohort.service.ts b/packages/core/src/modules/cohort/cohort.service.tsindex 7e42655e4..0f880513c 100644--- a/packages/core/src/modules/cohort/cohort.service.ts+++ b/packages/core/src/modules/cohort/cohort.service.ts@@ -865,40 +865,56 @@ export async function listCohortMemberProfiles(}): Promise<{ data: IServiceProfile[]; count: number }> {const { buildFilterWhere } = await import('../chart/src/table-filter-where');- const { getProfiles, profileSearchCondition } = await import(- '../profile/profile.service'- );+ const { getProfiles, latestProfileColumn, profileSearchCondition } =+ await import('../profile/profile.service');const offset = Math.max(0, (cursor ?? 0) * take);- const searchClause = profileSearchCondition(search);- const searchCondition = searchClause ? sql`AND ${searchClause}` : sql.empty;-- const extraConditions = filters?.length- ? Object.values(- buildFilterWhere(filters, projectId, {- selfTable: 'profiles',- profileIdExpr: 'id',- groupsExpr: 'groups',- })- )- : [];- const extraConditionSql = extraConditions.length- ? sql`AND ${sql.join(extraConditions, ' AND ')}`+ // The raw-row conditions only narrow the id set (a superset: the surviving+ // version is one of the rows), never the rows the aggregate reads; the+ // HAVING clause decides on the newest version.+ const conditionsOn = (latestVersion: boolean): SqlFragment[] => {+ const searchClause = profileSearchCondition(+ search,+ latestVersion ? latestProfileColumn : undefined+ );+ const filterClauses = filters?.length+ ? Object.values(+ buildFilterWhere(filters, projectId, {+ selfTable: 'profiles',+ profileIdExpr: 'id',+ groupsExpr: 'groups',+ latestVersion,+ })+ )+ : [];+ return searchClause ? [searchClause, ...filterClauses] : filterClauses;+ };+ const anyVersion = conditionsOn(false);+ const latestVersion = conditionsOn(true);+ const candidates = anyVersion.length+ ? sql`AND cityHash64(id) IN (SELECT cityHash64(id) FROM ${sql.id(TABLE.profiles)} WHERE project_id = ${sql.string(projectId)} AND ${sql.join(anyVersion, ' AND ')})`+ : sql.empty;+ const having = latestVersion.length+ ? sql`HAVING ${sql.join(latestVersion, ' AND ')}`: sql.empty;const rows = await chQuery<{ id: string; total_count: number }>(deps,sql`SELECT id, count() OVER () AS total_count- FROM ${sql.id(TABLE.profiles)} FINAL- WHERE project_id = ${sql.string(projectId)}- AND id IN (- SELECT profile_id FROM ${sql.id(TABLE.cohortMembers)} FINAL- WHERE cohort_id = ${sql.string(cohortId)}- AND project_id = ${sql.string(projectId)}- )- ${searchCondition}- ${extraConditionSql}+ FROM (+ SELECT id, argMax(created_at, last_seen_at) AS created_at+ FROM ${sql.id(TABLE.profiles)}+ WHERE project_id = ${sql.string(projectId)}+ AND id IN (+ SELECT profile_id FROM ${sql.id(TABLE.cohortMembers)} FINAL+ WHERE cohort_id = ${sql.string(cohortId)}+ AND project_id = ${sql.string(projectId)}+ )+ ${candidates}+ GROUP BY id+ ${having}+ )ORDER BY created_at DESCLIMIT ${sql.uint64(take)} OFFSET ${sql.uint64(offset)}`diff --git a/packages/core/src/modules/cohort/cohort.versions.test.ts b/packages/core/src/modules/cohort/cohort.versions.test.tsnew file mode 100644index 000000000..d7934c50b--- /dev/null+++ b/packages/core/src/modules/cohort/cohort.versions.test.ts@@ -0,0 +1,116 @@+// The cohort member page no longer reads `profiles FINAL`: search and+// filters decide on the newest version. Runs against profiles with several+// unmerged versions (test/profile-versions) enrolled in one cohort.++import { afterAll, beforeAll, describe, expect, it } from 'bun:test';+import { sql } from '@openpanel/db/src/clickhouse/sql';+import {+ otherProjectId,+ setupProfileVersions,+ teardownProfileVersions,+ VERSIONED,+} from '../../../../../test/profile-versions';+import { fetchRows } from '../../../test/clickhouse-rows';+import type { IChartEventFilter } from '../report/report.constants';++const PROJECT_ID = 'cohort-profile-versions';+const COHORT_ID = 'dddddddd-0000-4000-8000-00000000pv01';++let ch: typeof import('@openpanel/db/src/clickhouse/client').ch;+let deps: import('../../services').ServiceDeps;+let listCohortMemberProfiles: typeof import('./cohort.service').listCohortMemberProfiles;++beforeAll(async () => {+ ({ ch } = await import('@openpanel/db/src/clickhouse/client'));+ const { testServiceDeps } = await import('../../../test/service-deps');+ deps = await testServiceDeps();+ ({ listCohortMemberProfiles } = await import('./cohort.service'));+ const { bootstrapTestDatabases } = await import(+ '../../../../../test/bootstrap-databases'+ );+ await bootstrapTestDatabases();+ await setupProfileVersions(PROJECT_ID);+ await ch.insert({+ table: 'cohort_members',+ values: [VERSIONED.multi, VERSIONED.staleMatch, VERSIONED.single].map(+ (profileId) => ({+ project_id: PROJECT_ID,+ cohort_id: COHORT_ID,+ profile_id: profileId,+ matching_properties: {},+ version: 1,+ })+ ),+ format: 'JSONEachRow',+ });+ await ch.insert({+ table: 'cohort_members',+ values: [+ {+ project_id: otherProjectId(PROJECT_ID),+ cohort_id: COHORT_ID,+ profile_id: VERSIONED.multi,+ matching_properties: {},+ version: 1,+ },+ ],+ format: 'JSONEachRow',+ });+}, 30_000);++afterAll(async () => {+ await ch.command({+ query: `DELETE FROM cohort_members WHERE cohort_id = '${COHORT_ID}'`,+ });+ await teardownProfileVersions(PROJECT_ID);+});++const planIs = (value: string): IChartEventFilter => ({+ id: 'f',+ name: 'profile.properties.plan',+ operator: 'is',+ value: [value],+});++async function page(+ options: { search?: string; filters?: IChartEventFilter[] } = {}+) {+ const { data, count } = await listCohortMemberProfiles(deps, {+ projectId: PROJECT_ID,+ cohortId: COHORT_ID,+ take: 10,+ ...options,+ });+ return { ids: data.map((profile) => profile.id), count };+}++describe('listCohortMemberProfiles', () => {+ it('pages the members newest first with the total, like FINAL did', async () => {+ const reference = await fetchRows(+ ch,+ sql`SELECT id FROM profiles FINAL WHERE project_id = ${sql.string(PROJECT_ID)} AND id IN (SELECT profile_id FROM cohort_members FINAL WHERE cohort_id = ${sql.string(COHORT_ID)} AND project_id = ${sql.string(PROJECT_ID)}) ORDER BY created_at DESC LIMIT 10`+ );+ const { ids, count } = await page();+ expect(ids).toEqual([+ VERSIONED.single,+ VERSIONED.staleMatch,+ VERSIONED.multi,+ ]);+ expect([...ids].sort()).toEqual(+ reference.map((row) => JSON.parse(row).id).sort()+ );+ expect(count).toBe(3);+ });++ it('search and filters decide on the newest version', async () => {+ expect(await page({ filters: [planIs('pro')] })).toEqual({+ ids: [VERSIONED.single],+ count: 1,+ });+ expect(await page({ search: 'mia' })).toEqual({+ ids: [VERSIONED.multi],+ count: 1,+ });+ expect(await page({ search: 'Old' })).toEqual({ ids: [], count: 0 });+ });+});diff --git a/packages/core/src/modules/group/src/sql.test.ts b/packages/core/src/modules/group/src/sql.test.tsindex 61a7040f4..d6254c711 100644--- a/packages/core/src/modules/group/src/sql.test.ts+++ b/packages/core/src/modules/group/src/sql.test.ts@@ -167,18 +167,20 @@ describe('groupMemberProfilesQuery', () => {}).toStatement();const text = collapse(query);+ // Membership is decided on the newest version of `groups`; the raw-row+ // `has()` only narrows the candidate ids.expect(text).toContain(- 'count() OVER () AS total_count FROM profiles FINAL'- );- expect(text).toContain(- 'WHERE project_id = {p1:String} AND has(groups, {p2:String}) ORDER BY created_at DESC LIMIT {p3:UInt64} OFFSET {p4:UInt64}'+ 'count() OVER () AS total_count FROM ( SELECT id, argMax(created_at, last_seen_at) AS created_at FROM profiles WHERE project_id = {p1:String} AND cityHash64(id) IN (SELECT cityHash64(id) FROM profiles WHERE project_id = {p2:String} AND has(groups, {p3:String})) GROUP BY id HAVING has(argMax(groups, last_seen_at), {p4:String})) ORDER BY created_at DESC LIMIT {p5:UInt64} OFFSET {p6:UInt64}');+ expect(text).not.toContain('FINAL');expect(text).not.toContain('ILIKE');expect(query_params).toEqual({p1: PROJECT_ID,- p2: GROUP_ID,- p3: 50,- p4: 0,+ p2: PROJECT_ID,+ p3: GROUP_ID,+ p4: GROUP_ID,+ p5: 50,+ p6: 0,});});@@ -192,23 +194,21 @@ describe('groupMemberProfilesQuery', () => {}).toStatement();expect(collapse(query)).toContain(- 'AND (email ILIKE {p3:String} OR first_name ILIKE {p4:String} OR last_name ILIKE {p5:String})'+ 'AND (argMax(email, last_seen_at) ILIKE {p5:String} OR argMax(first_name, last_seen_at) ILIKE {p6:String} OR argMax(last_name, last_seen_at) ILIKE {p7:String})');expect(query).not.toContain(HOSTILE);- expect(query_params.p3).toBe(`%${HOSTILE}%`);expect(query_params.p5).toBe(`%${HOSTILE}%`);+ expect(query_params.p7).toBe(`%${HOSTILE}%`);});});describe('the trpc router queries', () => {const ref = { projectId: PROJECT_ID, groupId: HOSTILE };- test('every per-group query binds project then group and interpolates nothing', () => {+ test('every per-group event query binds project then group and interpolates nothing', () => {const builders = [groupEventMetricsQuery,- groupUniqueProfilesQuery,groupActivityQuery,- groupMemberGrowthQuery,groupMostEventsQuery,groupPopularRoutesQuery,];@@ -222,14 +222,31 @@ describe('the trpc router queries', () => {}});+ test('every per-group profile query decides membership on the newest version', () => {+ for (const build of [groupUniqueProfilesQuery, groupMemberGrowthQuery]) {+ const { query, query_params } = build(ref).toStatement();+ expect(collapse(query)).toContain(+ 'FROM profiles WHERE project_id = {p1:String} AND cityHash64(id) IN (SELECT cityHash64(id) FROM profiles WHERE project_id = {p2:String} AND has(groups, {p3:String})) GROUP BY id HAVING has(argMax(groups, last_seen_at), {p4:String})'+ );+ expect(query).not.toContain('FINAL');+ expect(query).not.toContain(HOSTILE);+ expect(query_params).toMatchObject({+ p1: PROJECT_ID,+ p2: PROJECT_ID,+ p3: HOSTILE,+ p4: HOSTILE,+ });+ }+ });+test('memberGrowth keeps the 30-day window and the WITH FILL frame', () => {const { query, query_params } = groupMemberGrowthQuery(ref).toStatement();const text = collapse(query);expect(text).toContain(- 'AND created_at >= now() - INTERVAL {p3:UInt64} DAY GROUP BY date ORDER BY date ASC WITH FILL FROM toDate(now() - INTERVAL {p4:UInt64} DAY) TO toDate(now() + INTERVAL 1 DAY) STEP 1'+ 'WHERE created_at >= now() - INTERVAL {p5:UInt64} DAY GROUP BY date ORDER BY date ASC WITH FILL FROM toDate(now() - INTERVAL {p6:UInt64} DAY) TO toDate(now() + INTERVAL 1 DAY) STEP 1');- expect(query_params).toMatchObject({ p3: 30, p4: 29 });+ expect(query_params).toMatchObject({ p5: 30, p6: 29 });});test('mostEvents excludes the lifecycle events; popularRoutes keeps only screen_view', () => {diff --git a/packages/core/src/modules/group/src/sql.ts b/packages/core/src/modules/group/src/sql.tsindex 2d33ceaff..19f9c19c5 100644--- a/packages/core/src/modules/group/src/sql.ts+++ b/packages/core/src/modules/group/src/sql.ts@@ -145,21 +145,45 @@ export interface GroupMemberProfilesQuery {search?: string;}+/**+ * The profiles whose newest version belongs to the group, with their first+ * seen. The raw-row `has()` is a superset (the surviving version is one of+ * the rows), so the GROUP BY only sees candidates, matched on a hash so the+ * set stays small; `having` adds conditions on the newest version.+ */+function groupMembers(+ projectId: string,+ groupId: string,+ having: SqlFragment[] = []+): SqlFragment {+ const project = sql`project_id = ${sql.string(projectId)}`;+ const group = sql.string(groupId);+ return sql`+ SELECT id, argMax(created_at, last_seen_at) AS created_at+ FROM ${sql.id(TABLE.profiles)}+ WHERE ${project}+ AND cityHash64(id) IN (SELECT cityHash64(id) FROM ${sql.id(TABLE.profiles)} WHERE ${project} AND has(groups, ${group}))+ GROUP BY id+ HAVING ${sql.join([sql`has(argMax(groups, last_seen_at), ${group})`, ...having], ' AND ')}`;+}+/** Member ids plus the window total, so one round trip pages and counts. */export function groupMemberProfilesQuery(query: GroupMemberProfilesQuery): SqlFragment {+ const search = query.search+ ? [+ (() => {+ const pattern = sql.string(`%${query.search}%`);+ return sql`(argMax(email, last_seen_at) ILIKE ${pattern} OR argMax(first_name, last_seen_at) ILIKE ${pattern} OR argMax(last_name, last_seen_at) ILIKE ${pattern})`;+ })(),+ ]+ : [];return sql`SELECTid,count() OVER () AS total_count- FROM ${sql.id(TABLE.profiles)} FINAL- WHERE project_id = ${sql.string(query.projectId)}- AND has(groups, ${sql.string(query.groupId)})- ${optional(query.search, () => {- const pattern = sql.string(`%${query.search}%`);- return sql`AND (email ILIKE ${pattern} OR first_name ILIKE ${pattern} OR last_name ILIKE ${pattern})`;- })}+ FROM (${groupMembers(query.projectId, query.groupId, search)})ORDER BY created_at DESCLIMIT ${sql.uint64(query.take)}OFFSET ${sql.uint64(query.offset)}@@ -187,9 +211,7 @@ export function groupUniqueProfilesQuery(query: {}): SqlFragment {return sql`SELECT count() AS uniqueProfiles- FROM ${sql.id(TABLE.profiles)} FINAL- WHERE project_id = ${sql.string(query.projectId)}- AND has(groups, ${sql.string(query.groupId)})+ FROM (${groupMembers(query.projectId, query.groupId)})`;}@@ -218,10 +240,8 @@ export function groupMemberGrowthQuery(query: {SELECTtoDate(toStartOfDay(created_at)) AS date,count() AS count- FROM ${sql.id(TABLE.profiles)} FINAL- WHERE project_id = ${sql.string(query.projectId)}- AND has(groups, ${sql.string(query.groupId)})- AND created_at >= now() - INTERVAL ${sql.uint64(MEMBER_GROWTH_WINDOW_DAYS)} DAY+ FROM (${groupMembers(query.projectId, query.groupId)})+ WHERE created_at >= now() - INTERVAL ${sql.uint64(MEMBER_GROWTH_WINDOW_DAYS)} DAYGROUP BY dateORDER BY date ASC WITH FILLFROM toDate(now() - INTERVAL ${sql.uint64(MEMBER_GROWTH_FILL_FROM_DAYS_AGO)} DAY)diff --git a/packages/core/src/modules/group/src/sql.versions.test.ts b/packages/core/src/modules/group/src/sql.versions.test.tsnew file mode 100644index 000000000..35306ffc2--- /dev/null+++ b/packages/core/src/modules/group/src/sql.versions.test.ts@@ -0,0 +1,91 @@+// The group member queries no longer read `profiles FINAL`: membership is+// decided on the newest version of `groups`. Runs against profiles with+// several unmerged versions (test/profile-versions).++import { afterAll, beforeAll, describe, expect, it } from 'bun:test';+import { sql } from '@openpanel/db/src/clickhouse/sql';+import {+ setupProfileVersions,+ teardownProfileVersions,+ VERSIONED,+} from '../../../../../../test/profile-versions';+import { expectSameRows, fetchRows } from '../../../../test/clickhouse-rows';+import {+ groupMemberGrowthQuery,+ groupMemberProfilesQuery,+ groupUniqueProfilesQuery,+} from './sql';++const PROJECT_ID = 'group-profile-versions';+const { g1, g2 } = VERSIONED.groups;++let ch: typeof import('@openpanel/db/src/clickhouse/client').ch;++beforeAll(async () => {+ ({ ch } = await import('@openpanel/db/src/clickhouse/client'));+ const { bootstrapTestDatabases } = await import(+ '../../../../../../test/bootstrap-databases'+ );+ await bootstrapTestDatabases();+ await setupProfileVersions(PROJECT_ID);+}, 30_000);++afterAll(() => teardownProfileVersions(PROJECT_ID));++const parsed = async (fragment: Parameters<typeof fetchRows>[1]) =>+ (await fetchRows(ch, fragment)).map((row) => JSON.parse(row));++describe('group members', () => {+ it('counts profiles whose newest version is in the group, like FINAL did', async () => {+ for (const [groupId, expected] of [+ [g1, 2],+ [g2, 2],+ ['pv-no-such-group', 0],+ ] as const) {+ const reference = sql`SELECT count() AS uniqueProfiles FROM profiles FINAL WHERE project_id = ${sql.string(PROJECT_ID)} AND has(groups, ${sql.string(groupId)})`;+ const [row] = await expectSameRows(+ ch,+ reference,+ groupUniqueProfilesQuery({ projectId: PROJECT_ID, groupId }),+ 1+ );+ expect(Number(JSON.parse(row as string).uniqueProfiles)).toBe(expected);+ }+ });++ it('pages the member ids with the total, and searches the newest version', async () => {+ const page = (search?: string) =>+ groupMemberProfilesQuery({+ projectId: PROJECT_ID,+ groupId: g1,+ take: 10,+ offset: 0,+ search,+ });+ const reference = sql`SELECT id, count() OVER () AS total_count FROM profiles FINAL WHERE project_id = ${sql.string(PROJECT_ID)} AND has(groups, ${sql.string(g1)}) ORDER BY created_at DESC LIMIT 10 OFFSET 0`;+ const members = await expectSameRows(ch, reference, page(), 2);+ expect(members.map((row) => JSON.parse(row).id).sort()).toEqual(+ [VERSIONED.multi, VERSIONED.staleMatch].sort()+ );+ expect(Number(JSON.parse(members[0] as string).total_count)).toBe(2);+ expect((await parsed(page('sam'))).map((row) => row.id)).toEqual([+ VERSIONED.staleMatch,+ ]);+ expect((await parsed(page('Old'))).map((row) => row.id)).toEqual([]);+ });++ it('member growth counts first-seen days of the newest version, like FINAL did', async () => {+ const reference = sql`SELECT toDate(toStartOfDay(created_at)) AS date, count() AS count FROM profiles FINAL WHERE project_id = ${sql.string(PROJECT_ID)} AND has(groups, ${sql.string(g2)}) AND created_at >= now() - INTERVAL 30 DAY GROUP BY date ORDER BY date ASC WITH FILL FROM toDate(now() - INTERVAL 29 DAY) TO toDate(now() + INTERVAL 1 DAY) STEP 1`;+ const rows = await expectSameRows(+ ch,+ reference,+ groupMemberGrowthQuery({ projectId: PROJECT_ID, groupId: g2 }),+ 30+ );+ const counted = rows+ .map((row) => JSON.parse(row))+ .filter((row) => Number(row.count) > 0);+ expect(counted).toHaveLength(1);+ expect(Number(counted[0].count)).toBe(1);+ });+});diff --git a/packages/core/src/modules/profile/profile.service.ts b/packages/core/src/modules/profile/profile.service.tsindex 3c48d3bfd..4773cb692 100644--- a/packages/core/src/modules/profile/profile.service.ts+++ b/packages/core/src/modules/profile/profile.service.ts@@ -36,7 +36,7 @@ import {profileValuesQuery,} from './src/sql';-export { profileSearchCondition } from './src/sql';+export { latestProfileColumn, profileSearchCondition } from './src/sql';const PROFILES_CACHE_SECONDS = 60 * 5;const PROPERTY_KEYS_CACHE_SECONDS = 60;@@ -166,12 +166,16 @@ export function transformProfile({async function compileProfileFilters(filters: IChartEventFilter[] | undefined,- projectId: string+ projectId: string,+ latestVersion = false): Promise<CompiledFilterClauses> {if (!filters?.length) {return {};}- return buildFilterWhere(filters, projectId, PROFILE_FILTER_TARGET);+ return buildFilterWhere(filters, projectId, {+ ...PROFILE_FILTER_TARGET,+ latestVersion,+ });}/**@@ -287,6 +291,11 @@ export async function getProfileList(startDate,endDate,filterClauses: await compileProfileFilters(filters, projectId),+ latestFilterClauses: await compileProfileFilters(+ filters,+ projectId,+ true+ ),}));return data.map(transformProfile);@@ -312,6 +321,11 @@ export async function getProfileListCount(startDate,endDate,filterClauses: await compileProfileFilters(filters, projectId),+ latestFilterClauses: await compileProfileFilters(+ filters,+ projectId,+ true+ ),}));return data[0]?.count ?? 0;@@ -390,6 +404,11 @@ export async function findProfilesCore(input.filters,input.projectId),+ latestFilterClauses: await compileProfileFilters(+ input.filters,+ input.projectId,+ true+ ),sortOrder: input.sortOrder === 'asc' ? 'asc' : 'desc',limit: Math.min(input.limit ?? FIND_PROFILES_DEFAULT_LIMIT,diff --git a/packages/core/src/modules/profile/src/sql.test.ts b/packages/core/src/modules/profile/src/sql.test.tsindex d03363e5a..c74addda3 100644--- a/packages/core/src/modules/profile/src/sql.test.ts+++ b/packages/core/src/modules/profile/src/sql.test.ts@@ -28,12 +28,24 @@ const PROJECT_ID = 'proj-1';const PROFILE_ID = 'prof-1';const HOSTILE = "x' OR 1=1 --";const NO_FILTERS = {};+const NO_FILTER_CLAUSES = {+ filterClauses: NO_FILTERS,+ latestFilterClauses: NO_FILTERS,+};+/** The outer statement of a page: every column from the newest version of the ids inside. */+const LATEST_COLUMNS_PAGE =+ 'SELECT id, argMax(first_name, p.last_seen_at) AS first_name, argMax(last_name, p.last_seen_at) AS last_name, argMax(email, p.last_seen_at) AS email, argMax(avatar, p.last_seen_at) AS avatar, argMax(properties, p.last_seen_at) AS properties, project_id, argMax(is_external, p.last_seen_at) AS is_external, argMax(created_at, p.last_seen_at) AS created_at, max(p.last_seen_at) AS last_seen_at, argMax(groups, p.last_seen_at) AS groups FROM profiles AS p';+/** The inner id query of a page or count up to `end`, with its placeholders unnumbered. */+function innerIdsOf(text: string, end: number): string {+ return text+ .slice(text.indexOf('SELECT id FROM profiles WHERE'), end)+ .replace(/\{p\d+:/g, '{p:');+}const WINDOW = {startDate: '2026-06-16 00:00:00',endDate: '2026-09-16 23:59:59',};const COLUMN_LIST = PROFILE_COLUMNS.join(', ');-const PLAIN_WHERE = /\sWHERE\s/;function collapse(text: string): string {return text.replace(/\s+/g, ' ').trim();@@ -141,24 +153,26 @@ describe('single-profile lookups', () => {});describe('profileListQuery', () => {- test('plain page: project + window, no OFFSET at offset 0', () => {+ test('plain page: newest version of the ids in the window, no OFFSET at offset 0', () => {const { query, query_params } = profileListQuery({projectId: PROJECT_ID,take: 50,offset: 0,- filterClauses: NO_FILTERS,+ ...NO_FILTER_CLAUSES,...WINDOW,}).toStatement();const text = collapse(query);expect(text).toBe(- 'SELECT * FROM profiles FINAL WHERE project_id = {p1:String} AND created_at BETWEEN toDateTime({p2:String}) AND toDateTime({p3:String}) ORDER BY created_at DESC LIMIT {p4:UInt64}'+ `${LATEST_COLUMNS_PAGE} WHERE p.project_id = {p1:String} AND p.id IN (SELECT id FROM profiles WHERE project_id = {p2:String} AND created_at BETWEEN toDateTime({p3:String}) AND toDateTime({p4:String}) GROUP BY id ORDER BY argMax(created_at, last_seen_at) DESC LIMIT {p5:UInt64}) GROUP BY p.id, p.project_id ORDER BY created_at DESC`);+ expect(text).not.toContain('FINAL');expect(query_params).toEqual({p1: PROJECT_ID,- p2: '2026-06-16 00:00:00',- p3: '2026-09-16 23:59:59',- p4: 50,+ p2: PROJECT_ID,+ p3: '2026-06-16 00:00:00',+ p4: '2026-09-16 23:59:59',+ p5: 50,});});@@ -170,52 +184,57 @@ describe('profileListQuery', () => {search: 'ann',isExternal: true,filterClauses: { f0: sql`(properties['plan'] = 'pro')` },+ latestFilterClauses: {+ f0: sql`(argMax(properties['plan'], last_seen_at) = 'pro')`,+ },...WINDOW,}).toStatement();const text = collapse(query);+ // The raw-row conditions only narrow the id set (a superset); the same+ // conditions on the newest version decide, in HAVING.expect(text).toContain(- "WHERE project_id = {p1:String} AND created_at BETWEEN toDateTime({p2:String}) AND toDateTime({p3:String}) AND ((id ILIKE {p4:String} OR email ILIKE {p5:String} OR first_name ILIKE {p6:String} OR last_name ILIKE {p7:String} OR concat(first_name, ' ', last_name) ILIKE {p8:String})) AND is_external = {p9:Bool} AND (properties['plan'] = 'pro') ORDER BY created_at DESC LIMIT {p10:UInt64} OFFSET {p11:UInt64}"+ "(SELECT id FROM profiles WHERE project_id = {p2:String} AND created_at BETWEEN toDateTime({p3:String}) AND toDateTime({p4:String}) AND cityHash64(id) IN (SELECT cityHash64(id) FROM profiles WHERE project_id = {p5:String} AND created_at BETWEEN toDateTime({p6:String}) AND toDateTime({p7:String}) AND ((id ILIKE {p8:String} OR email ILIKE {p9:String} OR first_name ILIKE {p10:String} OR last_name ILIKE {p11:String} OR concat(first_name, ' ', last_name) ILIKE {p12:String})) AND is_external = {p13:Bool} AND (properties['plan'] = 'pro')) GROUP BY id HAVING ((id ILIKE {p14:String} OR argMax(email, last_seen_at) ILIKE {p15:String} OR argMax(first_name, last_seen_at) ILIKE {p16:String} OR argMax(last_name, last_seen_at) ILIKE {p17:String} OR concat(argMax(first_name, last_seen_at), ' ', argMax(last_name, last_seen_at)) ILIKE {p18:String})) AND argMax(is_external, last_seen_at) = {p19:Bool} AND (argMax(properties['plan'], last_seen_at) = 'pro') ORDER BY argMax(created_at, last_seen_at) DESC LIMIT {p20:UInt64} OFFSET {p21:UInt64})");expect(query_params).toMatchObject({p1: PROJECT_ID,- p4: '%ann%',- p9: true,- p10: 50,- p11: 100,+ p8: '%ann%',+ p13: true,+ p14: '%ann%',+ p19: true,+ p20: 50,+ p21: 100,});});- test('count shares the exact WHERE of the page (minus FINAL)', () => {+ test('count counts exactly the ids the page selects from', () => {const list = {projectId: PROJECT_ID,search: 'ann',isExternal: false,filterClauses: { f0: sql`(email != '')` },+ latestFilterClauses: { f0: sql`(argMax(email, last_seen_at) != '')` },...WINDOW,};const page = collapse(profileListQuery({ ...list, take: 1, offset: 0 }).toStatement().query);const count = collapse(profileListCountQuery(list).toStatement().query);- const whereOf = (text: string) =>- text- .slice(text.indexOf('WHERE'))- .replace(/ ORDER BY.*$/, '')- .replace(/ GROUP BY.*$/, '');expect(count).toStartWith(- 'SELECT uniqExact(id) as count FROM profiles WHERE'+ 'SELECT count() as count FROM (SELECT id FROM profiles WHERE'+ );+ expect(count).not.toContain('FINAL');+ expect(innerIdsOf(count, count.lastIndexOf(')'))).toBe(+ innerIdsOf(page, page.indexOf(' ORDER BY')));- expect(count).toEndWith('GROUP BY project_id');- expect(whereOf(count)).toBe(whereOf(page));});});describe('findProfilesQuery', () => {const base = {projectId: PROJECT_ID,- filterClauses: NO_FILTERS,+ ...NO_FILTER_CLAUSES,sortOrder: 'desc' as const,limit: 25,};@@ -224,9 +243,9 @@ describe('findProfilesQuery', () => {const { query, query_params } = findProfilesQuery(base).toStatement();expect(collapse(query)).toBe(- `SELECT ${COLUMN_LIST} FROM profiles FINAL WHERE project_id = {p1:String} ORDER BY created_at DESC LIMIT {p2:UInt64}`+ `${LATEST_COLUMNS_PAGE} WHERE p.project_id = {p1:String} AND p.id IN (SELECT id FROM profiles WHERE project_id = {p2:String} GROUP BY id ORDER BY argMax(created_at, last_seen_at) DESC LIMIT {p3:UInt64}) GROUP BY p.id, p.project_id ORDER BY created_at DESC`);- expect(query_params).toEqual({ p1: PROJECT_ID, p2: 25 });+ expect(query_params).toEqual({ p1: PROJECT_ID, p2: PROJECT_ID, p3: 25 });});test('every optional condition binds and none interpolates', () => {@@ -243,36 +262,45 @@ describe('findProfilesQuery', () => {minSessions: 3,performedEvent: HOSTILE,filterClauses: { f0: sql`(1 = 1)` },+ latestFilterClauses: { f0: sql`(2 = 2)` },}).toStatement();const text = collapse(query);- expect(text).toContain('AND email ILIKE {p2:String}');+ // The id-level conditions filter rows; the column conditions narrow the+ // candidate ids on the raw rows and decide on the newest version in HAVING.+ expect(text).toContain(+ "AND id NOT IN ( SELECT DISTINCT profile_id FROM events WHERE project_id = {p3:String} AND profile_id != '' AND created_at >= now() - INTERVAL {p4:UInt64} DAY )"+ );+ expect(text).toContain(+ "AND id IN ( SELECT profile_id FROM sessions WHERE project_id = {p5:String} AND sign = 1 AND profile_id != '' GROUP BY profile_id HAVING count() >= {p6:UInt64} )"+ );expect(text).toContain(- 'AND properties[{p8:String}] = {p9:String} AND properties[{p10:String}] = {p11:String} AND properties[{p12:String}] = {p13:String} AND properties[{p14:String}] = {p15:String}'+ 'AND id IN ( SELECT DISTINCT profile_id FROM events WHERE project_id = {p7:String} AND name = {p8:String} )');expect(text).toContain(- "AND id NOT IN ( SELECT DISTINCT profile_id FROM events WHERE project_id = {p16:String} AND profile_id != '' AND created_at >= now() - INTERVAL {p17:UInt64} DAY )"+ 'AND cityHash64(id) IN (SELECT cityHash64(id) FROM profiles WHERE project_id = {p9:String} AND email ILIKE {p10:String} AND ((id ILIKE {p11:String}');expect(text).toContain(- "AND id IN ( SELECT profile_id FROM sessions WHERE project_id = {p18:String} AND sign = 1 AND profile_id != '' GROUP BY profile_id HAVING count() >= {p19:UInt64} )"+ 'AND properties[{p16:String}] = {p17:String} AND properties[{p18:String}] = {p19:String} AND properties[{p20:String}] = {p21:String} AND properties[{p22:String}] = {p23:String} AND (1 = 1)) GROUP BY id HAVING argMax(email, last_seen_at) ILIKE {p24:String}');expect(text).toContain(- 'AND id IN ( SELECT DISTINCT profile_id FROM events WHERE project_id = {p20:String} AND name = {p21:String} )'+ 'AND argMax(properties, last_seen_at)[{p30:String}] = {p31:String}');expect(text).toEndWith(- 'AND (1 = 1) ORDER BY created_at ASC LIMIT {p22:UInt64}'+ 'AND (2 = 2) ORDER BY argMax(created_at, last_seen_at) ASC LIMIT {p38:UInt64}) GROUP BY p.id, p.project_id ORDER BY created_at ASC');expect(query).not.toContain(HOSTILE);expect(query_params).toMatchObject({- p2: `%${HOSTILE}%`,- p8: 'country',- p9: 'SE',- p10: 'city',- p11: HOSTILE,- p17: 7,- p19: 3,- p21: HOSTILE,- p22: 25,+ p4: 7,+ p6: 3,+ p8: HOSTILE,+ p10: `%${HOSTILE}%`,+ p16: 'country',+ p17: 'SE',+ p18: 'city',+ p19: HOSTILE,+ p24: `%${HOSTILE}%`,+ p38: 25,});});@@ -285,8 +313,8 @@ describe('findProfilesQuery', () => {}).toStatement().query);expect(zero).toContain('NOT IN');- expect(zero).toContain('INTERVAL {p3:UInt64} DAY');- expect(zero).toContain('HAVING count() >= {p5:UInt64}');+ expect(zero).toContain('INTERVAL {p4:UInt64} DAY');+ expect(zero).toContain('HAVING count() >= {p6:UInt64}');});});diff --git a/packages/core/src/modules/profile/src/sql.ts b/packages/core/src/modules/profile/src/sql.tsindex fd14d79ac..9b4cffa8a 100644--- a/packages/core/src/modules/profile/src/sql.ts+++ b/packages/core/src/modules/profile/src/sql.ts@@ -7,7 +7,6 @@ import { toRangeBoundaryLiteral } from '../../../shared/ch-dates';import {type CompiledFilterClauses,compiledFilterFragments,- spliceCompiledFilters,} from './filter-clauses';const TABLE = {@@ -37,6 +36,19 @@ const PROFILE_COLUMNS_FRAGMENT = sql.join(const PROFILE_KEY_COLUMNS = ['id', 'project_id'] as const;+type ProfileColumn = (typeof PROFILE_COLUMNS)[number];++/** A column on the raw rows, or its newest version per id (for a `GROUP BY id`). */+export type ProfileColumnAccess = (column: ProfileColumn) => SqlFragment;++export const profileColumn: ProfileColumnAccess = (column) =>+ sql.id(column, PROFILE_COLUMNS);++export const latestProfileColumn: ProfileColumnAccess = (column) =>+ column === 'id'+ ? sql.id(column, PROFILE_COLUMNS)+ : sql`argMax(${sql.id(column, PROFILE_COLUMNS)}, last_seen_at)`;+/*** The newest version of every column, as `profiles FINAL` would return it,* for a `GROUP BY id, project_id` over the raw rows. `last_seen_at` is the@@ -86,7 +98,8 @@ export interface ProfileWindow {* when the search is blank.*/export function profileSearchCondition(- search: string | null | undefined+ search: string | null | undefined,+ column: ProfileColumnAccess = profileColumn): SqlFragment | null {const tokens = (search ?? '').trim()@@ -98,7 +111,7 @@ export function profileSearchCondition(}const perToken = tokens.map((token) => {const like = sql.string(`%${token}%`);- return sql`(id ILIKE ${like} OR email ILIKE ${like} OR first_name ILIKE ${like} OR last_name ILIKE ${like} OR concat(first_name, ' ', last_name) ILIKE ${like})`;+ return sql`(${column('id')} ILIKE ${like} OR ${column('email')} ILIKE ${like} OR ${column('first_name')} ILIKE ${like} OR ${column('last_name')} ILIKE ${like} OR concat(${column('first_name')}, ' ', ${column('last_name')}) ILIKE ${like})`;});return sql`(${sql.join(perToken, ' AND ')})`;}@@ -197,52 +210,147 @@ export interface ProfileListQuery extends ProfileWindow {projectId: string;search?: string;isExternal?: boolean;- /** Pre-compiled by `buildFilterWhere` (src/filter-clauses.ts). */+ /** Pre-compiled by `buildFilterWhere` on the raw rows (src/filter-clauses.ts). */filterClauses: CompiledFilterClauses;+ /** The same filters compiled on the newest version (`latestVersion: true`). */+ latestFilterClauses: CompiledFilterClauses;+}++/**+ * Ids whose newest version satisfies `having`, ranked by first seen.+ *+ * `scope` (project, first-seen window) and `where` (id-level subqueries) are+ * the same for every version of a profile, so they may filter rows.+ * `candidates` holds the column conditions on the raw rows: they only narrow+ * the id set, through a subquery, and never the rows the aggregate reads —+ * otherwise argMax would see nothing but the matching stale version and the+ * HAVING clause would pass on it. The set is a superset of the answer, since+ * the surviving version is one of the rows, and holds hashes: a String set+ * costs ~300 bytes per id, and a collision only adds a candidate.+ */+interface LatestProfileIdsPage {+ scope: SqlFragment;+ where?: SqlFragment[];+ candidates: SqlFragment[];+ having: SqlFragment[];+}++function latestProfileIdsWhere(page: LatestProfileIdsPage): SqlFragment {+ const conditions = [page.scope, ...(page.where ?? [])];+ if (page.candidates.length) {+ conditions.push(+ sql`cityHash64(id) IN (SELECT cityHash64(id) FROM ${sql.id(TABLE.profiles)} WHERE ${page.scope} AND ${sql.join(page.candidates, ' AND ')})`+ );+ }+ const having = page.having.length+ ? sql` HAVING ${sql.join(page.having, ' AND ')}`+ : sql.empty;+ return sql`WHERE ${sql.join(conditions, ' AND ')} GROUP BY id${having}`;+}++function latestProfileIdsQuery(+ page: LatestProfileIdsPage & {+ orderDirection: SqlFragment;+ limit: number;+ offset?: number;+ }+): SqlFragment {+ return sql`SELECT id FROM ${sql.id(TABLE.profiles)} ${latestProfileIdsWhere(page)} ORDER BY ${latestProfileColumn('created_at')} ${page.orderDirection} LIMIT ${sql.uint64(page.limit)}${optional(+ page.offset,+ () => sql` OFFSET ${sql.uint64(page.offset as number)}`+ )}`;+}++/** The page's rows, newest version of every column, in the ids' order. */+function latestProfilesPage(+ projectId: string,+ ids: SqlFragment,+ orderDirection: SqlFragment+): SqlFragment {+ return sql`+ SELECT ${PROFILE_LATEST_COLUMNS_FRAGMENT}+ FROM ${sql.id(TABLE.profiles)} AS p+ WHERE p.project_id = ${sql.string(projectId)} AND p.id IN (${ids})+ GROUP BY p.id, p.project_id+ ORDER BY created_at ${orderDirection}+ `;}/**- * `WHERE project_id =.. [AND search] [AND is_external =..] [AND filters]`,- * bounded to a window rather than reading every profile the project ever- * had — otherwise the cost grows with the tenant's lifetime rather than with- * anything the caller asked for. `created_at` is the column the list already- * orders by, so the bound and the order agree.+ * `project_id =.. AND created_at BETWEEN ..`: bounded to a window rather than+ * reading every profile the project ever had — otherwise the cost grows with+ * the tenant's lifetime rather than with anything the caller asked for.+ * `created_at` is the column the list already orders by, so the bound and+ * the order agree; the write path keeps it the same across versions.*/-function profileListCondition(query: ProfileListQuery): SqlFragment {- const search = profileSearchCondition(query.search);+function profileListScope(query: ProfileListQuery): SqlFragment {return sql`project_id = ${sql.string(query.projectId)} AND ${dateRangeWhere('created_at',query.startDate,query.endDate- )}${optional(search, () => sql` AND ${search as SqlFragment}`)}${optional(- query.isExternal !== undefined,- () => sql` AND is_external = ${sql.bool(query.isExternal as boolean)}`- )} ${spliceCompiledFilters(query.filterClauses)}`;+ )}`;+}++/**+ * `[search] [is_external =..] [filters]` as conditions on one version:+ * the raw rows (candidates) or the newest one per id (the answer).+ */+function profileListConditions(+ query: ProfileListQuery,+ column: ProfileColumnAccess,+ filterClauses: CompiledFilterClauses+): SqlFragment[] {+ const conditions: SqlFragment[] = [];+ const search = profileSearchCondition(query.search, column);+ if (search) {+ conditions.push(search);+ }+ if (query.isExternal !== undefined) {+ conditions.push(+ sql`${column('is_external')} = ${sql.bool(query.isExternal)}`+ );+ }+ conditions.push(...compiledFilterFragments(filterClauses));+ return conditions;+}++function profileListPage(query: ProfileListQuery): LatestProfileIdsPage {+ return {+ scope: profileListScope(query),+ candidates: profileListConditions(+ query,+ profileColumn,+ query.filterClauses+ ),+ having: profileListConditions(+ query,+ latestProfileColumn,+ query.latestFilterClauses+ ),+ };}export function profileListQuery(query: ProfileListQuery & { take: number; offset: number }): SqlFragment {- return sql`- SELECT *- FROM ${sql.id(TABLE.profiles)} FINAL- WHERE ${profileListCondition(query)}- ORDER BY created_at DESC- LIMIT ${sql.uint64(query.take)}- ${optional(query.offset, () => sql`OFFSET ${sql.uint64(query.offset)}`)}- `;+ const orderDirection = sql`DESC`;+ return latestProfilesPage(+ query.projectId,+ latestProfileIdsQuery({+ ...profileListPage(query),+ orderDirection,+ limit: query.take,+ offset: query.offset,+ }),+ orderDirection+ );}-// `uniqExact`, not `count`: the page reads `profiles FINAL` so it sees one row-// per id, while the count deliberately skips FINAL (it is the expensive half-// on a large tenant). Counting rows therefore counted every unmerged-// ReplacingMergeTree version and the total ran ahead of the list.+/** One per profile whose newest version matches: exactly what the page shows. */export function profileListCountQuery(query: ProfileListQuery): SqlFragment {return sql`- SELECT uniqExact(id) as count- FROM ${sql.id(TABLE.profiles)}- WHERE ${profileListCondition(query)}- GROUP BY project_id+ SELECT count() as count+ FROM (SELECT id FROM ${sql.id(TABLE.profiles)} ${latestProfileIdsWhere(profileListPage(query))})`;}@@ -260,6 +368,7 @@ export interface FindProfilesQuery {minSessions?: number;performedEvent?: string;filterClauses: CompiledFilterClauses;+ latestFilterClauses: CompiledFilterClauses;sortOrder: 'asc' | 'desc';limit: number;}@@ -271,28 +380,52 @@ const PROFILE_PROPERTY_FILTERS = ['browser',] as const;-function findProfilesConditions(query: FindProfilesQuery): SqlFragment[] {+/**+ * `where` holds the id-level conditions, the same for every version;+ * `candidates` every column condition on the raw rows and `having` the same+ * on the newest version (see latestProfileIdsQuery).+ */+function findProfilesConditions(query: FindProfilesQuery): {+ where: SqlFragment[];+ candidates: SqlFragment[];+ having: SqlFragment[];+} {const projectId = sql.string(query.projectId);- const conditions: SqlFragment[] = [sql`project_id = ${projectId}`];+ const where: SqlFragment[] = [];+ const candidates: SqlFragment[] = [];+ const having: SqlFragment[] = [];+ const onBothVersions = (+ condition: (column: ProfileColumnAccess) => SqlFragment | null+ ) => {+ const anyVersion = condition(profileColumn);+ const latestVersion = condition(latestProfileColumn);+ if (anyVersion && latestVersion) {+ candidates.push(anyVersion);+ having.push(latestVersion);+ }+ };if (query.email) {- conditions.push(sql`email ILIKE ${sql.string(`%${query.email}%`)}`);+ onBothVersions(+ (column) =>+ sql`${column('email')} ILIKE ${sql.string(`%${query.email}%`)}`+ );}- const nameCondition = profileSearchCondition(query.name);- if (query.name && nameCondition) {- conditions.push(nameCondition);+ if (query.name) {+ onBothVersions((column) => profileSearchCondition(query.name, column));}for (const key of PROFILE_PROPERTY_FILTERS) {const value = query[key];if (value) {- conditions.push(- sql`properties[${sql.string(key)}] = ${sql.string(value)}`+ onBothVersions(+ (column) =>+ sql`${column('properties')}[${sql.string(key)}] = ${sql.string(value)}`);}}if (query.inactiveDays !== undefined) {- conditions.push(sql`id NOT IN (+ where.push(sql`id NOT IN (SELECT DISTINCT profile_id FROM ${sql.id(TABLE.events)}WHERE project_id = ${projectId}AND profile_id != ''@@ -301,7 +434,7 @@ function findProfilesConditions(query: FindProfilesQuery): SqlFragment[] {}if (query.minSessions !== undefined) {- conditions.push(sql`id IN (+ where.push(sql`id IN (SELECT profile_id FROM ${sql.id(TABLE.sessions)}WHERE project_id = ${projectId}AND sign = 1@@ -312,27 +445,34 @@ function findProfilesConditions(query: FindProfilesQuery): SqlFragment[] {}if (query.performedEvent) {- conditions.push(sql`id IN (+ where.push(sql`id IN (SELECT DISTINCT profile_id FROM ${sql.id(TABLE.events)}WHERE project_id = ${projectId}AND name = ${sql.string(query.performedEvent)})`);}- conditions.push(...compiledFilterFragments(query.filterClauses));+ candidates.push(...compiledFilterFragments(query.filterClauses));+ having.push(...compiledFilterFragments(query.latestFilterClauses));- return conditions;+ return { where, candidates, having };}export function findProfilesQuery(query: FindProfilesQuery): SqlFragment {const orderDirection = query.sortOrder === 'asc' ? sql`ASC` : sql`DESC`;- return sql`- SELECT ${PROFILE_COLUMNS_FRAGMENT}- FROM ${sql.id(TABLE.profiles)} FINAL- WHERE ${sql.join(findProfilesConditions(query), ' AND ')}- ORDER BY created_at ${orderDirection}- LIMIT ${sql.uint64(query.limit)}- `;+ const { where, candidates, having } = findProfilesConditions(query);+ return latestProfilesPage(+ query.projectId,+ latestProfileIdsQuery({+ scope: sql`project_id = ${sql.string(query.projectId)}`,+ where,+ candidates,+ having,+ orderDirection,+ limit: query.limit,+ }),+ orderDirection+ );}export function profileRowQuery(query: {diff --git a/packages/core/src/modules/profile/src/sql.versions.test.ts b/packages/core/src/modules/profile/src/sql.versions.test.tsindex 3e7c49b42..916eeb4f1 100644--- a/packages/core/src/modules/profile/src/sql.versions.test.ts+++ b/packages/core/src/modules/profile/src/sql.versions.test.ts@@ -5,6 +5,7 @@import { afterAll, beforeAll, describe, expect, it } from 'bun:test';import { sql } from '@openpanel/db/src/clickhouse/sql';+import { timeAgo } from '../../../../../../test/fixtures';import {otherProjectId,setupProfileVersions,@@ -12,9 +13,14 @@ import {VERSIONED,} from '../../../../../../test/profile-versions';import { expectSameRows, fetchRows } from '../../../../test/clickhouse-rows';+import { buildFilterWhere } from '../../chart/src/table-filter-where';+import type { IChartEventFilter } from '../../report/report.constants';import {+ findProfilesQuery,PROFILE_COLUMNS,profileByIdQuery,+ profileListCountQuery,+ profileListQuery,profileMetricsQuery,profileRowQuery,profilesByIdsQuery,@@ -27,6 +33,7 @@ const columns = sql.join();let ch: typeof import('@openpanel/db/src/clickhouse/client').ch;+let now: Date;beforeAll(async () => {({ ch } = await import('@openpanel/db/src/clickhouse/client'));@@ -34,7 +41,7 @@ beforeAll(async () => {'../../../../../../test/bootstrap-databases');await bootstrapTestDatabases();- await setupProfileVersions(PROJECT_ID);+ now = await setupProfileVersions(PROJECT_ID);}, 30_000);afterAll(() => teardownProfileVersions(PROJECT_ID));@@ -141,3 +148,128 @@ describe('profileMetricsQuery', () => {expect(JSON.parse(metrics as string)).toMatchObject(expected);});});++const PROFILE_TARGET = {+ selfTable: 'profiles',+ profileIdExpr: 'id',+ groupsExpr: 'groups',+} as const;+const planIs = (value: string): IChartEventFilter => ({+ id: 'f',+ name: 'profile.properties.plan',+ operator: 'is',+ value: [value],+});+function compiled(filters: IChartEventFilter[]) {+ return {+ filterClauses: buildFilterWhere(filters, PROJECT_ID, PROFILE_TARGET),+ latestFilterClauses: buildFilterWhere(filters, PROJECT_ID, {+ ...PROFILE_TARGET,+ latestVersion: true,+ }),+ };+}+const ids = (rows: string[]) => rows.map((row) => JSON.parse(row).id);++describe('profileListQuery / profileListCountQuery', () => {+ const window = () => ({+ startDate: timeAgo(now, 60),+ endDate: timeAgo(now, 0),+ });+ const list = (+ overrides: Partial<Parameters<typeof profileListQuery>[0]> = {},+ filters: IChartEventFilter[] = []+ ) =>+ profileListQuery({+ projectId: PROJECT_ID,+ take: 50,+ offset: 0,+ ...window(),+ ...compiled(filters),+ ...overrides,+ });+ const count = (+ overrides: Partial<Parameters<typeof profileListCountQuery>[0]> = {},+ filters: IChartEventFilter[] = []+ ) =>+ profileListCountQuery({+ projectId: PROJECT_ID,+ ...window(),+ ...compiled(filters),+ ...overrides,+ });+ const countOf = async (fragment: ReturnType<typeof count>) =>+ Number(JSON.parse((await fetchRows(ch, fragment))[0] as string).count);++ it('pages the newest version of every profile in the window, like FINAL did', async () => {+ const reference = sql`SELECT * FROM profiles FINAL WHERE project_id = ${sql.string(PROJECT_ID)} AND created_at BETWEEN toDateTime(${sql.string(timeAgo(now, 60))}) AND toDateTime(${sql.string(timeAgo(now, 0))}) ORDER BY created_at DESC LIMIT 50`;+ const rows = await expectSameRows(ch, reference, list(), 3);+ expect(ids(rows).sort()).toEqual(+ [VERSIONED.multi, VERSIONED.staleMatch, VERSIONED.single].sort()+ );+ expect(await countOf(count())).toBe(3);+ });++ it('keeps the newest-first order and the offset', async () => {+ const page = (await fetchRows(ch, list({ take: 2, offset: 1 }))).map(+ (row) => JSON.parse(row)+ );+ const ordered = page.sort((a, b) => (a.created_at < b.created_at ? 1 : -1));+ expect(ordered.map((row) => row.id)).toEqual([+ VERSIONED.staleMatch,+ VERSIONED.multi,+ ]);+ });++ it('filters, searches and is_external read the newest version', async () => {+ expect(ids(await fetchRows(ch, list({}, [planIs('pro')])))).toEqual([+ VERSIONED.single,+ ]);+ expect(await countOf(count({}, [planIs('pro')]))).toBe(1);+ expect(ids(await fetchRows(ch, list({ search: 'mia' })))).toEqual([+ VERSIONED.multi,+ ]);+ expect(ids(await fetchRows(ch, list({ search: 'Old' })))).toEqual([]);+ expect(await countOf(count({ isExternal: true }))).toBe(3);+ expect(await countOf(count({ isExternal: false }))).toBe(0);+ });+});++describe('findProfilesQuery', () => {+ const find = (+ overrides: Partial<Parameters<typeof findProfilesQuery>[0]> = {},+ filters: IChartEventFilter[] = []+ ) =>+ findProfilesQuery({+ projectId: PROJECT_ID,+ sortOrder: 'desc',+ limit: 20,+ ...compiled(filters),+ ...overrides,+ });++ it('matches the FINAL form it replaced, row for row', async () => {+ const reference = sql`SELECT ${columns} FROM profiles FINAL WHERE project_id = ${sql.string(PROJECT_ID)} ORDER BY created_at DESC LIMIT 20`;+ await expectSameRows(ch, reference, find(), 3);+ });++ it('every column condition reads the newest version', async () => {+ expect(ids(await fetchRows(ch, find({ email: 'mia' })))).toEqual([+ VERSIONED.multi,+ ]);+ expect(ids(await fetchRows(ch, find({ email: 'old.example' })))).toEqual(+ []+ );+ expect(ids(await fetchRows(ch, find({ country: 'US' })))).toEqual([+ VERSIONED.staleMatch,+ ]);+ expect(ids(await fetchRows(ch, find({ name: 'Kim-Lee' })))).toEqual([+ VERSIONED.multi,+ ]);+ expect(+ ids(+ await fetchRows(ch, find({ performedEvent: 'signup' }, [planIs('pro')]))+ )+ ).toEqual([VERSIONED.single]);+ });+});