OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.
ralph(M31-003): Group C fix 8: give powerUsers, profile.list and retention an explicit window
ralphac9cf1d21a23
diff --git a/packages/core/src/modules/profile/profile.rpc.ts b/packages/core/src/modules/profile/profile.rpc.tsindex 733a0679b..45725e354 100644--- a/packages/core/src/modules/profile/profile.rpc.ts+++ b/packages/core/src/modules/profile/profile.rpc.ts@@ -9,7 +9,9 @@import { z } from 'zod';import { createTRPCRouter, protectedProcedure } from '../../rpc/base';-import { zChartEventFilter } from '../report/report.constants';+import { getSettingsForProject } from '../organization/organization.service';+import { zChartEventFilter, zRange } from '../report/report.constants';+import { getChartStartEndDate } from '../report/src/chart-dates';import {getPowerUsers,getProfileActivity,@@ -24,6 +26,22 @@ import {const DEFAULT_LIST_TAKE = 50;+/**+ * `list` and `powerUsers` had no date filter at all, so their cost grew with+ * the tenant's lifetime rather than with anything the caller chose+ * (`docs/ANALYTICS_PERFORMANCE.md` §6.8: 44.2 M rows read for one page of 50).+ * Both now take the report vocabulary's window; this is the default when the+ * caller names none, and it is the range §6.8 itself uses to describe the+ * change ("a power user over 90 days is not the same set as over all time").+ */+const PROFILE_WINDOW_DEFAULT_RANGE = '3m';++const zProfileWindow = {+ range: zRange.default(PROFILE_WINDOW_DEFAULT_RANGE),+ startDate: z.string().nullish(),+ endDate: z.string().nullish(),+};+const zProfileRef = z.object({ profileId: z.string(), projectId: z.string() });export const profileRouter = createTRPCRouter({@@ -106,6 +124,7 @@ export const profileRouter = createTRPCRouter({search: z.string().optional(),isExternal: z.boolean().optional(),filters: z.array(zChartEventFilter).default([]),+ ...zProfileWindow,})).query(async ({ input, ctx }) => {@@ -115,7 +134,11 @@ export const profileRouter = createTRPCRouter({level: 'read',});- return getProfileListPage(ctx, input);+ const { timezone } = await getSettingsForProject(ctx, input.projectId);+ return getProfileListPage(ctx, {+ ...input,+ ...getChartStartEndDate(input, timezone),+ });}),powerUsers: protectedProcedure@@ -124,6 +147,7 @@ export const profileRouter = createTRPCRouter({projectId: z.string(),cursor: z.number().optional(),take: z.number().default(DEFAULT_LIST_TAKE),+ ...zProfileWindow,})).query(async ({ input, ctx }) => {@@ -133,7 +157,11 @@ export const profileRouter = createTRPCRouter({level: 'read',});- return getPowerUsers(ctx, input);+ const { timezone } = await getSettingsForProject(ctx, input.projectId);+ return getPowerUsers(ctx, {+ ...input,+ ...getChartStartEndDate(input, timezone),+ });}),values: protectedProcedurediff --git a/packages/core/src/modules/profile/profile.service.ts b/packages/core/src/modules/profile/profile.service.tsindex a5521ae4a..93b449541 100644--- a/packages/core/src/modules/profile/profile.service.ts+++ b/packages/core/src/modules/profile/profile.service.ts@@ -31,6 +31,7 @@ import {toNullIfDefaultMinDate,} from './src/dates';import type { CompiledFilterClauses } from './src/filter-clauses';+import type { ProfileWindow } from './src/sql';import {findProfilesQuery,powerUsersQuery,@@ -253,7 +254,7 @@ export const getProfilesCached = cacheablePerDeps(PROFILES_CACHE_SECONDS);-export interface GetProfileListOptions {+export interface GetProfileListOptions extends ProfileWindow {projectId: string;take: number;cursor?: number;@@ -271,6 +272,8 @@ export async function getProfileList(filters,search,isExternal,+ startDate,+ endDate,}: GetProfileListOptions): Promise<IServiceProfile[]> {const data = await chQuery<IClickhouseProfile>(@@ -281,6 +284,8 @@ export async function getProfileList(offset: Math.max(0, (cursor ?? 0) * take),search,isExternal,+ startDate,+ endDate,filterClauses: await compileProfileFilters(filters, projectId),}));@@ -294,6 +299,8 @@ export async function getProfileListCount(filters,isExternal,search,+ startDate,+ endDate,}: Omit<GetProfileListOptions, 'cursor' | 'take'>): Promise<number> {const data = await chQuery<{ count: number }>(@@ -302,6 +309,8 @@ export async function getProfileListCount(projectId,search,isExternal,+ startDate,+ endDate,filterClauses: await compileProfileFilters(filters, projectId),}));@@ -553,7 +562,7 @@ export async function getProfileListPage(export async function getPowerUsers(deps: ServiceDeps,- input: {+ input: ProfileWindow & {projectId: string;cursor?: number;take: number;@@ -565,6 +574,8 @@ export async function getPowerUsers(projectId: input.projectId,take: input.take,offset: input.cursor ? input.cursor * input.take : 0,+ startDate: input.startDate,+ endDate: input.endDate,}));const profiles = await getProfiles(diff --git a/packages/core/src/modules/profile/src/profile.sql.proof.md b/packages/core/src/modules/profile/src/profile.sql.proof.mdindex 365031616..240f1ac57 100644--- a/packages/core/src/modules/profile/src/profile.sql.proof.md+++ b/packages/core/src/modules/profile/src/profile.sql.proof.md@@ -22,6 +22,18 @@ Every query builder in `src/sql.ts` (`profile.sql.ts` at proof time; renamed und- The per-statement `rows_read V1/V2` and `wall V1/V2` figures below are from the 2026-09-04 run andtherefore describe the pre-amendment V2 text.+- **Amended by M31-003 (2026-09-16): `profileListQuery`, `profileListCountQuery` and `powerUsersQuery`+ are bounded to an explicit window.** M25 Group C fix 8, approved by Carl on 2026-09-15: all three+ had no date filter at all, so their cost grew with the tenant's lifetime rather than with anything+ the caller chose (`docs/ANALYTICS_PERFORMANCE.md` §6.8). Each now carries+ `created_at BETWEEN toDateTime({pN:String}) AND toDateTime({pN:String})`, spelled as+ `overview/src/pages.sql.ts` spells it. **This is the approved semantic change** — "power users of+ all time" is now "power users of the range" — so these three cases are no longer V1-identical by+ construction, and the V1 lines below stand as the record of what they used to be. `created_at` is+ the column both tables can prune on (`profiles` is `PARTITION BY toYYYYMM(created_at)`; `events`+ has it in the sort key) and the column the list already orders by; `profiles.last_seen_at` is in+ neither and would prune nothing. See the M31-003 section at the end for the measurements.+- **Date**: 2026-09-04. **Data**: local prod-copy `openpanel` (319,499,000 events, static), project `redcollege` unless noted; cases needing rows the prod-copy lacks (`cohort_members`, `events_bots` are empty there) also ran on the isolated `openpanel_test` database with 3,380 events copied from `redcollege` under project `m7-002-proof`, plus seeded cohort/bot rows (deleted afterwards).- **Machine**: single-node ClickHouse 26.1.3.52 on a 4-vCPU box — timings are directional only; production is 2 shards × 2 replicas (`docs/ENVIRONMENT.md`). `rows_read` differences on JOIN statements are ClickHouse's per-run accounting of the right-hand side, not a plan difference: the SQL text is identical up to `{pN:Type}` binding.- **Verdict**: 20 cases / 23 statements, all IDENTICAL. No conversion changed a result set.@@ -335,3 +347,70 @@ Expression ((Project names + (Projection + (Change column names to column identiParsed and resolved (73.6 ms). The plan shows one `ReadFromMergeTree` on `events` — the property the string tests guard; the `profiles` CTE is consumed through scalar subqueries, which ClickHouse evaluates ahead of the plan it prints. The result-set diff for the same builder on a real profile is the `profileMetricsQuery` case above.+## M31-003 — the explicit window (2026-09-16)++Run against the local prod-copy `openpanel` on **2026-09-16**, `use_query_condition_cache=0`,+`session_timezone=UTC`, numbers off `X-ClickHouse-Summary`, every statement rendered by the builder+itself (not retyped) and bound through `param_pN=`. Old and new were **interleaved** within each+5-iteration loop so neither owns the warm cache; the ms column is the median of 5, the rows/bytes+columns are invariant across iterations. `LIMIT 50`, which is `DEFAULT_LIST_TAKE`.++**A caveat this copy forces, stated first.** The copy's whole history fits inside three months —+`events` end 2026-08-25 and `profiles.created_at` spans 2026-07-01 -> 2026-08-30 — so at the default+`3m` range the window covers **all** the data and cannot prune anything. The `3m` column below is+therefore the *floor* case (what the change costs when it buys nothing), and the `7d` column is what+it buys as soon as the window is narrower than the tenant's history. On a tenant older than the+window the `3m` column moves toward the `7d` one; nothing on this box can show that, because no+project here is older than the window.++### `powerUsersQuery` (`events`)++| project | unbounded (V1) | default `3m` | `7d` (2026-08-18 -> 08-25) |+|---|---|---|---|+| `verdict` | 657 ms / 44,164,383 rows / 1017 MiB | 834 ms / 44,164,383 / 1354 MiB | **105 ms / 4,821,286 / 154 MiB** |+| `earlysalary-production` | 621 ms / 26,787,424 / 1624 MiB | 725 ms / 26,787,424 / 1828 MiB | **100 ms / 3,465,150 / 228 MiB** |+| `chatpaper` | 666 ms / 19,797,378 / 888 MiB | 777 ms / 19,797,378 / 1039 MiB | **101 ms / 2,662,273 / 134 MiB** |++At a window narrower than the data this is **6.3x fewer rows and ~6x faster**. At the default on this+copy it is **15-27 % slower**, and that is not noise: the `BETWEEN` adds `created_at` to the columns+read (+337 / +204 / +151 MiB) while pruning nothing, because everything is inside the window. That is+the unavoidable floor price of any date filter on this table, and it is bounded — after the change+the endpoint can never read more than three months, where before it read the tenant's whole life.++### `profileListQuery` / `profileListCountQuery` (`profiles`)++| statement | project | unbounded (V1) | default `3m` | `7d` |+|---|---|---|---|---|+| list | `chatpaper` | 309 ms / 1,858,944 / 707 MiB | 292 ms / 1,858,944 / 707 MiB | **125 ms / 794,624 / 301 MiB** |+| list | `earlysalary-production` | 278 ms / 1,827,940 / 861 MiB | 279 ms / 1,827,940 / 861 MiB | **128 ms / 851,968 / 406 MiB** |+| list | `verdict` | 156 ms / 916,864 / 380 MiB | 155 ms / 916,864 / 380 MiB | **71 ms / 434,176 / 178 MiB** |+| count | `chatpaper` | 65 ms / 1,662,816 / 74 MiB | 78 ms / 1,662,816 / 87 MiB | **40 ms / 745,472 / 39 MiB** |+| count | `earlysalary-production` | 49 ms / 1,624,345 / 95 MiB | 54 ms / 1,624,345 / 107 MiB | **24 ms / 802,816 / 33 MiB** |+| count | `verdict` | 30 ms / 745,312 / 32 MiB | 37 ms / 745,312 / 37 MiB | **31 ms / 385,024 / 19 MiB** |++The list pays **nothing** at the default — `SELECT *` already read `created_at` — and halves at a+narrower window. Only the count pays the extra column (+5 to +13 ms), for the same reason+`powerUsers` does.++### Result sets: what changed, and what did not++The window is a pure restriction, so where it covers all the data the answer must be unchanged. It+is, on every anchor — compared as result **sets** (`sorted(rows)`, sha256), `LIMIT 50`:++| query | `verdict` | `chatpaper` | `earlysalary-production` | `bayse` | `website-8103` |+|---|---|---|---|---|---|+| `powerUsersQuery` unbounded vs default `3m` | IDENTICAL | IDENTICAL | IDENTICAL | IDENTICAL | IDENTICAL |+| `profileListQuery` unbounded vs default `3m` | IDENTICAL | IDENTICAL | — | — | — |+| `profileListCountQuery` unbounded vs default `3m` | 696,160 = 696,160 | 1,609,483 = 1,609,483 | — | — | — |++At `7d` they differ, which is the approved change: `verdict`'s top-50 power users keep **23 of 50**+ids, `bayse` **39 of 50**; the profile count falls 1,609,483 -> 208,529 on `chatpaper` and+696,160 -> 92,275 on `verdict`.++**One thing to know before comparing `powerUsersQuery` by ordered rows: it was already+nondeterministic, and still is.** `ORDER BY count() DESC` has no tiebreak, so equal-count rows come+back in whatever order the parallel aggregation produced. Five runs of the **unbounded** statement,+unchanged, on 2026-09-16: `chatpaper` returned **4 distinct** ordered-row hashes, `earlysalary-production`+**5 distinct**. The top-50 *set* was stable across all of them. This is pre-existing (M31-003 changed+no `ORDER BY`), and it is why the table above compares sets — an ordered-row diff of this builder+reports a difference that is not a change.diff --git a/packages/core/src/modules/profile/src/sql.test.ts b/packages/core/src/modules/profile/src/sql.test.tsindex e3cc45d8e..9b5afe940 100644--- a/packages/core/src/modules/profile/src/sql.test.ts+++ b/packages/core/src/modules/profile/src/sql.test.ts@@ -30,6 +30,10 @@ const PROJECT_ID = 'proj-1';const PROFILE_ID = 'prof-1';const HOSTILE = "x' OR 1=1 --";const NO_FILTERS = {};+const WINDOW = {+ startDate: '2026-06-16 00:00:00',+ endDate: '2026-09-16 23:59:59',+};const COLUMN_LIST = PROFILE_COLUMNS.join(', ');function collapse(text: string): string {@@ -136,19 +140,25 @@ describe('single-profile lookups', () => {});describe('profileListQuery', () => {- test('plain page: project only, no OFFSET at offset 0', () => {+ test('plain page: project + window, no OFFSET at offset 0', () => {const { query, query_params } = profileListQuery({projectId: PROJECT_ID,take: 50,offset: 0,filterClauses: NO_FILTERS,+ ...WINDOW,}).toStatement();const text = collapse(query);expect(text).toBe(- 'SELECT * FROM profiles FINAL WHERE project_id = {p1:String} ORDER BY created_at DESC LIMIT {p2:UInt64}'+ 'SELECT * FROM profiles FINAL WHERE project_id = {p1:String} AND created_at BETWEEN toDateTime({p2:String}) AND toDateTime({p3:String}) ORDER BY created_at DESC LIMIT {p4:UInt64}');- expect(query_params).toEqual({ p1: PROJECT_ID, p2: 50 });+ expect(query_params).toEqual({+ p1: PROJECT_ID,+ p2: '2026-06-16 00:00:00',+ p3: '2026-09-16 23:59:59',+ p4: 50,+ });});test('search + isExternal + filters + offset all splice in V1 order', () => {@@ -159,18 +169,19 @@ describe('profileListQuery', () => {search: 'ann',isExternal: true,filterClauses: { f0: sql`(properties['plan'] = 'pro')` },+ ...WINDOW,}).toStatement();const text = collapse(query);expect(text).toContain(- "WHERE project_id = {p1:String} AND ((id ILIKE {p2:String} OR email ILIKE {p3:String} OR first_name ILIKE {p4:String} OR last_name ILIKE {p5:String} OR concat(first_name, ' ', last_name) ILIKE {p6:String})) AND is_external = {p7:Bool} AND (properties['plan'] = 'pro') ORDER BY created_at DESC LIMIT {p8:UInt64} OFFSET {p9:UInt64}"+ "WHERE project_id = {p1:String} AND created_at BETWEEN toDateTime({p2:String}) AND toDateTime({p3:String}) AND ((id ILIKE {p4:String} OR email ILIKE {p5:String} OR first_name ILIKE {p6:String} OR last_name ILIKE {p7:String} OR concat(first_name, ' ', last_name) ILIKE {p8:String})) AND is_external = {p9:Bool} AND (properties['plan'] = 'pro') ORDER BY created_at DESC LIMIT {p10:UInt64} OFFSET {p11:UInt64}");expect(query_params).toMatchObject({p1: PROJECT_ID,- p2: '%ann%',- p7: true,- p8: 50,- p9: 100,+ p4: '%ann%',+ p9: true,+ p10: 50,+ p11: 100,});});@@ -180,6 +191,7 @@ describe('profileListQuery', () => {search: 'ann',isExternal: false,filterClauses: { f0: sql`(email != '')` },+ ...WINDOW,};const page = collapse(profileListQuery({ ...list, take: 1, offset: 0 }).toStatement().query@@ -326,21 +338,47 @@ describe('the trpc router queries', () => {projectId: PROJECT_ID,take: 50,offset: 0,+ ...WINDOW,}).toStatement();const later = powerUsersQuery({projectId: PROJECT_ID,take: 50,offset: 50,+ ...WINDOW,}).toStatement();expect(collapse(first.query)).toEndWith(- 'GROUP BY profile_id ORDER BY count() DESC LIMIT {p2:UInt64}'+ 'GROUP BY profile_id ORDER BY count() DESC LIMIT {p4:UInt64}');- expect(first.query_params).toEqual({ p1: PROJECT_ID, p2: 50 });+ expect(first.query_params).toEqual({+ p1: PROJECT_ID,+ p2: '2026-06-16 00:00:00',+ p3: '2026-09-16 23:59:59',+ p4: 50,+ });expect(collapse(later.query)).toEndWith(- 'LIMIT {p2:UInt64} OFFSET {p3:UInt64}'+ 'LIMIT {p4:UInt64} OFFSET {p5:UInt64}'+ );+ expect(later.query_params).toEqual({+ p1: PROJECT_ID,+ p2: '2026-06-16 00:00:00',+ p3: '2026-09-16 23:59:59',+ p4: 50,+ p5: 50,+ });+ });++ test('powerUsers is bounded to the window it was given', () => {+ const { query } = powerUsersQuery({+ projectId: PROJECT_ID,+ take: 50,+ offset: 0,+ ...WINDOW,+ }).toStatement();++ expect(collapse(query)).toContain(+ 'project_id = {p1:String} AND created_at BETWEEN toDateTime({p2:String}) AND toDateTime({p3:String}) GROUP BY profile_id');- expect(later.query_params).toEqual({ p1: PROJECT_ID, p2: 50, p3: 50 });});});diff --git a/packages/core/src/modules/profile/src/sql.ts b/packages/core/src/modules/profile/src/sql.tsindex f73d83b4e..db445a893 100644--- a/packages/core/src/modules/profile/src/sql.ts+++ b/packages/core/src/modules/profile/src/sql.ts@@ -55,6 +55,25 @@ function optional(return condition ? fragment() : sql.empty;}+function toDateTimeLiteral(value: string): string {+ return new Date(value).toISOString().slice(0, 19).replace('T', ' ');+}++/** Same spelling `overview/src/pages.sql.ts` uses, so the two read alike. */+function dateRangeWhere(+ column: string,+ startDate: string,+ endDate: string+): SqlFragment {+ return sql`${sql.id(column)} BETWEEN toDateTime(${sql.string(toDateTimeLiteral(startDate))}) AND toDateTime(${sql.string(toDateTimeLiteral(endDate))})`;+}++/** The window every profile-list / power-user statement is now bounded to. */+export interface ProfileWindow {+ startDate: string;+ endDate: string;+}+/*** Multi-token profile search: every whitespace-separated token (max 5) must* match SOME of id/email/first/last/full name, case-insensitively. `null`@@ -151,7 +170,7 @@ export function profilesByIdsQuery(query: {`;}-export interface ProfileListQuery {+export interface ProfileListQuery extends ProfileWindow {projectId: string;search?: string;isExternal?: boolean;@@ -159,13 +178,20 @@ export interface ProfileListQuery {filterClauses: CompiledFilterClauses;}-/** V1's `WHERE project_id = .. [AND search] [AND is_external = ..] [AND filters]`. */+/**+ * V1's `WHERE project_id = .. [AND search] [AND is_external = ..] [AND filters]`,+ * plus the M31-003 window: V1 read every profile the project ever had, so the+ * cost grew with the tenant's lifetime rather than with anything the caller+ * asked for. `created_at` is the column the list already orders by, so the+ * bound and the order agree.+ */function profileListCondition(query: ProfileListQuery): SqlFragment {const search = profileSearchCondition(query.search);- return sql`project_id = ${sql.string(query.projectId)}${optional(- search,- () => sql` AND ${search as SqlFragment}`- )}${optional(+ return sql`project_id = ${sql.string(query.projectId)} AND ${dateRangeWhere(+ 'created_at',+ query.startDate,+ query.endDate+ )}${optional(search, () => sql` AND ${search as SqlFragment}`)}${optional(query.isExternal !== undefined,() => sql` AND is_external = ${sql.bool(query.isExternal as boolean)}`)} ${spliceCompiledFilters(query.filterClauses)}`;@@ -345,17 +371,25 @@ export function profilePropertyNamesQuery(projectId: string): SqlFragment {return sql`SELECT distinct mapKeys(properties) as keys from ${sql.id(TABLE.profiles)} where project_id = ${sql.string(projectId)};`;}-export function powerUsersQuery(query: {- projectId: string;- take: number;- offset: number;-}): SqlFragment {+/**+ * M31-003: bounded to the caller's window. Unbounded this ranked every event+ * the project ever recorded (44.2 M rows on the busiest anchor), so "power+ * users of all time" is now "power users of the range".+ */+export function powerUsersQuery(+ query: ProfileWindow & {+ projectId: string;+ take: number;+ offset: number;+ }+): SqlFragment {return sql`SELECT profile_id, count(*) as countFROM ${sql.id(TABLE.events)}WHEREprofile_id != ''AND project_id = ${sql.string(query.projectId)}+ AND ${dateRangeWhere('created_at', query.startDate, query.endDate)}GROUP BY profile_idORDER BY count() DESCLIMIT ${sql.uint64(query.take)} ${optional(query.offset, () => sql`OFFSET ${sql.uint64(query.offset)}`)}`;