Skip to content

OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.

ralph(M31-003): Group C fix 8: give powerUsers, profile.list and retention an explicit window

ralphac9cf1d21a23
diff --git a/packages/core/src/modules/profile/profile.rpc.ts b/packages/core/src/modules/profile/profile.rpc.ts
index 733a0679b..45725e354 100644
--- a/packages/core/src/modules/profile/profile.rpc.ts
+++ b/packages/core/src/modules/profile/profile.rpc.ts
@@ -9,7 +9,9 @@
import { z } from 'zod';
import { createTRPCRouter, protectedProcedure } from '../../rpc/base';
-import { zChartEventFilter } from '../report/report.constants';
+import { getSettingsForProject } from '../organization/organization.service';
+import { zChartEventFilter, zRange } from '../report/report.constants';
+import { getChartStartEndDate } from '../report/src/chart-dates';
import {
getPowerUsers,
getProfileActivity,
@@ -24,6 +26,22 @@ import {
const DEFAULT_LIST_TAKE = 50;
+/**
+ * `list` and `powerUsers` had no date filter at all, so their cost grew with
+ * the tenant's lifetime rather than with anything the caller chose
+ * (`docs/ANALYTICS_PERFORMANCE.md` §6.8: 44.2 M rows read for one page of 50).
+ * Both now take the report vocabulary's window; this is the default when the
+ * caller names none, and it is the range §6.8 itself uses to describe the
+ * change ("a power user over 90 days is not the same set as over all time").
+ */
+const PROFILE_WINDOW_DEFAULT_RANGE = '3m';
+
+const zProfileWindow = {
+ range: zRange.default(PROFILE_WINDOW_DEFAULT_RANGE),
+ startDate: z.string().nullish(),
+ endDate: z.string().nullish(),
+};
+
const zProfileRef = z.object({ profileId: z.string(), projectId: z.string() });
export const profileRouter = createTRPCRouter({
@@ -106,6 +124,7 @@ export const profileRouter = createTRPCRouter({
search: z.string().optional(),
isExternal: z.boolean().optional(),
filters: z.array(zChartEventFilter).default([]),
+ ...zProfileWindow,
})
)
.query(async ({ input, ctx }) => {
@@ -115,7 +134,11 @@ export const profileRouter = createTRPCRouter({
level: 'read',
});
- return getProfileListPage(ctx, input);
+ const { timezone } = await getSettingsForProject(ctx, input.projectId);
+ return getProfileListPage(ctx, {
+ ...input,
+ ...getChartStartEndDate(input, timezone),
+ });
}),
powerUsers: protectedProcedure
@@ -124,6 +147,7 @@ export const profileRouter = createTRPCRouter({
projectId: z.string(),
cursor: z.number().optional(),
take: z.number().default(DEFAULT_LIST_TAKE),
+ ...zProfileWindow,
})
)
.query(async ({ input, ctx }) => {
@@ -133,7 +157,11 @@ export const profileRouter = createTRPCRouter({
level: 'read',
});
- return getPowerUsers(ctx, input);
+ const { timezone } = await getSettingsForProject(ctx, input.projectId);
+ return getPowerUsers(ctx, {
+ ...input,
+ ...getChartStartEndDate(input, timezone),
+ });
}),
values: protectedProcedure
diff --git a/packages/core/src/modules/profile/profile.service.ts b/packages/core/src/modules/profile/profile.service.ts
index a5521ae4a..93b449541 100644
--- a/packages/core/src/modules/profile/profile.service.ts
+++ b/packages/core/src/modules/profile/profile.service.ts
@@ -31,6 +31,7 @@ import {
toNullIfDefaultMinDate,
} from './src/dates';
import type { CompiledFilterClauses } from './src/filter-clauses';
+import type { ProfileWindow } from './src/sql';
import {
findProfilesQuery,
powerUsersQuery,
@@ -253,7 +254,7 @@ export const getProfilesCached = cacheablePerDeps(
PROFILES_CACHE_SECONDS
);
-export interface GetProfileListOptions {
+export interface GetProfileListOptions extends ProfileWindow {
projectId: string;
take: number;
cursor?: number;
@@ -271,6 +272,8 @@ export async function getProfileList(
filters,
search,
isExternal,
+ startDate,
+ endDate,
}: GetProfileListOptions
): Promise<IServiceProfile[]> {
const data = await chQuery<IClickhouseProfile>(
@@ -281,6 +284,8 @@ export async function getProfileList(
offset: Math.max(0, (cursor ?? 0) * take),
search,
isExternal,
+ startDate,
+ endDate,
filterClauses: await compileProfileFilters(filters, projectId),
})
);
@@ -294,6 +299,8 @@ export async function getProfileListCount(
filters,
isExternal,
search,
+ startDate,
+ endDate,
}: Omit<GetProfileListOptions, 'cursor' | 'take'>
): Promise<number> {
const data = await chQuery<{ count: number }>(
@@ -302,6 +309,8 @@ export async function getProfileListCount(
projectId,
search,
isExternal,
+ startDate,
+ endDate,
filterClauses: await compileProfileFilters(filters, projectId),
})
);
@@ -553,7 +562,7 @@ export async function getProfileListPage(
export async function getPowerUsers(
deps: ServiceDeps,
- input: {
+ input: ProfileWindow & {
projectId: string;
cursor?: number;
take: number;
@@ -565,6 +574,8 @@ export async function getPowerUsers(
projectId: input.projectId,
take: input.take,
offset: input.cursor ? input.cursor * input.take : 0,
+ startDate: input.startDate,
+ endDate: input.endDate,
})
);
const profiles = await getProfiles(
diff --git a/packages/core/src/modules/profile/src/profile.sql.proof.md b/packages/core/src/modules/profile/src/profile.sql.proof.md
index 365031616..240f1ac57 100644
--- a/packages/core/src/modules/profile/src/profile.sql.proof.md
+++ b/packages/core/src/modules/profile/src/profile.sql.proof.md
@@ -22,6 +22,18 @@ Every query builder in `src/sql.ts` (`profile.sql.ts` at proof time; renamed und
- The per-statement `rows_read V1/V2` and `wall V1/V2` figures below are from the 2026-09-04 run and
therefore describe the pre-amendment V2 text.
+- **Amended by M31-003 (2026-09-16): `profileListQuery`, `profileListCountQuery` and `powerUsersQuery`
+ are bounded to an explicit window.** M25 Group C fix 8, approved by Carl on 2026-09-15: all three
+ had no date filter at all, so their cost grew with the tenant's lifetime rather than with anything
+ the caller chose (`docs/ANALYTICS_PERFORMANCE.md` §6.8). Each now carries
+ `created_at BETWEEN toDateTime({pN:String}) AND toDateTime({pN:String})`, spelled as
+ `overview/src/pages.sql.ts` spells it. **This is the approved semantic change** — "power users of
+ all time" is now "power users of the range" — so these three cases are no longer V1-identical by
+ construction, and the V1 lines below stand as the record of what they used to be. `created_at` is
+ the column both tables can prune on (`profiles` is `PARTITION BY toYYYYMM(created_at)`; `events`
+ has it in the sort key) and the column the list already orders by; `profiles.last_seen_at` is in
+ neither and would prune nothing. See the M31-003 section at the end for the measurements.
+
- **Date**: 2026-09-04. **Data**: local prod-copy `openpanel` (319,499,000 events, static), project `redcollege` unless noted; cases needing rows the prod-copy lacks (`cohort_members`, `events_bots` are empty there) also ran on the isolated `openpanel_test` database with 3,380 events copied from `redcollege` under project `m7-002-proof`, plus seeded cohort/bot rows (deleted afterwards).
- **Machine**: single-node ClickHouse 26.1.3.52 on a 4-vCPU box — timings are directional only; production is 2 shards × 2 replicas (`docs/ENVIRONMENT.md`). `rows_read` differences on JOIN statements are ClickHouse's per-run accounting of the right-hand side, not a plan difference: the SQL text is identical up to `{pN:Type}` binding.
- **Verdict**: 20 cases / 23 statements, all IDENTICAL. No conversion changed a result set.
@@ -335,3 +347,70 @@ Expression ((Project names + (Projection + (Change column names to column identi
Parsed and resolved (73.6 ms). The plan shows one `ReadFromMergeTree` on `events` — the property the string tests guard; the `profiles` CTE is consumed through scalar subqueries, which ClickHouse evaluates ahead of the plan it prints. The result-set diff for the same builder on a real profile is the `profileMetricsQuery` case above.
+## M31-003 — the explicit window (2026-09-16)
+
+Run against the local prod-copy `openpanel` on **2026-09-16**, `use_query_condition_cache=0`,
+`session_timezone=UTC`, numbers off `X-ClickHouse-Summary`, every statement rendered by the builder
+itself (not retyped) and bound through `param_pN=`. Old and new were **interleaved** within each
+5-iteration loop so neither owns the warm cache; the ms column is the median of 5, the rows/bytes
+columns are invariant across iterations. `LIMIT 50`, which is `DEFAULT_LIST_TAKE`.
+
+**A caveat this copy forces, stated first.** The copy's whole history fits inside three months —
+`events` end 2026-08-25 and `profiles.created_at` spans 2026-07-01 -> 2026-08-30 — so at the default
+`3m` range the window covers **all** the data and cannot prune anything. The `3m` column below is
+therefore the *floor* case (what the change costs when it buys nothing), and the `7d` column is what
+it buys as soon as the window is narrower than the tenant's history. On a tenant older than the
+window the `3m` column moves toward the `7d` one; nothing on this box can show that, because no
+project here is older than the window.
+
+### `powerUsersQuery` (`events`)
+
+| project | unbounded (V1) | default `3m` | `7d` (2026-08-18 -> 08-25) |
+|---|---|---|---|
+| `verdict` | 657 ms / 44,164,383 rows / 1017 MiB | 834 ms / 44,164,383 / 1354 MiB | **105 ms / 4,821,286 / 154 MiB** |
+| `earlysalary-production` | 621 ms / 26,787,424 / 1624 MiB | 725 ms / 26,787,424 / 1828 MiB | **100 ms / 3,465,150 / 228 MiB** |
+| `chatpaper` | 666 ms / 19,797,378 / 888 MiB | 777 ms / 19,797,378 / 1039 MiB | **101 ms / 2,662,273 / 134 MiB** |
+
+At a window narrower than the data this is **6.3x fewer rows and ~6x faster**. At the default on this
+copy it is **15-27 % slower**, and that is not noise: the `BETWEEN` adds `created_at` to the columns
+read (+337 / +204 / +151 MiB) while pruning nothing, because everything is inside the window. That is
+the unavoidable floor price of any date filter on this table, and it is bounded — after the change
+the endpoint can never read more than three months, where before it read the tenant's whole life.
+
+### `profileListQuery` / `profileListCountQuery` (`profiles`)
+
+| statement | project | unbounded (V1) | default `3m` | `7d` |
+|---|---|---|---|---|
+| list | `chatpaper` | 309 ms / 1,858,944 / 707 MiB | 292 ms / 1,858,944 / 707 MiB | **125 ms / 794,624 / 301 MiB** |
+| list | `earlysalary-production` | 278 ms / 1,827,940 / 861 MiB | 279 ms / 1,827,940 / 861 MiB | **128 ms / 851,968 / 406 MiB** |
+| list | `verdict` | 156 ms / 916,864 / 380 MiB | 155 ms / 916,864 / 380 MiB | **71 ms / 434,176 / 178 MiB** |
+| count | `chatpaper` | 65 ms / 1,662,816 / 74 MiB | 78 ms / 1,662,816 / 87 MiB | **40 ms / 745,472 / 39 MiB** |
+| count | `earlysalary-production` | 49 ms / 1,624,345 / 95 MiB | 54 ms / 1,624,345 / 107 MiB | **24 ms / 802,816 / 33 MiB** |
+| count | `verdict` | 30 ms / 745,312 / 32 MiB | 37 ms / 745,312 / 37 MiB | **31 ms / 385,024 / 19 MiB** |
+
+The list pays **nothing** at the default — `SELECT *` already read `created_at` — and halves at a
+narrower window. Only the count pays the extra column (+5 to +13 ms), for the same reason
+`powerUsers` does.
+
+### Result sets: what changed, and what did not
+
+The window is a pure restriction, so where it covers all the data the answer must be unchanged. It
+is, on every anchor — compared as result **sets** (`sorted(rows)`, sha256), `LIMIT 50`:
+
+| query | `verdict` | `chatpaper` | `earlysalary-production` | `bayse` | `website-8103` |
+|---|---|---|---|---|---|
+| `powerUsersQuery` unbounded vs default `3m` | IDENTICAL | IDENTICAL | IDENTICAL | IDENTICAL | IDENTICAL |
+| `profileListQuery` unbounded vs default `3m` | IDENTICAL | IDENTICAL | — | — | — |
+| `profileListCountQuery` unbounded vs default `3m` | 696,160 = 696,160 | 1,609,483 = 1,609,483 | — | — | — |
+
+At `7d` they differ, which is the approved change: `verdict`'s top-50 power users keep **23 of 50**
+ids, `bayse` **39 of 50**; the profile count falls 1,609,483 -> 208,529 on `chatpaper` and
+696,160 -> 92,275 on `verdict`.
+
+**One thing to know before comparing `powerUsersQuery` by ordered rows: it was already
+nondeterministic, and still is.** `ORDER BY count() DESC` has no tiebreak, so equal-count rows come
+back in whatever order the parallel aggregation produced. Five runs of the **unbounded** statement,
+unchanged, on 2026-09-16: `chatpaper` returned **4 distinct** ordered-row hashes, `earlysalary-production`
+**5 distinct**. The top-50 *set* was stable across all of them. This is pre-existing (M31-003 changed
+no `ORDER BY`), and it is why the table above compares sets — an ordered-row diff of this builder
+reports a difference that is not a change.
diff --git a/packages/core/src/modules/profile/src/sql.test.ts b/packages/core/src/modules/profile/src/sql.test.ts
index e3cc45d8e..9b5afe940 100644
--- a/packages/core/src/modules/profile/src/sql.test.ts
+++ b/packages/core/src/modules/profile/src/sql.test.ts
@@ -30,6 +30,10 @@ const PROJECT_ID = 'proj-1';
const PROFILE_ID = 'prof-1';
const HOSTILE = "x' OR 1=1 --";
const NO_FILTERS = {};
+const WINDOW = {
+ startDate: '2026-06-16 00:00:00',
+ endDate: '2026-09-16 23:59:59',
+};
const COLUMN_LIST = PROFILE_COLUMNS.join(', ');
function collapse(text: string): string {
@@ -136,19 +140,25 @@ describe('single-profile lookups', () => {
});
describe('profileListQuery', () => {
- test('plain page: project only, no OFFSET at offset 0', () => {
+ test('plain page: project + window, no OFFSET at offset 0', () => {
const { query, query_params } = profileListQuery({
projectId: PROJECT_ID,
take: 50,
offset: 0,
filterClauses: NO_FILTERS,
+ ...WINDOW,
}).toStatement();
const text = collapse(query);
expect(text).toBe(
- 'SELECT * FROM profiles FINAL WHERE project_id = {p1:String} ORDER BY created_at DESC LIMIT {p2:UInt64}'
+ 'SELECT * FROM profiles FINAL WHERE project_id = {p1:String} AND created_at BETWEEN toDateTime({p2:String}) AND toDateTime({p3:String}) ORDER BY created_at DESC LIMIT {p4:UInt64}'
);
- expect(query_params).toEqual({ p1: PROJECT_ID, p2: 50 });
+ expect(query_params).toEqual({
+ p1: PROJECT_ID,
+ p2: '2026-06-16 00:00:00',
+ p3: '2026-09-16 23:59:59',
+ p4: 50,
+ });
});
test('search + isExternal + filters + offset all splice in V1 order', () => {
@@ -159,18 +169,19 @@ describe('profileListQuery', () => {
search: 'ann',
isExternal: true,
filterClauses: { f0: sql`(properties['plan'] = 'pro')` },
+ ...WINDOW,
}).toStatement();
const text = collapse(query);
expect(text).toContain(
- "WHERE project_id = {p1:String} AND ((id ILIKE {p2:String} OR email ILIKE {p3:String} OR first_name ILIKE {p4:String} OR last_name ILIKE {p5:String} OR concat(first_name, ' ', last_name) ILIKE {p6:String})) AND is_external = {p7:Bool} AND (properties['plan'] = 'pro') ORDER BY created_at DESC LIMIT {p8:UInt64} OFFSET {p9:UInt64}"
+ "WHERE project_id = {p1:String} AND created_at BETWEEN toDateTime({p2:String}) AND toDateTime({p3:String}) AND ((id ILIKE {p4:String} OR email ILIKE {p5:String} OR first_name ILIKE {p6:String} OR last_name ILIKE {p7:String} OR concat(first_name, ' ', last_name) ILIKE {p8:String})) AND is_external = {p9:Bool} AND (properties['plan'] = 'pro') ORDER BY created_at DESC LIMIT {p10:UInt64} OFFSET {p11:UInt64}"
);
expect(query_params).toMatchObject({
p1: PROJECT_ID,
- p2: '%ann%',
- p7: true,
- p8: 50,
- p9: 100,
+ p4: '%ann%',
+ p9: true,
+ p10: 50,
+ p11: 100,
});
});
@@ -180,6 +191,7 @@ describe('profileListQuery', () => {
search: 'ann',
isExternal: false,
filterClauses: { f0: sql`(email != '')` },
+ ...WINDOW,
};
const page = collapse(
profileListQuery({ ...list, take: 1, offset: 0 }).toStatement().query
@@ -326,21 +338,47 @@ describe('the trpc router queries', () => {
projectId: PROJECT_ID,
take: 50,
offset: 0,
+ ...WINDOW,
}).toStatement();
const later = powerUsersQuery({
projectId: PROJECT_ID,
take: 50,
offset: 50,
+ ...WINDOW,
}).toStatement();
expect(collapse(first.query)).toEndWith(
- 'GROUP BY profile_id ORDER BY count() DESC LIMIT {p2:UInt64}'
+ 'GROUP BY profile_id ORDER BY count() DESC LIMIT {p4:UInt64}'
);
- expect(first.query_params).toEqual({ p1: PROJECT_ID, p2: 50 });
+ expect(first.query_params).toEqual({
+ p1: PROJECT_ID,
+ p2: '2026-06-16 00:00:00',
+ p3: '2026-09-16 23:59:59',
+ p4: 50,
+ });
expect(collapse(later.query)).toEndWith(
- 'LIMIT {p2:UInt64} OFFSET {p3:UInt64}'
+ 'LIMIT {p4:UInt64} OFFSET {p5:UInt64}'
+ );
+ expect(later.query_params).toEqual({
+ p1: PROJECT_ID,
+ p2: '2026-06-16 00:00:00',
+ p3: '2026-09-16 23:59:59',
+ p4: 50,
+ p5: 50,
+ });
+ });
+
+ test('powerUsers is bounded to the window it was given', () => {
+ const { query } = powerUsersQuery({
+ projectId: PROJECT_ID,
+ take: 50,
+ offset: 0,
+ ...WINDOW,
+ }).toStatement();
+
+ expect(collapse(query)).toContain(
+ 'project_id = {p1:String} AND created_at BETWEEN toDateTime({p2:String}) AND toDateTime({p3:String}) GROUP BY profile_id'
);
- expect(later.query_params).toEqual({ p1: PROJECT_ID, p2: 50, p3: 50 });
});
});
diff --git a/packages/core/src/modules/profile/src/sql.ts b/packages/core/src/modules/profile/src/sql.ts
index f73d83b4e..db445a893 100644
--- a/packages/core/src/modules/profile/src/sql.ts
+++ b/packages/core/src/modules/profile/src/sql.ts
@@ -55,6 +55,25 @@ function optional(
return condition ? fragment() : sql.empty;
}
+function toDateTimeLiteral(value: string): string {
+ return new Date(value).toISOString().slice(0, 19).replace('T', ' ');
+}
+
+/** Same spelling `overview/src/pages.sql.ts` uses, so the two read alike. */
+function dateRangeWhere(
+ column: string,
+ startDate: string,
+ endDate: string
+): SqlFragment {
+ return sql`${sql.id(column)} BETWEEN toDateTime(${sql.string(toDateTimeLiteral(startDate))}) AND toDateTime(${sql.string(toDateTimeLiteral(endDate))})`;
+}
+
+/** The window every profile-list / power-user statement is now bounded to. */
+export interface ProfileWindow {
+ startDate: string;
+ endDate: string;
+}
+
/**
* Multi-token profile search: every whitespace-separated token (max 5) must
* match SOME of id/email/first/last/full name, case-insensitively. `null`
@@ -151,7 +170,7 @@ export function profilesByIdsQuery(query: {
`;
}
-export interface ProfileListQuery {
+export interface ProfileListQuery extends ProfileWindow {
projectId: string;
search?: string;
isExternal?: boolean;
@@ -159,13 +178,20 @@ export interface ProfileListQuery {
filterClauses: CompiledFilterClauses;
}
-/** V1's `WHERE project_id = .. [AND search] [AND is_external = ..] [AND filters]`. */
+/**
+ * V1's `WHERE project_id = .. [AND search] [AND is_external = ..] [AND filters]`,
+ * plus the M31-003 window: V1 read every profile the project ever had, so the
+ * cost grew with the tenant's lifetime rather than with anything the caller
+ * asked for. `created_at` is the column the list already orders by, so the
+ * bound and the order agree.
+ */
function profileListCondition(query: ProfileListQuery): SqlFragment {
const search = profileSearchCondition(query.search);
- return sql`project_id = ${sql.string(query.projectId)}${optional(
- search,
- () => sql` AND ${search as SqlFragment}`
- )}${optional(
+ return sql`project_id = ${sql.string(query.projectId)} AND ${dateRangeWhere(
+ 'created_at',
+ query.startDate,
+ query.endDate
+ )}${optional(search, () => sql` AND ${search as SqlFragment}`)}${optional(
query.isExternal !== undefined,
() => sql` AND is_external = ${sql.bool(query.isExternal as boolean)}`
)} ${spliceCompiledFilters(query.filterClauses)}`;
@@ -345,17 +371,25 @@ export function profilePropertyNamesQuery(projectId: string): SqlFragment {
return sql`SELECT distinct mapKeys(properties) as keys from ${sql.id(TABLE.profiles)} where project_id = ${sql.string(projectId)};`;
}
-export function powerUsersQuery(query: {
- projectId: string;
- take: number;
- offset: number;
-}): SqlFragment {
+/**
+ * M31-003: bounded to the caller's window. Unbounded this ranked every event
+ * the project ever recorded (44.2 M rows on the busiest anchor), so "power
+ * users of all time" is now "power users of the range".
+ */
+export function powerUsersQuery(
+ query: ProfileWindow & {
+ projectId: string;
+ take: number;
+ offset: number;
+ }
+): SqlFragment {
return sql`
SELECT profile_id, count(*) as count
FROM ${sql.id(TABLE.events)}
WHERE
profile_id != ''
AND project_id = ${sql.string(query.projectId)}
+ AND ${dateRangeWhere('created_at', query.startDate, query.endDate)}
GROUP BY profile_id
ORDER BY count() DESC
LIMIT ${sql.uint64(query.take)} ${optional(query.offset, () => sql`OFFSET ${sql.uint64(query.offset)}`)}`;