Skip to content

OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.

ralph(M34-002): Finish Group C fix 8: the REST retention series and page_titles bounded to range

ralph9c4c08714ee4
diff --git a/packages/core/src/modules/chart/chart.constants.ts b/packages/core/src/modules/chart/chart.constants.ts
index d9cb96b9c..97271d9b2 100644
--- a/packages/core/src/modules/chart/chart.constants.ts
+++ b/packages/core/src/modules/chart/chart.constants.ts
@@ -6,6 +6,14 @@
import { z } from 'zod';
import { zChartEvent } from '../report/report.constants';
+/**
+ * Window the REST retention routes (`/insights/:projectId/retention`,
+ * `/engagement`) read when the caller names none; M31-003's default for the
+ * same fix, wider than engagement's 60-day churn bucket. Not applied to
+ * callers of the service that pass no dates: those still read all time.
+ */
+export const RETENTION_SERIES_DEFAULT_RANGE = '3m';
+
/** User-flow depth: how many events one sankey path may span. */
export const DEFAULT_SANKEY_STEPS = 5;
diff --git a/packages/core/src/modules/chart/retention.service.test.ts b/packages/core/src/modules/chart/retention.service.test.ts
index bcab277fa..778d9eaed 100644
--- a/packages/core/src/modules/chart/retention.service.test.ts
+++ b/packages/core/src/modules/chart/retention.service.test.ts
@@ -11,6 +11,7 @@ import { testServiceDeps } from '../../../test/service-deps';
import type { ServiceDeps } from '../../services';
import {
getRetentionCohort,
+ getRetentionLastSeenSeries,
getRetentionSeries,
type IRetentionCohortRow,
processCohortData,
@@ -22,6 +23,9 @@ const PROJECT_ID = 'test-retention-cohort';
// same `ServiceDeps` main.ts does (pointed at `openpanel_test` by preload.ts).
let deps: ServiceDeps;
const { day, week } = RETENTION_FIXTURE;
+/** RU1-RU5 and WU1-WU2, per the dataset table in retention-fixtures.ts. */
+const DAY_SCENARIO_USERS = 5;
+const WEEK_SCENARIO_USERS = 2;
// bun:test's `toEqual` overloads (unlike vitest's) reject a `readonly` actual
// against a mutable expected type — the blueprint fixture is `as const`. Pure
@@ -239,6 +243,56 @@ describe('getRetentionSeries', () => {
const rows = await getRetentionSeries(deps, { projectId: PROJECT_ID });
expect(rows).toEqual(mutable(RETENTION_BLUEPRINT.weeklySeries));
});
+
+ it('only reads events inside the requested window', async () => {
+ const rows = await getRetentionSeries(deps, {
+ projectId: PROJECT_ID,
+ startDate: week.start,
+ endDate: week.end,
+ });
+ expect(rows).toEqual(mutable(RETENTION_BLUEPRINT.weeklySeries.slice(1)));
+ });
+
+ it('reads the same series through a window spanning every event', async () => {
+ const rows = await getRetentionSeries(deps, {
+ projectId: PROJECT_ID,
+ startDate: day.start,
+ endDate: week.end,
+ });
+ expect(rows).toEqual(mutable(RETENTION_BLUEPRINT.weeklySeries));
+ });
+});
+
+describe('getRetentionLastSeenSeries', () => {
+ beforeAll(async () => {
+ deps = await testServiceDeps();
+ await setupRetentionFixtures(PROJECT_ID);
+ });
+
+ afterAll(async () => {
+ await teardownRetentionFixtures(PROJECT_ID);
+ });
+
+ async function usersSeen(window: { startDate?: string; endDate?: string }) {
+ const rows = await getRetentionLastSeenSeries(deps, {
+ projectId: PROJECT_ID,
+ ...window,
+ });
+ return rows.reduce((total, row) => total + row.users, 0);
+ }
+
+ it('counts every identified profile when no window is given', async () => {
+ expect(await usersSeen({})).toBe(DAY_SCENARIO_USERS + WEEK_SCENARIO_USERS);
+ });
+
+ it('counts only profiles seen inside the requested window', async () => {
+ expect(await usersSeen({ startDate: day.start, endDate: day.end })).toBe(
+ DAY_SCENARIO_USERS
+ );
+ expect(await usersSeen({ startDate: week.start, endDate: week.end })).toBe(
+ WEEK_SCENARIO_USERS
+ );
+ });
});
describe('processCohortData weighted average (pure)', () => {
diff --git a/packages/core/src/modules/chart/retention.service.ts b/packages/core/src/modules/chart/retention.service.ts
index 82ca6425a..7065766ec 100644
--- a/packages/core/src/modules/chart/retention.service.ts
+++ b/packages/core/src/modules/chart/retention.service.ts
@@ -14,6 +14,7 @@ import type { IChartEventFilter } from '../report/report.constants';
import {
type IRetentionCriteria,
type IRetentionInterval,
+ type RetentionSeriesQueryInput,
retentionCohortQuery,
retentionLastSeenSeriesQuery,
retentionSeriesQuery,
@@ -44,9 +45,7 @@ const ROLLING_ACTIVE_USER_LABELS: Record<number, string> = {
const RETENTION_PERCENTAGE_DECIMALS = 2;
const RETENTION_COUNT_DECIMALS = 0;
-interface IGetWeekRetentionInput {
- projectId: string;
-}
+type IGetWeekRetentionInput = RetentionSeriesQueryInput;
export interface IServiceRetentionRollingActiveUsers {
date: string;
@@ -88,21 +87,21 @@ export interface IGetRetentionCohortInput {
// self-joins that much smaller set to its next week.
export function getRetentionSeries(
deps: ServiceDeps,
- { projectId }: IGetWeekRetentionInput
+ input: IGetWeekRetentionInput
) {
return runQuery<{
date: string;
active_users: number;
retained_users: number;
retention: number;
- }>(deps, retentionSeriesQuery(projectId));
+ }>(deps, retentionSeriesQuery(input));
}
// https://medium.com/@andre_bodro/how-to-fast-calculating-mau-in-clickhouse-fd793559b229
// Rolling active users
export function getRollingActiveUsers(
deps: ServiceDeps,
- { projectId, days }: IGetWeekRetentionInput & { days: number }
+ { projectId, days }: { projectId: string; days: number }
) {
return runQuery<IServiceRetentionRollingActiveUsers>(
deps,
@@ -112,12 +111,12 @@ export function getRollingActiveUsers(
export function getRetentionLastSeenSeries(
deps: ServiceDeps,
- { projectId }: IGetWeekRetentionInput
+ input: IGetWeekRetentionInput
) {
return runQuery<{
days: number;
users: number;
- }>(deps, retentionLastSeenSeriesQuery(projectId));
+ }>(deps, retentionLastSeenSeriesQuery(input));
}
export async function getRollingActiveUsersCore(
@@ -137,9 +136,9 @@ export async function getRollingActiveUsersCore(
export async function getWeeklyRetentionSeriesCore(
deps: ServiceDeps,
- projectId: string
+ input: IGetWeekRetentionInput
) {
- return getRetentionSeries(deps, { projectId });
+ return getRetentionSeries(deps, input);
}
// Weekly active-user retention cohort over the last 12 weeks, computed by the
@@ -159,8 +158,11 @@ export async function getRetentionCohortCore(
});
}
-export async function getEngagementCore(deps: ServiceDeps, projectId: string) {
- const raw = await getRetentionLastSeenSeries(deps, { projectId });
+export async function getEngagementCore(
+ deps: ServiceDeps,
+ input: IGetWeekRetentionInput
+) {
+ const raw = await getRetentionLastSeenSeries(deps, input);
let active_0_7 = 0;
let active_8_14 = 0;
@@ -421,12 +423,11 @@ export function createRetentionService(
): ReturnType<typeof getRollingActiveUsersCore> =>
getRollingActiveUsersCore(deps, input),
getWeeklyRetentionSeriesCore: (
- projectId: string
+ input: IGetWeekRetentionInput
): ReturnType<typeof getWeeklyRetentionSeriesCore> =>
- getWeeklyRetentionSeriesCore(deps, projectId),
+ getWeeklyRetentionSeriesCore(deps, input),
getEngagementCore: (
- projectId: string
- ): ReturnType<typeof getEngagementCore> =>
- getEngagementCore(deps, projectId),
+ input: IGetWeekRetentionInput
+ ): ReturnType<typeof getEngagementCore> => getEngagementCore(deps, input),
};
}
diff --git a/packages/core/src/modules/chart/src/retention.sql.proof.md b/packages/core/src/modules/chart/src/retention.sql.proof.md
index 056d2745d..fbcad41e3 100644
--- a/packages/core/src/modules/chart/src/retention.sql.proof.md
+++ b/packages/core/src/modules/chart/src/retention.sql.proof.md
@@ -176,3 +176,97 @@ WITH cohort_users AS ( SELECT profile_id AS userID, toDate(min(created_at)) AS c
WITH cohort_users AS ( SELECT profile_id AS userID, toDate(min(created_at)) AS cohort_interval FROM cohort_events_mv WHERE project_id = {p1:String} AND created_at >= toDateTime({p2:String}) AND created_at <= toDateTime({p3:String}) GROUP BY profile_id ), last_event AS ( SELECT profile_id, toDate(created_at) AS event_date FROM cohort_events_mv WHERE project_id = {p4:String} AND created_at >= toDateTime({p5:String}) AND created_at <= toDateTime({p6:String}) + INTERVAL {p7:UInt64} DAY ), retention_matrix AS ( SELECT f.cohort_interval, l.profile_id, dateDiff({p8:String}, f.cohort_interval, toDate(l.event_date)) AS x_after_cohort FROM cohort_users AS f INNER JOIN last_event AS l ON f.userID = l.profile_id WHERE l.event_date >= f.cohort_interval AND dateDiff({p9:String}, f.cohort_interval, toDate(l.event_date)) <= {p10:UInt64} ), interval_users AS ( SELECT cohort_interval, groupUniqArrayIf(profile_id, x_after_cohort >= {p11:UInt64}) AS interval_0_users, groupUniqArrayIf(profile_id, x_after_cohort >= {p12:UInt64}) AS interval_1_users, groupUniqArrayIf(profile_id, x_after_cohort >= {p13:UInt64}) AS interval_2_users, groupUniqArrayIf(profile_id, x_after_cohort >= {p14:UInt64}) AS interval_3_users FROM retention_matrix GROUP BY cohort_interval ), cohort_sizes AS ( SELECT cohort_interval, COUNT(DISTINCT userID) AS total_first_event_count FROM cohort_users GROUP BY cohort_interval ) SELECT interval_users.cohort_interval AS cohort_interval, cs.total_first_event_count AS total_first_event_count, length(interval_0_users) AS interval_0_user_count, length(interval_1_users) AS interval_1_user_count, length(interval_2_users) AS interval_2_user_count, length(interval_3_users) AS interval_3_user_count FROM interval_users LEFT JOIN cohort_sizes AS cs ON interval_users.cohort_interval = cs.cohort_interval ORDER BY cohort_interval ASC
-- V2 params: {"p1":"verdict","p2":"2026-07-06 00:00:00","p3":"2026-07-09 23:59:59","p4":"verdict","p5":"2026-07-06 00:00:00","p6":"2026-07-09 23:59:59","p7":3,"p8":"DAY","p9":"DAY","p10":3,"p11":0,"p12":1,"p13":2,"p14":3}
```
+
+## M34-002 — the REST retention series bounded to a window (Group C fix 8, remainder)
+
+- **Date**: 2026-09-16, 09:20–09:30 UTC. **Data**: local prod-copy `openpanel`, read-only
+ (`readonly=2`; `events` read 321,191,857 before the run). `use_query_condition_cache=0`,
+ `max_memory_usage` 12 GiB. **Both statements were rendered by the builders themselves.** "Before" is
+ `retention.sql.ts` at `26a56aa0`, copied next to this file for the run and deleted afterwards.
+ "After" is this change. Parameters were bound through `param_pN=`. Timings are the median of 3
+ `FORMAT Null` runs, taken from `X-ClickHouse-Summary`. Each builder's three runs were back to back,
+ not interleaved. The result hash is the sha256 of one extra `FORMAT JSONEachRow` run. Only counts,
+ timings and hashes were kept; no row left the box.
+- **Windows**: `1d` = 2026-08-25, `7d` = 2026-08-19 -> 08-25, `30d` = 2026-07-27 -> 08-25
+ (`23:59:59`-closed; the copy's last event is 2026-08-25 08:49). `default3m` is what a caller
+ naming no window got on the run day: `2026-06-16 00:00:00` -> `2026-09-17 00:00:00`.
+
+Both `weekly_active` (series) and `last_active` (engagement) read every identified event the project
+ever had. They now also carry `AND created_at BETWEEN toDateTime({pN:String}) AND
+toDateTime({pM:String})`, the same spelling as `pages.sql.ts` and `profile/src/sql.ts`.
+`/insights/:projectId/retention` and `/engagement` now accept `range` / `startDate` / `endDate`,
+resolved by `resolveInsightsDateRange` like the sibling routes, with
+`RETENTION_SERIES_DEFAULT_RANGE = '3m'` when `range` is omitted. That default lives **on the
+routes only**. A service caller that passes no `startDate`/`endDate` still gets the all-time
+statement, token-for-token and parameter-for-parameter the one at `26a56aa0` (checked by rendering
+both builders). Only whitespace differs, because the bound fragment renders empty.
+
+**The approved change:**
+- series: a week before the window disappears, and a week the window cuts in half counts only its
+ in-window users.
+- engagement: a profile whose last event is before the window is no longer counted. With `3m` it
+ stays wider than the 60-day `churned_60_plus` bucket, so that bucket still gets users.
+- **Not changed:** the MCP tools `get_weekly_retention_series` (`mcp/.../active-users.ts`) and
+ `get_user_last_seen_distribution` (`mcp/.../engagement.ts`). They call the service with only
+ `projectId`, so they keep reading all time. Their contracts depend on that: up to
+ `MAX_WEEKLY_POINTS = 260` weeks, and a `churned_60_plus_days` bucket that must count profiles gone
+ for longer than any default window. Bounding them is a decision for the `mcp` module. It is out of
+ this task's scope and is handed forward.
+
+> **Correction (attempt 2).** The first attempt put the `3m` fallback in the shared service
+> functions. That silently capped both MCP tools to ~13 weeks and dropped long-churned profiles from
+> the engagement distribution. Review rejected it. The fallback now lives only in
+> `export.routes.ts`, and `retention.service.test.ts` keeps the no-window call asserting the full
+> all-time series and all seven fixture profiles.
+
+### Result sets and cost — before (all-time) vs after
+
+`R` = result rows. Hash = the first 12 hex digits of the JSON result's sha256.
+
+| project | statement | before (all-time) | after 1d | after 7d | after 30d | after `default3m` |
+|---|---|---|---|---|---|---|
+| `verdict` | series | 3,004 ms / 88.33 M rows / 63 MiB / R 9 / `700f4be4a890` | 36 ms / 0.60 M | 385 ms / 8.69 M | 1,648 ms / 44.59 M | 3,123 ms / 88.33 M / R 9 / **`700f4be4a890`** |
+| `verdict` | lastSeen | 1,238 ms / 44.16 M / R 56 / `6476bf15f406` | 21 ms / 0.30 M | 142 ms / 4.35 M | 673 ms / 22.30 M | 1,288 ms / 44.16 M / R 56 / **`6476bf15f406`** |
+| `bayse` | series | 2,423 ms / 76.49 M / R 9 / `8fdc236bf7e1` | 32 ms / 0.65 M | 334 ms / 9.30 M | 1,216 ms / 33.36 M | 2,601 ms / 76.49 M / **`8fdc236bf7e1`** |
+| `bayse` | lastSeen | 1,039 ms / 38.25 M / R 56 / `0b08b3135340` | 16 ms / 0.33 M | 146 ms / 4.65 M | 522 ms / 16.68 M | 1,123 ms / 38.25 M / **`0b08b3135340`** |
+| `earlysalary-production` | series | 2,159 ms / 53.58 M / 271 MiB / R 9 / `ac3fdda7c4c3` | 48 ms / 0.84 M | 285 ms / 5.96 M | 1,330 ms / 30.10 M | 2,293 ms / 53.58 M / **`ac3fdda7c4c3`** |
+| `earlysalary-production` | lastSeen | 964 ms / 26.79 M / R 56 / `f9ac35c14bc3` | 24 ms / 0.42 M | 153 ms / 2.98 M | 586 ms / 15.05 M | 1,041 ms / 26.79 M / **`f9ac35c14bc3`** |
+| `website-8103` | series | 131 ms / 46.25 M / R 0 | 24 ms / 0.53 M | 40 ms / 3.81 M | 185 ms / 24.24 M | 332 ms / 46.25 M / R 0 |
+| `website-8103` | lastSeen | 63 ms / 23.13 M / R 0 | 11 ms / 0.26 M | 20 ms / 1.91 M | 90 ms / 12.12 M | 163 ms / 23.13 M / R 0 |
+| `chatpaper` | series | 1,160 ms / 39.59 M / R 0 | 34 ms / 0.64 M | 163 ms / 4.64 M | 894 ms / 26.46 M | 1,288 ms / 39.59 M / R 0 |
+| `chatpaper` | lastSeen | 583 ms / 19.80 M / R 0 | 15 ms / 0.32 M | 83 ms / 2.32 M | 447 ms / 13.23 M | 640 ms / 19.80 M / R 0 |
+| `secure-privacy` | series | 75 ms / 1.11 M / R 9 / `87ee95b1e36e` | 30 ms / 0.36 M | 23 ms / 0.36 M | 50 ms / 0.97 M | 81 ms / 1.11 M / **`87ee95b1e36e`** |
+| `secure-privacy` | lastSeen | 37 ms / 0.56 M / R 56 / `cddc379beaa3` | 9 ms / 0.18 M | 11 ms / 0.18 M | 27 ms / 0.48 M | 38 ms / 0.56 M / **`cddc379beaa3`** |
+
+(`website-8103` and `chatpaper` have no identified profiles (`profile_id != device_id`), so both
+statements return 0 rows either way.) The series reads each project's events **twice**, because
+ClickHouse runs the `weekly_active` CTE once per reference. The self-join needs it on both sides.
+
+**What this shows.**
+- **`default3m` is byte-identical to the old all-time statement on all six projects.** On the run
+ day, every identified event in the copy (earliest 2026-07-01) is inside the default window, so the
+ answer is the same and so is the work (same `read_rows`; timings are within run-to-run noise).
+- **Short windows are what gets cheaper:** 1d is 30–80x cheaper, and 7d is 7–8x cheaper.
+- **This copy hides the real cost, and it cuts both ways, as with `page_titles`.** The copy holds
+ only ~8 weeks of data, so "all-time" here is small. In production, the old statement read the
+ tenant's **whole lifetime** on every call. There, a `3m` default is a strict improvement for any
+ tenant older than three months. The cost that is genuinely new is for **explicit windows longer
+ than the tenant's lifetime**, and there is none: a window can never read more rows than all-time
+ did. Unlike `page_titles`, this fix cannot make any statement more expensive than before. The one
+ thing it adds is on the REST routes: a Postgres lookup of the project timezone, the same one the
+ sibling `/insights` routes already do.
+
+### Golden coverage — predicted: zero changes today, with a known date when that stops
+
+The cases are `insights-retention-weekly` (`verdict`), `insights-retention-weekly-secure` and
+`insights-engagement` (`verdict`). All three call the REST routes with **no query**, so they get
+`default3m`. The
+table above shows the identical result for both projects on the run day. The `insights-engagement`
+normaliser works on dates and on summary sums, which are unchanged because the row set is unchanged.
+
+**These cases stop being time-independent.** `insights-retention-weekly*` is marked
+`deterministic`. That was true when the statement read all time. With a `3m` default, the window
+start is `now - 3 months`. It passes the two projects' first identified event (2026-07-01, in UTC)
+on **2026-10-02** (UTC; an organisation timezone other than UTC moves this by hours). From then on, the first week row (`2026-06-28`) shrinks and the cases differ
+for a reason that has nothing to do with code. To stay reproducible, a re-capture would have to pin
+`startDate`/`endDate`. That is M34-003's call; this task may not touch `verification/`.
diff --git a/packages/core/src/modules/chart/src/retention.sql.ts b/packages/core/src/modules/chart/src/retention.sql.ts
index 907993e5c..c043d4739 100644
--- a/packages/core/src/modules/chart/src/retention.sql.ts
+++ b/packages/core/src/modules/chart/src/retention.sql.ts
@@ -16,6 +16,7 @@
import { type SqlFragment, sql } from '@openpanel/db/src/clickhouse/sql';
import type { IChartEventFilter } from '../../report/report.constants';
import { compiledText } from './compiled';
+import { formatClickhouseDate } from './dates';
import { CHART_TABLE } from './field-resolution';
import { getEventFiltersWhereClause } from './filter-where';
@@ -45,8 +46,36 @@ const COUNT_CRITERIA: Record<IRetentionCriteria, string> = {
on: '=',
};
-/** Week-over-week active-user retention, one row per week. */
-export function retentionSeriesQuery(projectId: string): SqlFragment {
+/**
+ * Without both dates the statement reads the project's whole lifetime, as V1
+ * did: the MCP tools pass none and their contracts rely on it (M34-002).
+ */
+export interface RetentionSeriesQueryInput {
+ projectId: string;
+ startDate?: string;
+ endDate?: string;
+}
+
+/** Same spelling as `overview/src/pages.sql.ts` and `profile/src/sql.ts`. */
+function createdAtWithin({
+ startDate,
+ endDate,
+}: RetentionSeriesQueryInput): SqlFragment {
+ if (!(startDate && endDate)) {
+ return sql.empty;
+ }
+ return sql`AND created_at BETWEEN toDateTime(${sql.string(formatClickhouseDate(startDate))}) AND toDateTime(${sql.string(formatClickhouseDate(endDate))})`;
+}
+
+/**
+ * Week-over-week active-user retention, one row per week, within the window
+ * when one is given (Group C fix 8); a week whose following week falls
+ * outside it reports zero retained users.
+ */
+export function retentionSeriesQuery(
+ input: RetentionSeriesQueryInput
+): SqlFragment {
+ const { projectId } = input;
return sql`
WITH weekly_active AS (
SELECT
@@ -55,6 +84,7 @@ export function retentionSeriesQuery(projectId: string): SqlFragment {
FROM ${sql.id(CHART_TABLE.events)}
WHERE project_id = ${sql.string(projectId)}
AND profile_id != device_id
+ ${createdAtWithin(input)}
GROUP BY profile_id, week
)
SELECT
@@ -100,8 +130,14 @@ export function rollingActiveUsersQuery(
GROUP BY date`;
}
-/** Days-since-last-seen distribution over identified profiles. */
-export function retentionLastSeenSeriesQuery(projectId: string): SqlFragment {
+/**
+ * Days-since-last-seen distribution over identified profiles; with a window, a
+ * profile last seen before it is no longer counted (fix 8).
+ */
+export function retentionLastSeenSeriesQuery(
+ input: RetentionSeriesQueryInput
+): SqlFragment {
+ const { projectId } = input;
return sql`
WITH last_active AS (
SELECT
@@ -109,6 +145,7 @@ export function retentionLastSeenSeriesQuery(projectId: string): SqlFragment {
profile_id
FROM ${sql.id(CHART_TABLE.events)}
WHERE (project_id = ${sql.string(projectId)}) AND (device_id != profile_id)
+ ${createdAtWithin(input)}
GROUP BY profile_id
)
SELECT
diff --git a/packages/core/src/modules/export/export.routes.ts b/packages/core/src/modules/export/export.routes.ts
index ef7394ed8..52674a303 100644
--- a/packages/core/src/modules/export/export.routes.ts
+++ b/packages/core/src/modules/export/export.routes.ts
@@ -33,6 +33,7 @@ import type { ClientType } from '../../http/client-auth';
import { defineRoutes } from '../../http/define';
import { parseQueryStringTransform } from '../../http/query';
import { HttpError } from '../../shared/errors';
+import { RETENTION_SERIES_DEFAULT_RANGE } from '../chart/chart.constants';
import type { GetEventListOptions } from '../event/event.service';
import {
getEventList,
@@ -363,6 +364,19 @@ async function getOverviewGeneric(
});
}
+/** Both retention series read `RETENTION_SERIES_DEFAULT_RANGE` unless the caller names a window. */
+async function resolveRetentionSeriesWindow(
+ ctx: Ctx,
+ projectId: string,
+ query: z.infer<typeof zDateRange>
+) {
+ const window = await resolveInsightsDateRange(ctx, projectId, {
+ ...query,
+ range: query.range ?? RETENTION_SERIES_DEFAULT_RANGE,
+ });
+ return { projectId, ...window };
+}
+
const zEntryExitQuery = zDateRange.extend({
mode: z.enum(['entry', 'exit']).default('entry'),
});
@@ -574,16 +588,19 @@ export const insightsRoutes = defineRoutes((app) =>
)
.get(
'/insights/:projectId/retention',
- async ({ params, client, ctx }) => {
+ async ({ params, query, client, ctx }) => {
const projectId = await resolveInsightsProjectId(ctx, client, params);
- return ctx.services.chart.getWeeklyRetentionSeriesCore(projectId);
+ return ctx.services.chart.getWeeklyRetentionSeriesCore(
+ await resolveRetentionSeriesWindow(ctx, projectId, query)
+ );
},
{
clientAuth: CLIENT_ALLOW,
params: projectIdParam,
+ query: zDateRange,
detail: {
tags: INSIGHTS_TAGS,
- description: 'Get weekly retention series data.',
+ description: `Get weekly retention series data within the date range (default \`${RETENTION_SERIES_DEFAULT_RANGE}\`).`,
},
}
)
@@ -1133,16 +1150,19 @@ export const insightsRoutes = defineRoutes((app) =>
)
.get(
'/insights/:projectId/engagement',
- async ({ params, client, ctx }) => {
+ async ({ params, query, client, ctx }) => {
const projectId = await resolveInsightsProjectId(ctx, client, params);
- return ctx.services.chart.getEngagementCore(projectId);
+ return ctx.services.chart.getEngagementCore(
+ await resolveRetentionSeriesWindow(ctx, projectId, query)
+ );
},
{
clientAuth: CLIENT_ALLOW,
params: projectIdParam,
+ query: zDateRange,
detail: {
tags: INSIGHTS_TAGS,
- description: 'Get engagement metrics for the project.',
+ description: `Get engagement metrics for profiles seen within the date range (default \`${RETENTION_SERIES_DEFAULT_RANGE}\`).`,
},
}
)
diff --git a/packages/core/src/modules/overview/src/pages.sql.proof.md b/packages/core/src/modules/overview/src/pages.sql.proof.md
index 8284b5876..790fc997b 100644
--- a/packages/core/src/modules/overview/src/pages.sql.proof.md
+++ b/packages/core/src/modules/overview/src/pages.sql.proof.md
@@ -115,3 +115,89 @@ Omitting `topPagesPerBucket` renders the query exactly as before: token-identica
parameter-identical to the version at `4b7afb8e`, differing only by trailing whitespace where
the two conditional fragments render empty. `event.pageTimeseries` (the `origin`+`path`-filtered
variant) never passes it and is unchanged.
+
+## M34-002 — `page_titles` bounded to the query's own range (Group C fix 8, remainder)
+
+- **Date**: 2026-09-16, 09:02–09:06 UTC. **Data**: local prod-copy `openpanel`, read-only `SELECT`
+ (`events` read 321,191,847 before the run). `use_query_condition_cache=0`,
+ `session_timezone=UTC`, `max_memory_usage=6 GB`. Both statements rendered by the builders
+ themselves — "before" is `pages.sql.ts` at `26a56aa0`, "after" is this change — and bound through
+ `param_pN=`. Timings interleave old/new, median of 3, `FORMAT Null`, from `X-ClickHouse-Summary`.
+- Windows: `1d` = 2026-08-24, `7d` = 2026-08-18 -> 08-24, `30d` = 2026-07-26 -> 08-24 (all
+ `23:59:59`-closed).
+
+The `page_titles` CTE read `created_at >= now() - INTERVAL 30 DAY`, whatever the caller asked for.
+It now reads `created_at BETWEEN toDateTime({p2:String}) AND toDateTime({p3:String})` — the
+spelling the other two CTEs and M31-003 already use. Only the title column can change: the pages
+and their numbers come from `screen_view_durations` and `sessions`, which were already bounded.
+
+**The approved change:** a page whose title shows up **only outside the requested range** now gets
+an empty title. The reverse also happens: a page viewed in the range whose only titles are more than
+30 days before *now* used to have an empty title and now gets one.
+
+### Result sets — before vs after, joined on `(origin, path)`, no `LIMIT`
+
+| project | window | pages (both) | numeric columns differ | title differs |
+|---|---|---:|---:|---:|
+| `verdict` | 1d / 7d / 30d | 0 / 0 / 0 (no `screen_view`) | 0 | 0 |
+| `earlysalary-production` | 1d / 7d / 30d | 1 / 2 / 2 | 0 | 0 |
+| `website-8103` | 1d / 7d / 30d | 3,918 / 13,995 / 104,824 | 0 | 0 |
+| `bayse` | 1d / 7d / 30d | 440 / 2,036 / 7,653 | 0 | 67 / 124 / 4,638 |
+| `chatpaper` | 1d / 7d | 62,198 / 594,845 | 0 | 192 / 12,915 |
+
+(`chatpaper` 30d was not joined: each side is 2.68 M pages.) **The pages and every numeric column
+are identical everywhere.** Only titles differ, and part of that difference is not caused by this
+change: `anyLast` already flickered **before** it (see the M7-005 section above). The old statement
+compared with itself gave 14 differing titles on `bayse` 7d and 888 on `chatpaper` 1d. The new
+statement compared with itself gave 75 on `bayse` 7d.
+
+To measure only the approved effect, I counted **whether any non-empty title exists** for each
+page in each window. That count does not flicker:
+
+| project | window | loses its only title (titled only outside range, inside old window) | gains a title (titled only in range, outside old window) |
+|---|---|---:|---:|
+| `bayse` | 1d / 7d / 30d | 7 / 3 / 1 | 0 / 0 / **4,950** |
+| `chatpaper` | 1d / 7d | 70 / **3,311** | 0 / 0 |
+| `website-8103` | 1d / 7d / 30d | 0 / 0 / 0 | 0 / 0 / 0 |
+
+On this copy, the old window was 2026-08-17 -> "now" (2026-09-16), so it held only the last eight
+days of data. That is why a 30-day report on `bayse` *gained* 4,950 titles: those pages were titled
+only before 08-17.
+
+### Timing (median of 3, `LIMIT 50`)
+
+| project / window | before (`now() - 30 DAY`) | after (range) |
+|---|---|---|
+| `verdict` 1d | 50 ms / 596,328 rows / 7 MiB read | 39 ms / 186,788 / 5 MiB |
+| `verdict` 30d | 159 ms / 2,752,863 / 49 MiB | 164 ms / 3,784,293 / 52 MiB |
+| `bayse` 1d | 452 ms / 6,691,332 / 1,743 MiB | **103 ms / 1,809,700 / 241 MiB** |
+| `bayse` 7d | 563 ms / 10,991,899 / 1,944 MiB | 533 ms / 10,394,455 / 1,765 MiB |
+| `bayse` 30d | 1,047 ms / 23,230,204 / 2,502 MiB | 1,937 ms / 34,436,987 / 6,122 MiB |
+| `earlysalary-production` 1d | 860 ms / 4,549,922 / 9,208 MiB | **232 ms / 1,482,682 / 1,314 MiB** |
+| `earlysalary-production` 7d | 968 ms / 7,146,746 / 9,457 MiB | 930 ms / 6,651,762 / 8,265 MiB |
+| `earlysalary-production` 30d | 1,583 ms / 19,631,267 / 10,618 MiB | 4,120 ms / 31,137,529 / 39,818 MiB |
+| `website-8103` 1d | 986 ms / 3,025,109 / 4,542 MiB | **326 ms / 879,433 / 459 MiB** |
+| `website-8103` 7d | 1,196 ms / 4,822,514 / 4,708 MiB | 1,083 ms / 4,474,243 / 4,069 MiB |
+| `website-8103` 30d | 2,285 ms / 14,965,439 / 5,661 MiB | 5,702 ms / 24,735,702 / 23,842 MiB |
+| `chatpaper` 1d | 452 ms / 3,456,236 / 537 MiB | **143 ms / 950,030 / 67 MiB** |
+| `chatpaper` 7d | 1,168 ms / 6,208,658 / 759 MiB | 1,079 ms / 5,865,077 / 695 MiB |
+| `chatpaper` 30d | 6,706 ms / 19,798,407 / 1,848 MiB | 8,671 ms / 30,300,939 / 3,957 MiB |
+
+(`verdict` 7d: 51 ms either way.) **Short windows get rid of the 30-day floor: 3–4x faster at 1d.**
+The 30d rows get slower, and **this copy exaggerates that**. With the copy's last event on
+2026-08-25, the old CTE read only eight days of data here. In production, "now" is current, so the
+old CTE always read a full 30 days, and a 30d report reads about the same amount either way. The
+real cost is for windows **longer than 30 days**: they now read the `properties` Map over the whole
+range (§6.5 flagged this as the price of fix 8). Removing that Map read altogether is the separate
+`__title` materialisation option in §6.5, and it was not proposed.
+
+### Golden coverage
+
+Only `getPagePerformanceCore` (`/insights/:projectId/pages/performance`) reaches this builder from
+the REST surface. That covers the three `insights-pages-performance-*` cases, all on
+`secure-privacy`, 2026-08-20. I ran each case's statement (`limit: 1000`, with the case's
+`search`) and compared the results. The new statement matches both the old one as of today and the
+old one with `now()` pinned to the capture instant (`2026-09-07 02:43:03`): same pages, numbers and
+titles. It also matches every page in the captured response: 28/28, 28/28 and 6/6 titles equal.
+These cases are `clock-anchored` (`require-same-utc-day`), so the gate reports them as stale. The
+reason given for that rule is this CTE's `now()`, which no longer exists.
diff --git a/packages/core/src/modules/overview/src/pages.sql.test.ts b/packages/core/src/modules/overview/src/pages.sql.test.ts
index 50c38c264..f6285611f 100644
--- a/packages/core/src/modules/overview/src/pages.sql.test.ts
+++ b/packages/core/src/modules/overview/src/pages.sql.test.ts
@@ -3,7 +3,7 @@
* `overview.sql.test.ts` — see its header comment.
*/
-import { beforeAll, describe, it } from 'bun:test';
+import { beforeAll, describe, expect, it } from 'bun:test';
import type { SqlFragment } from '@openpanel/db/src/clickhouse/sql';
const PROJECT_ID = 'test-sql-validation';
@@ -48,6 +48,25 @@ describe('pages.sql — parses against ClickHouse', () => {
);
});
+ it('topPagesQuery — page_titles is bounded to the requested range', () => {
+ const { query, query_params } = PG.topPagesQuery({
+ projectId: PROJECT_ID,
+ startDate: START,
+ endDate: END,
+ }).toStatement();
+ const titlesCte = query.slice(
+ query.indexOf('page_titles AS ('),
+ query.indexOf('screen_view_durations AS (')
+ );
+
+ expect(query).not.toContain('now()');
+ expect(titlesCte).toContain(
+ 'created_at BETWEEN toDateTime({p2:String}) AND toDateTime({p3:String})'
+ );
+ expect(query_params.p2).toBe(START);
+ expect(query_params.p3).toBe(END);
+ });
+
it('pageTimeseriesQuery — with origin/path filters, week bucket', async () => {
await explain(
PG.pageTimeseriesQuery({
diff --git a/packages/core/src/modules/overview/src/pages.sql.ts b/packages/core/src/modules/overview/src/pages.sql.ts
index 1874f73e2..5e8d8d75a 100644
--- a/packages/core/src/modules/overview/src/pages.sql.ts
+++ b/packages/core/src/modules/overview/src/pages.sql.ts
@@ -38,12 +38,16 @@ function searchClause(search: string | undefined): SqlFragment {
}
export function topPagesQuery(input: TopPagesQueryInput): SqlFragment {
+ // M34-002: titles come from the requested range, not V1's `now() - 30 DAY`,
+ // which read a month of the widest Map column for any window and made the
+ // answer depend on the wall clock. A page titled only outside the range now
+ // resolves to '' (approved, Group C fix 8).
const titlesCte = sql`
SELECT concat(origin, path) as page_key, anyLast(properties['__title']) as title
FROM events
WHERE project_id = ${sql.string(input.projectId)}
AND name = 'screen_view'
- AND created_at >= now() - INTERVAL 30 DAY
+ AND ${dateRangeWhere('created_at', input.startDate, input.endDate)}
GROUP BY origin, path
`;
diff --git a/packages/core/src/modules/profile/src/profile.sql.proof.md b/packages/core/src/modules/profile/src/profile.sql.proof.md
index 240f1ac57..6e1290b0e 100644
--- a/packages/core/src/modules/profile/src/profile.sql.proof.md
+++ b/packages/core/src/modules/profile/src/profile.sql.proof.md
@@ -414,3 +414,15 @@ unchanged, on 2026-09-16: `chatpaper` returned **4 distinct** ordered-row hashes
**5 distinct**. The top-50 *set* was stable across all of them. This is pre-existing (M31-003 changed
no `ORDER BY`), and it is why the table above compares sets — an ordered-row diff of this builder
reports a difference that is not a change.
+
+### M34-002 — the rest of fix 8
+
+- **`page_titles` in `event.pages` / `event.previousPages`**: done. It uses the same
+ `created_at BETWEEN toDateTime(..) AND toDateTime(..)` spelling. The proof is in
+ `overview/src/pages.sql.proof.md`, section "M34-002".
+- **The REST retention series** (`/insights/:projectId/retention`, `/engagement`): done, after the
+ task was re-scoped to `chart` and `export`. `retentionSeriesQuery` and
+ `retentionLastSeenSeriesQuery` use the same `created_at BETWEEN toDateTime(..) AND toDateTime(..)`
+ spelling. Both routes now accept `range` / `startDate` / `endDate`, and the default is `3m`, the
+ same as here. The default is applied on the routes only: the MCP tools that call the same
+ service without dates still read all time. The proof is in `chart/src/retention.sql.proof.md`, section "M34-002".