OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.
ralph(M34-002): Finish Group C fix 8: the REST retention series and page_titles bounded to range
ralph9c4c08714ee4
diff --git a/packages/core/src/modules/chart/chart.constants.ts b/packages/core/src/modules/chart/chart.constants.tsindex d9cb96b9c..97271d9b2 100644--- a/packages/core/src/modules/chart/chart.constants.ts+++ b/packages/core/src/modules/chart/chart.constants.ts@@ -6,6 +6,14 @@import { z } from 'zod';import { zChartEvent } from '../report/report.constants';+/**+ * Window the REST retention routes (`/insights/:projectId/retention`,+ * `/engagement`) read when the caller names none; M31-003's default for the+ * same fix, wider than engagement's 60-day churn bucket. Not applied to+ * callers of the service that pass no dates: those still read all time.+ */+export const RETENTION_SERIES_DEFAULT_RANGE = '3m';+/** User-flow depth: how many events one sankey path may span. */export const DEFAULT_SANKEY_STEPS = 5;diff --git a/packages/core/src/modules/chart/retention.service.test.ts b/packages/core/src/modules/chart/retention.service.test.tsindex bcab277fa..778d9eaed 100644--- a/packages/core/src/modules/chart/retention.service.test.ts+++ b/packages/core/src/modules/chart/retention.service.test.ts@@ -11,6 +11,7 @@ import { testServiceDeps } from '../../../test/service-deps';import type { ServiceDeps } from '../../services';import {getRetentionCohort,+ getRetentionLastSeenSeries,getRetentionSeries,type IRetentionCohortRow,processCohortData,@@ -22,6 +23,9 @@ const PROJECT_ID = 'test-retention-cohort';// same `ServiceDeps` main.ts does (pointed at `openpanel_test` by preload.ts).let deps: ServiceDeps;const { day, week } = RETENTION_FIXTURE;+/** RU1-RU5 and WU1-WU2, per the dataset table in retention-fixtures.ts. */+const DAY_SCENARIO_USERS = 5;+const WEEK_SCENARIO_USERS = 2;// bun:test's `toEqual` overloads (unlike vitest's) reject a `readonly` actual// against a mutable expected type — the blueprint fixture is `as const`. Pure@@ -239,6 +243,56 @@ describe('getRetentionSeries', () => {const rows = await getRetentionSeries(deps, { projectId: PROJECT_ID });expect(rows).toEqual(mutable(RETENTION_BLUEPRINT.weeklySeries));});++ it('only reads events inside the requested window', async () => {+ const rows = await getRetentionSeries(deps, {+ projectId: PROJECT_ID,+ startDate: week.start,+ endDate: week.end,+ });+ expect(rows).toEqual(mutable(RETENTION_BLUEPRINT.weeklySeries.slice(1)));+ });++ it('reads the same series through a window spanning every event', async () => {+ const rows = await getRetentionSeries(deps, {+ projectId: PROJECT_ID,+ startDate: day.start,+ endDate: week.end,+ });+ expect(rows).toEqual(mutable(RETENTION_BLUEPRINT.weeklySeries));+ });+});++describe('getRetentionLastSeenSeries', () => {+ beforeAll(async () => {+ deps = await testServiceDeps();+ await setupRetentionFixtures(PROJECT_ID);+ });++ afterAll(async () => {+ await teardownRetentionFixtures(PROJECT_ID);+ });++ async function usersSeen(window: { startDate?: string; endDate?: string }) {+ const rows = await getRetentionLastSeenSeries(deps, {+ projectId: PROJECT_ID,+ ...window,+ });+ return rows.reduce((total, row) => total + row.users, 0);+ }++ it('counts every identified profile when no window is given', async () => {+ expect(await usersSeen({})).toBe(DAY_SCENARIO_USERS + WEEK_SCENARIO_USERS);+ });++ it('counts only profiles seen inside the requested window', async () => {+ expect(await usersSeen({ startDate: day.start, endDate: day.end })).toBe(+ DAY_SCENARIO_USERS+ );+ expect(await usersSeen({ startDate: week.start, endDate: week.end })).toBe(+ WEEK_SCENARIO_USERS+ );+ });});describe('processCohortData weighted average (pure)', () => {diff --git a/packages/core/src/modules/chart/retention.service.ts b/packages/core/src/modules/chart/retention.service.tsindex 82ca6425a..7065766ec 100644--- a/packages/core/src/modules/chart/retention.service.ts+++ b/packages/core/src/modules/chart/retention.service.ts@@ -14,6 +14,7 @@ import type { IChartEventFilter } from '../report/report.constants';import {type IRetentionCriteria,type IRetentionInterval,+ type RetentionSeriesQueryInput,retentionCohortQuery,retentionLastSeenSeriesQuery,retentionSeriesQuery,@@ -44,9 +45,7 @@ const ROLLING_ACTIVE_USER_LABELS: Record<number, string> = {const RETENTION_PERCENTAGE_DECIMALS = 2;const RETENTION_COUNT_DECIMALS = 0;-interface IGetWeekRetentionInput {- projectId: string;-}+type IGetWeekRetentionInput = RetentionSeriesQueryInput;export interface IServiceRetentionRollingActiveUsers {date: string;@@ -88,21 +87,21 @@ export interface IGetRetentionCohortInput {// self-joins that much smaller set to its next week.export function getRetentionSeries(deps: ServiceDeps,- { projectId }: IGetWeekRetentionInput+ input: IGetWeekRetentionInput) {return runQuery<{date: string;active_users: number;retained_users: number;retention: number;- }>(deps, retentionSeriesQuery(projectId));+ }>(deps, retentionSeriesQuery(input));}// https://medium.com/@andre_bodro/how-to-fast-calculating-mau-in-clickhouse-fd793559b229// Rolling active usersexport function getRollingActiveUsers(deps: ServiceDeps,- { projectId, days }: IGetWeekRetentionInput & { days: number }+ { projectId, days }: { projectId: string; days: number }) {return runQuery<IServiceRetentionRollingActiveUsers>(deps,@@ -112,12 +111,12 @@ export function getRollingActiveUsers(export function getRetentionLastSeenSeries(deps: ServiceDeps,- { projectId }: IGetWeekRetentionInput+ input: IGetWeekRetentionInput) {return runQuery<{days: number;users: number;- }>(deps, retentionLastSeenSeriesQuery(projectId));+ }>(deps, retentionLastSeenSeriesQuery(input));}export async function getRollingActiveUsersCore(@@ -137,9 +136,9 @@ export async function getRollingActiveUsersCore(export async function getWeeklyRetentionSeriesCore(deps: ServiceDeps,- projectId: string+ input: IGetWeekRetentionInput) {- return getRetentionSeries(deps, { projectId });+ return getRetentionSeries(deps, input);}// Weekly active-user retention cohort over the last 12 weeks, computed by the@@ -159,8 +158,11 @@ export async function getRetentionCohortCore(});}-export async function getEngagementCore(deps: ServiceDeps, projectId: string) {- const raw = await getRetentionLastSeenSeries(deps, { projectId });+export async function getEngagementCore(+ deps: ServiceDeps,+ input: IGetWeekRetentionInput+) {+ const raw = await getRetentionLastSeenSeries(deps, input);let active_0_7 = 0;let active_8_14 = 0;@@ -421,12 +423,11 @@ export function createRetentionService(): ReturnType<typeof getRollingActiveUsersCore> =>getRollingActiveUsersCore(deps, input),getWeeklyRetentionSeriesCore: (- projectId: string+ input: IGetWeekRetentionInput): ReturnType<typeof getWeeklyRetentionSeriesCore> =>- getWeeklyRetentionSeriesCore(deps, projectId),+ getWeeklyRetentionSeriesCore(deps, input),getEngagementCore: (- projectId: string- ): ReturnType<typeof getEngagementCore> =>- getEngagementCore(deps, projectId),+ input: IGetWeekRetentionInput+ ): ReturnType<typeof getEngagementCore> => getEngagementCore(deps, input),};}diff --git a/packages/core/src/modules/chart/src/retention.sql.proof.md b/packages/core/src/modules/chart/src/retention.sql.proof.mdindex 056d2745d..fbcad41e3 100644--- a/packages/core/src/modules/chart/src/retention.sql.proof.md+++ b/packages/core/src/modules/chart/src/retention.sql.proof.md@@ -176,3 +176,97 @@ WITH cohort_users AS ( SELECT profile_id AS userID, toDate(min(created_at)) AS cWITH cohort_users AS ( SELECT profile_id AS userID, toDate(min(created_at)) AS cohort_interval FROM cohort_events_mv WHERE project_id = {p1:String} AND created_at >= toDateTime({p2:String}) AND created_at <= toDateTime({p3:String}) GROUP BY profile_id ), last_event AS ( SELECT profile_id, toDate(created_at) AS event_date FROM cohort_events_mv WHERE project_id = {p4:String} AND created_at >= toDateTime({p5:String}) AND created_at <= toDateTime({p6:String}) + INTERVAL {p7:UInt64} DAY ), retention_matrix AS ( SELECT f.cohort_interval, l.profile_id, dateDiff({p8:String}, f.cohort_interval, toDate(l.event_date)) AS x_after_cohort FROM cohort_users AS f INNER JOIN last_event AS l ON f.userID = l.profile_id WHERE l.event_date >= f.cohort_interval AND dateDiff({p9:String}, f.cohort_interval, toDate(l.event_date)) <= {p10:UInt64} ), interval_users AS ( SELECT cohort_interval, groupUniqArrayIf(profile_id, x_after_cohort >= {p11:UInt64}) AS interval_0_users, groupUniqArrayIf(profile_id, x_after_cohort >= {p12:UInt64}) AS interval_1_users, groupUniqArrayIf(profile_id, x_after_cohort >= {p13:UInt64}) AS interval_2_users, groupUniqArrayIf(profile_id, x_after_cohort >= {p14:UInt64}) AS interval_3_users FROM retention_matrix GROUP BY cohort_interval ), cohort_sizes AS ( SELECT cohort_interval, COUNT(DISTINCT userID) AS total_first_event_count FROM cohort_users GROUP BY cohort_interval ) SELECT interval_users.cohort_interval AS cohort_interval, cs.total_first_event_count AS total_first_event_count, length(interval_0_users) AS interval_0_user_count, length(interval_1_users) AS interval_1_user_count, length(interval_2_users) AS interval_2_user_count, length(interval_3_users) AS interval_3_user_count FROM interval_users LEFT JOIN cohort_sizes AS cs ON interval_users.cohort_interval = cs.cohort_interval ORDER BY cohort_interval ASC-- V2 params: {"p1":"verdict","p2":"2026-07-06 00:00:00","p3":"2026-07-09 23:59:59","p4":"verdict","p5":"2026-07-06 00:00:00","p6":"2026-07-09 23:59:59","p7":3,"p8":"DAY","p9":"DAY","p10":3,"p11":0,"p12":1,"p13":2,"p14":3}```++## M34-002 — the REST retention series bounded to a window (Group C fix 8, remainder)++- **Date**: 2026-09-16, 09:20–09:30 UTC. **Data**: local prod-copy `openpanel`, read-only+ (`readonly=2`; `events` read 321,191,857 before the run). `use_query_condition_cache=0`,+ `max_memory_usage` 12 GiB. **Both statements were rendered by the builders themselves.** "Before" is+ `retention.sql.ts` at `26a56aa0`, copied next to this file for the run and deleted afterwards.+ "After" is this change. Parameters were bound through `param_pN=`. Timings are the median of 3+ `FORMAT Null` runs, taken from `X-ClickHouse-Summary`. Each builder's three runs were back to back,+ not interleaved. The result hash is the sha256 of one extra `FORMAT JSONEachRow` run. Only counts,+ timings and hashes were kept; no row left the box.+- **Windows**: `1d` = 2026-08-25, `7d` = 2026-08-19 -> 08-25, `30d` = 2026-07-27 -> 08-25+ (`23:59:59`-closed; the copy's last event is 2026-08-25 08:49). `default3m` is what a caller+ naming no window got on the run day: `2026-06-16 00:00:00` -> `2026-09-17 00:00:00`.++Both `weekly_active` (series) and `last_active` (engagement) read every identified event the project+ever had. They now also carry `AND created_at BETWEEN toDateTime({pN:String}) AND+toDateTime({pM:String})`, the same spelling as `pages.sql.ts` and `profile/src/sql.ts`.+`/insights/:projectId/retention` and `/engagement` now accept `range` / `startDate` / `endDate`,+resolved by `resolveInsightsDateRange` like the sibling routes, with+`RETENTION_SERIES_DEFAULT_RANGE = '3m'` when `range` is omitted. That default lives **on the+routes only**. A service caller that passes no `startDate`/`endDate` still gets the all-time+statement, token-for-token and parameter-for-parameter the one at `26a56aa0` (checked by rendering+both builders). Only whitespace differs, because the bound fragment renders empty.++**The approved change:**+- series: a week before the window disappears, and a week the window cuts in half counts only its+ in-window users.+- engagement: a profile whose last event is before the window is no longer counted. With `3m` it+ stays wider than the 60-day `churned_60_plus` bucket, so that bucket still gets users.+- **Not changed:** the MCP tools `get_weekly_retention_series` (`mcp/.../active-users.ts`) and+ `get_user_last_seen_distribution` (`mcp/.../engagement.ts`). They call the service with only+ `projectId`, so they keep reading all time. Their contracts depend on that: up to+ `MAX_WEEKLY_POINTS = 260` weeks, and a `churned_60_plus_days` bucket that must count profiles gone+ for longer than any default window. Bounding them is a decision for the `mcp` module. It is out of+ this task's scope and is handed forward.++> **Correction (attempt 2).** The first attempt put the `3m` fallback in the shared service+> functions. That silently capped both MCP tools to ~13 weeks and dropped long-churned profiles from+> the engagement distribution. Review rejected it. The fallback now lives only in+> `export.routes.ts`, and `retention.service.test.ts` keeps the no-window call asserting the full+> all-time series and all seven fixture profiles.++### Result sets and cost — before (all-time) vs after++`R` = result rows. Hash = the first 12 hex digits of the JSON result's sha256.++| project | statement | before (all-time) | after 1d | after 7d | after 30d | after `default3m` |+|---|---|---|---|---|---|---|+| `verdict` | series | 3,004 ms / 88.33 M rows / 63 MiB / R 9 / `700f4be4a890` | 36 ms / 0.60 M | 385 ms / 8.69 M | 1,648 ms / 44.59 M | 3,123 ms / 88.33 M / R 9 / **`700f4be4a890`** |+| `verdict` | lastSeen | 1,238 ms / 44.16 M / R 56 / `6476bf15f406` | 21 ms / 0.30 M | 142 ms / 4.35 M | 673 ms / 22.30 M | 1,288 ms / 44.16 M / R 56 / **`6476bf15f406`** |+| `bayse` | series | 2,423 ms / 76.49 M / R 9 / `8fdc236bf7e1` | 32 ms / 0.65 M | 334 ms / 9.30 M | 1,216 ms / 33.36 M | 2,601 ms / 76.49 M / **`8fdc236bf7e1`** |+| `bayse` | lastSeen | 1,039 ms / 38.25 M / R 56 / `0b08b3135340` | 16 ms / 0.33 M | 146 ms / 4.65 M | 522 ms / 16.68 M | 1,123 ms / 38.25 M / **`0b08b3135340`** |+| `earlysalary-production` | series | 2,159 ms / 53.58 M / 271 MiB / R 9 / `ac3fdda7c4c3` | 48 ms / 0.84 M | 285 ms / 5.96 M | 1,330 ms / 30.10 M | 2,293 ms / 53.58 M / **`ac3fdda7c4c3`** |+| `earlysalary-production` | lastSeen | 964 ms / 26.79 M / R 56 / `f9ac35c14bc3` | 24 ms / 0.42 M | 153 ms / 2.98 M | 586 ms / 15.05 M | 1,041 ms / 26.79 M / **`f9ac35c14bc3`** |+| `website-8103` | series | 131 ms / 46.25 M / R 0 | 24 ms / 0.53 M | 40 ms / 3.81 M | 185 ms / 24.24 M | 332 ms / 46.25 M / R 0 |+| `website-8103` | lastSeen | 63 ms / 23.13 M / R 0 | 11 ms / 0.26 M | 20 ms / 1.91 M | 90 ms / 12.12 M | 163 ms / 23.13 M / R 0 |+| `chatpaper` | series | 1,160 ms / 39.59 M / R 0 | 34 ms / 0.64 M | 163 ms / 4.64 M | 894 ms / 26.46 M | 1,288 ms / 39.59 M / R 0 |+| `chatpaper` | lastSeen | 583 ms / 19.80 M / R 0 | 15 ms / 0.32 M | 83 ms / 2.32 M | 447 ms / 13.23 M | 640 ms / 19.80 M / R 0 |+| `secure-privacy` | series | 75 ms / 1.11 M / R 9 / `87ee95b1e36e` | 30 ms / 0.36 M | 23 ms / 0.36 M | 50 ms / 0.97 M | 81 ms / 1.11 M / **`87ee95b1e36e`** |+| `secure-privacy` | lastSeen | 37 ms / 0.56 M / R 56 / `cddc379beaa3` | 9 ms / 0.18 M | 11 ms / 0.18 M | 27 ms / 0.48 M | 38 ms / 0.56 M / **`cddc379beaa3`** |++(`website-8103` and `chatpaper` have no identified profiles (`profile_id != device_id`), so both+statements return 0 rows either way.) The series reads each project's events **twice**, because+ClickHouse runs the `weekly_active` CTE once per reference. The self-join needs it on both sides.++**What this shows.**+- **`default3m` is byte-identical to the old all-time statement on all six projects.** On the run+ day, every identified event in the copy (earliest 2026-07-01) is inside the default window, so the+ answer is the same and so is the work (same `read_rows`; timings are within run-to-run noise).+- **Short windows are what gets cheaper:** 1d is 30–80x cheaper, and 7d is 7–8x cheaper.+- **This copy hides the real cost, and it cuts both ways, as with `page_titles`.** The copy holds+ only ~8 weeks of data, so "all-time" here is small. In production, the old statement read the+ tenant's **whole lifetime** on every call. There, a `3m` default is a strict improvement for any+ tenant older than three months. The cost that is genuinely new is for **explicit windows longer+ than the tenant's lifetime**, and there is none: a window can never read more rows than all-time+ did. Unlike `page_titles`, this fix cannot make any statement more expensive than before. The one+ thing it adds is on the REST routes: a Postgres lookup of the project timezone, the same one the+ sibling `/insights` routes already do.++### Golden coverage — predicted: zero changes today, with a known date when that stops++The cases are `insights-retention-weekly` (`verdict`), `insights-retention-weekly-secure` and+`insights-engagement` (`verdict`). All three call the REST routes with **no query**, so they get+`default3m`. The+table above shows the identical result for both projects on the run day. The `insights-engagement`+normaliser works on dates and on summary sums, which are unchanged because the row set is unchanged.++**These cases stop being time-independent.** `insights-retention-weekly*` is marked+`deterministic`. That was true when the statement read all time. With a `3m` default, the window+start is `now - 3 months`. It passes the two projects' first identified event (2026-07-01, in UTC)+on **2026-10-02** (UTC; an organisation timezone other than UTC moves this by hours). From then on, the first week row (`2026-06-28`) shrinks and the cases differ+for a reason that has nothing to do with code. To stay reproducible, a re-capture would have to pin+`startDate`/`endDate`. That is M34-003's call; this task may not touch `verification/`.diff --git a/packages/core/src/modules/chart/src/retention.sql.ts b/packages/core/src/modules/chart/src/retention.sql.tsindex 907993e5c..c043d4739 100644--- a/packages/core/src/modules/chart/src/retention.sql.ts+++ b/packages/core/src/modules/chart/src/retention.sql.ts@@ -16,6 +16,7 @@import { type SqlFragment, sql } from '@openpanel/db/src/clickhouse/sql';import type { IChartEventFilter } from '../../report/report.constants';import { compiledText } from './compiled';+import { formatClickhouseDate } from './dates';import { CHART_TABLE } from './field-resolution';import { getEventFiltersWhereClause } from './filter-where';@@ -45,8 +46,36 @@ const COUNT_CRITERIA: Record<IRetentionCriteria, string> = {on: '=',};-/** Week-over-week active-user retention, one row per week. */-export function retentionSeriesQuery(projectId: string): SqlFragment {+/**+ * Without both dates the statement reads the project's whole lifetime, as V1+ * did: the MCP tools pass none and their contracts rely on it (M34-002).+ */+export interface RetentionSeriesQueryInput {+ projectId: string;+ startDate?: string;+ endDate?: string;+}++/** Same spelling as `overview/src/pages.sql.ts` and `profile/src/sql.ts`. */+function createdAtWithin({+ startDate,+ endDate,+}: RetentionSeriesQueryInput): SqlFragment {+ if (!(startDate && endDate)) {+ return sql.empty;+ }+ return sql`AND created_at BETWEEN toDateTime(${sql.string(formatClickhouseDate(startDate))}) AND toDateTime(${sql.string(formatClickhouseDate(endDate))})`;+}++/**+ * Week-over-week active-user retention, one row per week, within the window+ * when one is given (Group C fix 8); a week whose following week falls+ * outside it reports zero retained users.+ */+export function retentionSeriesQuery(+ input: RetentionSeriesQueryInput+): SqlFragment {+ const { projectId } = input;return sql`WITH weekly_active AS (SELECT@@ -55,6 +84,7 @@ export function retentionSeriesQuery(projectId: string): SqlFragment {FROM ${sql.id(CHART_TABLE.events)}WHERE project_id = ${sql.string(projectId)}AND profile_id != device_id+ ${createdAtWithin(input)}GROUP BY profile_id, week)SELECT@@ -100,8 +130,14 @@ export function rollingActiveUsersQuery(GROUP BY date`;}-/** Days-since-last-seen distribution over identified profiles. */-export function retentionLastSeenSeriesQuery(projectId: string): SqlFragment {+/**+ * Days-since-last-seen distribution over identified profiles; with a window, a+ * profile last seen before it is no longer counted (fix 8).+ */+export function retentionLastSeenSeriesQuery(+ input: RetentionSeriesQueryInput+): SqlFragment {+ const { projectId } = input;return sql`WITH last_active AS (SELECT@@ -109,6 +145,7 @@ export function retentionLastSeenSeriesQuery(projectId: string): SqlFragment {profile_idFROM ${sql.id(CHART_TABLE.events)}WHERE (project_id = ${sql.string(projectId)}) AND (device_id != profile_id)+ ${createdAtWithin(input)}GROUP BY profile_id)SELECTdiff --git a/packages/core/src/modules/export/export.routes.ts b/packages/core/src/modules/export/export.routes.tsindex ef7394ed8..52674a303 100644--- a/packages/core/src/modules/export/export.routes.ts+++ b/packages/core/src/modules/export/export.routes.ts@@ -33,6 +33,7 @@ import type { ClientType } from '../../http/client-auth';import { defineRoutes } from '../../http/define';import { parseQueryStringTransform } from '../../http/query';import { HttpError } from '../../shared/errors';+import { RETENTION_SERIES_DEFAULT_RANGE } from '../chart/chart.constants';import type { GetEventListOptions } from '../event/event.service';import {getEventList,@@ -363,6 +364,19 @@ async function getOverviewGeneric(});}+/** Both retention series read `RETENTION_SERIES_DEFAULT_RANGE` unless the caller names a window. */+async function resolveRetentionSeriesWindow(+ ctx: Ctx,+ projectId: string,+ query: z.infer<typeof zDateRange>+) {+ const window = await resolveInsightsDateRange(ctx, projectId, {+ ...query,+ range: query.range ?? RETENTION_SERIES_DEFAULT_RANGE,+ });+ return { projectId, ...window };+}+const zEntryExitQuery = zDateRange.extend({mode: z.enum(['entry', 'exit']).default('entry'),});@@ -574,16 +588,19 @@ export const insightsRoutes = defineRoutes((app) =>).get('/insights/:projectId/retention',- async ({ params, client, ctx }) => {+ async ({ params, query, client, ctx }) => {const projectId = await resolveInsightsProjectId(ctx, client, params);- return ctx.services.chart.getWeeklyRetentionSeriesCore(projectId);+ return ctx.services.chart.getWeeklyRetentionSeriesCore(+ await resolveRetentionSeriesWindow(ctx, projectId, query)+ );},{clientAuth: CLIENT_ALLOW,params: projectIdParam,+ query: zDateRange,detail: {tags: INSIGHTS_TAGS,- description: 'Get weekly retention series data.',+ description: `Get weekly retention series data within the date range (default \`${RETENTION_SERIES_DEFAULT_RANGE}\`).`,},})@@ -1133,16 +1150,19 @@ export const insightsRoutes = defineRoutes((app) =>).get('/insights/:projectId/engagement',- async ({ params, client, ctx }) => {+ async ({ params, query, client, ctx }) => {const projectId = await resolveInsightsProjectId(ctx, client, params);- return ctx.services.chart.getEngagementCore(projectId);+ return ctx.services.chart.getEngagementCore(+ await resolveRetentionSeriesWindow(ctx, projectId, query)+ );},{clientAuth: CLIENT_ALLOW,params: projectIdParam,+ query: zDateRange,detail: {tags: INSIGHTS_TAGS,- description: 'Get engagement metrics for the project.',+ description: `Get engagement metrics for profiles seen within the date range (default \`${RETENTION_SERIES_DEFAULT_RANGE}\`).`,},})diff --git a/packages/core/src/modules/overview/src/pages.sql.proof.md b/packages/core/src/modules/overview/src/pages.sql.proof.mdindex 8284b5876..790fc997b 100644--- a/packages/core/src/modules/overview/src/pages.sql.proof.md+++ b/packages/core/src/modules/overview/src/pages.sql.proof.md@@ -115,3 +115,89 @@ Omitting `topPagesPerBucket` renders the query exactly as before: token-identicaparameter-identical to the version at `4b7afb8e`, differing only by trailing whitespace wherethe two conditional fragments render empty. `event.pageTimeseries` (the `origin`+`path`-filteredvariant) never passes it and is unchanged.++## M34-002 — `page_titles` bounded to the query's own range (Group C fix 8, remainder)++- **Date**: 2026-09-16, 09:02–09:06 UTC. **Data**: local prod-copy `openpanel`, read-only `SELECT`+ (`events` read 321,191,847 before the run). `use_query_condition_cache=0`,+ `session_timezone=UTC`, `max_memory_usage=6 GB`. Both statements rendered by the builders+ themselves — "before" is `pages.sql.ts` at `26a56aa0`, "after" is this change — and bound through+ `param_pN=`. Timings interleave old/new, median of 3, `FORMAT Null`, from `X-ClickHouse-Summary`.+- Windows: `1d` = 2026-08-24, `7d` = 2026-08-18 -> 08-24, `30d` = 2026-07-26 -> 08-24 (all+ `23:59:59`-closed).++The `page_titles` CTE read `created_at >= now() - INTERVAL 30 DAY`, whatever the caller asked for.+It now reads `created_at BETWEEN toDateTime({p2:String}) AND toDateTime({p3:String})` — the+spelling the other two CTEs and M31-003 already use. Only the title column can change: the pages+and their numbers come from `screen_view_durations` and `sessions`, which were already bounded.++**The approved change:** a page whose title shows up **only outside the requested range** now gets+an empty title. The reverse also happens: a page viewed in the range whose only titles are more than+30 days before *now* used to have an empty title and now gets one.++### Result sets — before vs after, joined on `(origin, path)`, no `LIMIT`++| project | window | pages (both) | numeric columns differ | title differs |+|---|---|---:|---:|---:|+| `verdict` | 1d / 7d / 30d | 0 / 0 / 0 (no `screen_view`) | 0 | 0 |+| `earlysalary-production` | 1d / 7d / 30d | 1 / 2 / 2 | 0 | 0 |+| `website-8103` | 1d / 7d / 30d | 3,918 / 13,995 / 104,824 | 0 | 0 |+| `bayse` | 1d / 7d / 30d | 440 / 2,036 / 7,653 | 0 | 67 / 124 / 4,638 |+| `chatpaper` | 1d / 7d | 62,198 / 594,845 | 0 | 192 / 12,915 |++(`chatpaper` 30d was not joined: each side is 2.68 M pages.) **The pages and every numeric column+are identical everywhere.** Only titles differ, and part of that difference is not caused by this+change: `anyLast` already flickered **before** it (see the M7-005 section above). The old statement+compared with itself gave 14 differing titles on `bayse` 7d and 888 on `chatpaper` 1d. The new+statement compared with itself gave 75 on `bayse` 7d.++To measure only the approved effect, I counted **whether any non-empty title exists** for each+page in each window. That count does not flicker:++| project | window | loses its only title (titled only outside range, inside old window) | gains a title (titled only in range, outside old window) |+|---|---|---:|---:|+| `bayse` | 1d / 7d / 30d | 7 / 3 / 1 | 0 / 0 / **4,950** |+| `chatpaper` | 1d / 7d | 70 / **3,311** | 0 / 0 |+| `website-8103` | 1d / 7d / 30d | 0 / 0 / 0 | 0 / 0 / 0 |++On this copy, the old window was 2026-08-17 -> "now" (2026-09-16), so it held only the last eight+days of data. That is why a 30-day report on `bayse` *gained* 4,950 titles: those pages were titled+only before 08-17.++### Timing (median of 3, `LIMIT 50`)++| project / window | before (`now() - 30 DAY`) | after (range) |+|---|---|---|+| `verdict` 1d | 50 ms / 596,328 rows / 7 MiB read | 39 ms / 186,788 / 5 MiB |+| `verdict` 30d | 159 ms / 2,752,863 / 49 MiB | 164 ms / 3,784,293 / 52 MiB |+| `bayse` 1d | 452 ms / 6,691,332 / 1,743 MiB | **103 ms / 1,809,700 / 241 MiB** |+| `bayse` 7d | 563 ms / 10,991,899 / 1,944 MiB | 533 ms / 10,394,455 / 1,765 MiB |+| `bayse` 30d | 1,047 ms / 23,230,204 / 2,502 MiB | 1,937 ms / 34,436,987 / 6,122 MiB |+| `earlysalary-production` 1d | 860 ms / 4,549,922 / 9,208 MiB | **232 ms / 1,482,682 / 1,314 MiB** |+| `earlysalary-production` 7d | 968 ms / 7,146,746 / 9,457 MiB | 930 ms / 6,651,762 / 8,265 MiB |+| `earlysalary-production` 30d | 1,583 ms / 19,631,267 / 10,618 MiB | 4,120 ms / 31,137,529 / 39,818 MiB |+| `website-8103` 1d | 986 ms / 3,025,109 / 4,542 MiB | **326 ms / 879,433 / 459 MiB** |+| `website-8103` 7d | 1,196 ms / 4,822,514 / 4,708 MiB | 1,083 ms / 4,474,243 / 4,069 MiB |+| `website-8103` 30d | 2,285 ms / 14,965,439 / 5,661 MiB | 5,702 ms / 24,735,702 / 23,842 MiB |+| `chatpaper` 1d | 452 ms / 3,456,236 / 537 MiB | **143 ms / 950,030 / 67 MiB** |+| `chatpaper` 7d | 1,168 ms / 6,208,658 / 759 MiB | 1,079 ms / 5,865,077 / 695 MiB |+| `chatpaper` 30d | 6,706 ms / 19,798,407 / 1,848 MiB | 8,671 ms / 30,300,939 / 3,957 MiB |++(`verdict` 7d: 51 ms either way.) **Short windows get rid of the 30-day floor: 3–4x faster at 1d.**+The 30d rows get slower, and **this copy exaggerates that**. With the copy's last event on+2026-08-25, the old CTE read only eight days of data here. In production, "now" is current, so the+old CTE always read a full 30 days, and a 30d report reads about the same amount either way. The+real cost is for windows **longer than 30 days**: they now read the `properties` Map over the whole+range (§6.5 flagged this as the price of fix 8). Removing that Map read altogether is the separate+`__title` materialisation option in §6.5, and it was not proposed.++### Golden coverage++Only `getPagePerformanceCore` (`/insights/:projectId/pages/performance`) reaches this builder from+the REST surface. That covers the three `insights-pages-performance-*` cases, all on+`secure-privacy`, 2026-08-20. I ran each case's statement (`limit: 1000`, with the case's+`search`) and compared the results. The new statement matches both the old one as of today and the+old one with `now()` pinned to the capture instant (`2026-09-07 02:43:03`): same pages, numbers and+titles. It also matches every page in the captured response: 28/28, 28/28 and 6/6 titles equal.+These cases are `clock-anchored` (`require-same-utc-day`), so the gate reports them as stale. The+reason given for that rule is this CTE's `now()`, which no longer exists.diff --git a/packages/core/src/modules/overview/src/pages.sql.test.ts b/packages/core/src/modules/overview/src/pages.sql.test.tsindex 50c38c264..f6285611f 100644--- a/packages/core/src/modules/overview/src/pages.sql.test.ts+++ b/packages/core/src/modules/overview/src/pages.sql.test.ts@@ -3,7 +3,7 @@* `overview.sql.test.ts` — see its header comment.*/-import { beforeAll, describe, it } from 'bun:test';+import { beforeAll, describe, expect, it } from 'bun:test';import type { SqlFragment } from '@openpanel/db/src/clickhouse/sql';const PROJECT_ID = 'test-sql-validation';@@ -48,6 +48,25 @@ describe('pages.sql — parses against ClickHouse', () => {);});+ it('topPagesQuery — page_titles is bounded to the requested range', () => {+ const { query, query_params } = PG.topPagesQuery({+ projectId: PROJECT_ID,+ startDate: START,+ endDate: END,+ }).toStatement();+ const titlesCte = query.slice(+ query.indexOf('page_titles AS ('),+ query.indexOf('screen_view_durations AS (')+ );++ expect(query).not.toContain('now()');+ expect(titlesCte).toContain(+ 'created_at BETWEEN toDateTime({p2:String}) AND toDateTime({p3:String})'+ );+ expect(query_params.p2).toBe(START);+ expect(query_params.p3).toBe(END);+ });+it('pageTimeseriesQuery — with origin/path filters, week bucket', async () => {await explain(PG.pageTimeseriesQuery({diff --git a/packages/core/src/modules/overview/src/pages.sql.ts b/packages/core/src/modules/overview/src/pages.sql.tsindex 1874f73e2..5e8d8d75a 100644--- a/packages/core/src/modules/overview/src/pages.sql.ts+++ b/packages/core/src/modules/overview/src/pages.sql.ts@@ -38,12 +38,16 @@ function searchClause(search: string | undefined): SqlFragment {}export function topPagesQuery(input: TopPagesQueryInput): SqlFragment {+ // M34-002: titles come from the requested range, not V1's `now() - 30 DAY`,+ // which read a month of the widest Map column for any window and made the+ // answer depend on the wall clock. A page titled only outside the range now+ // resolves to '' (approved, Group C fix 8).const titlesCte = sql`SELECT concat(origin, path) as page_key, anyLast(properties['__title']) as titleFROM eventsWHERE project_id = ${sql.string(input.projectId)}AND name = 'screen_view'- AND created_at >= now() - INTERVAL 30 DAY+ AND ${dateRangeWhere('created_at', input.startDate, input.endDate)}GROUP BY origin, path`;diff --git a/packages/core/src/modules/profile/src/profile.sql.proof.md b/packages/core/src/modules/profile/src/profile.sql.proof.mdindex 240f1ac57..6e1290b0e 100644--- a/packages/core/src/modules/profile/src/profile.sql.proof.md+++ b/packages/core/src/modules/profile/src/profile.sql.proof.md@@ -414,3 +414,15 @@ unchanged, on 2026-09-16: `chatpaper` returned **4 distinct** ordered-row hashes**5 distinct**. The top-50 *set* was stable across all of them. This is pre-existing (M31-003 changedno `ORDER BY`), and it is why the table above compares sets — an ordered-row diff of this builderreports a difference that is not a change.++### M34-002 — the rest of fix 8++- **`page_titles` in `event.pages` / `event.previousPages`**: done. It uses the same+ `created_at BETWEEN toDateTime(..) AND toDateTime(..)` spelling. The proof is in+ `overview/src/pages.sql.proof.md`, section "M34-002".+- **The REST retention series** (`/insights/:projectId/retention`, `/engagement`): done, after the+ task was re-scoped to `chart` and `export`. `retentionSeriesQuery` and+ `retentionLastSeenSeriesQuery` use the same `created_at BETWEEN toDateTime(..) AND toDateTime(..)`+ spelling. Both routes now accept `range` / `startDate` / `endDate`, and the default is `3m`, the+ same as here. The default is applied on the routes only: the MCP tools that call the same+ service without dates still read all time. The proof is in `chart/src/retention.sql.proof.md`, section "M34-002".