OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.
fix: improve retention
Carl-Gerhard Lindesvärdbc406a81926d
diff --git a/apps/api/src/agents/tools/base.ts b/apps/api/src/agents/tools/base.tsindex 515009855..576ae2970 100644--- a/apps/api/src/agents/tools/base.ts+++ b/apps/api/src/agents/tools/base.ts@@ -711,7 +711,7 @@ export const getRetentionCohort = chatTool({name: 'get_retention_cohort',description:- 'Weekly user retention cohort table. Each row is a cohort week, columns are retention percentages for subsequent weeks.',+ 'Weekly active-user retention cohort for the last 12 weeks. Each row is a cohort (the week users were first seen) with cohort size (sum), retained user counts (values) and retained share (percentages) for subsequent weeks; a leading weighted-average row summarises all cohorts.',schema: z.object({}),},async (_input, context) => getRetentionCohortCore(context.projectId),diff --git a/apps/start/src/components/report-chart/retention/index.tsx b/apps/start/src/components/report-chart/retention/index.tsxindex ba1fc0c8e..ad0ca583e 100644--- a/apps/start/src/components/report-chart/retention/index.tsx+++ b/apps/start/src/components/report-chart/retention/index.tsx@@ -14,6 +14,11 @@ export function ReportRetentionChart() {const eventSeries = report.series.filter((item) => item.type === 'event');const firstEvent = (eventSeries[0]?.filters?.[0]?.value ?? []).map(String);const secondEvent = (eventSeries[1]?.filters?.[0]?.value ?? []).map(String);+ // filters[0] on each retention series is the event-name selector; everything+ // else is an audience filter (property or cohort) applied to the cohort.+ const filters = eventSeries.flatMap((item) =>+ (item.filters ?? []).filter((filter) => filter.name !== 'name'),+ );const isEnabled =firstEvent.length > 0 && secondEvent.length > 0 && !isLazyLoading;@@ -26,6 +31,7 @@ export function ReportRetentionChart() {{firstEvent,secondEvent,+ filters,projectId: report.projectId,range: report.range,startDate: report.startDate,diff --git a/apps/start/src/components/report/sidebar/ReportSeries.tsx b/apps/start/src/components/report/sidebar/ReportSeries.tsxindex 6f29306a4..bd46b070b 100644--- a/apps/start/src/components/report/sidebar/ReportSeries.tsx+++ b/apps/start/src/components/report/sidebar/ReportSeries.tsx@@ -125,7 +125,9 @@ export function ReportSeries() {});const showSegment = !['retention', 'funnel', 'sankey'].includes(chartType);- const showAddFilter = !['retention', 'sankey'].includes(chartType);+ // Retention supports property + cohort filters (hybrid cohort_events_mv /+ // raw-events engine), so the filter UI is enabled for it.+ const showAddFilter = !['sankey'].includes(chartType);const showDisplayNameInput = !['retention', 'sankey'].includes(chartType);const options = useSelector((state) => state.report.options);const isSankey = chartType === 'sankey';diff --git a/apps/start/src/components/report/sidebar/ReportSeriesItem.tsx b/apps/start/src/components/report/sidebar/ReportSeriesItem.tsxindex 5d51f4394..57bbf5584 100644--- a/apps/start/src/components/report/sidebar/ReportSeriesItem.tsx+++ b/apps/start/src/components/report/sidebar/ReportSeriesItem.tsx@@ -147,8 +147,11 @@ export function ReportSeriesItem({</div>)}- {/* Filters - only for events */}- {chartEvent && !isSelectManyEvents && <FiltersList event={chartEvent} />}+ {/* Filters list. For multi-event series (retention) the first filter is+ the event-name selector, so hide it and show only added filters. */}+ {chartEvent && (+ <FiltersList event={chartEvent} skipNameFilter={isSelectManyEvents} />+ )}</div>);}diff --git a/apps/start/src/components/report/sidebar/filters/FiltersList.tsx b/apps/start/src/components/report/sidebar/filters/FiltersList.tsxindex 8a5424d02..b5308aacb 100644--- a/apps/start/src/components/report/sidebar/filters/FiltersList.tsx+++ b/apps/start/src/components/report/sidebar/filters/FiltersList.tsx@@ -5,13 +5,30 @@ import { FilterItem } from './FilterItem';interface ReportEventFiltersProps {event: IChartEvent;+ /**+ * Skip the synthetic `name` filter (the event-name selector used by+ * multi-event series like retention) so only real property/cohort filters+ * are listed. Items still update the full event.filters by id.+ */+ skipNameFilter?: boolean;}-export function FiltersList({ event }: ReportEventFiltersProps) {+export function FiltersList({+ event,+ skipNameFilter = false,+}: ReportEventFiltersProps) {+ const filters = skipNameFilter+ ? event.filters.filter((filter) => filter.name !== 'name')+ : event.filters;++ if (filters.length === 0) {+ return null;+ }+return (<div><div className="bg-def-100 flex flex-col divide-y overflow-hidden rounded-b-md">- {event.filters.map((filter) => {+ {filters.map((filter) => {const isCohortFilter =filter.operator === 'inCohort' ||filter.operator === 'notInCohort';diff --git a/packages/db/src/services/retention.service.test.ts b/packages/db/src/services/retention.service.test.tsnew file mode 100644index 000000000..926967b50--- /dev/null+++ b/packages/db/src/services/retention.service.test.ts@@ -0,0 +1,279 @@+import { round } from '@openpanel/common';+import { afterAll, beforeAll, describe, expect, it } from 'vitest';++import {+ RETENTION_BLUEPRINT,+ RETENTION_FIXTURE,+ setupRetentionFixtures,+ teardownRetentionFixtures,+} from '../../../../test/retention-fixtures';+import { TABLE_NAMES, chQuery } from '../clickhouse/client';+import {+ getRetentionCohort,+ getRetentionSeries,+ type IRetentionCohortRow,+ processCohortData,+} from './retention.service';++const PROJECT_ID = 'test-retention-cohort';+const { day, week } = RETENTION_FIXTURE;++// Keep only the per-cohort rows (drop the leading "Weighted Average" row) and+// the fields the blueprint pins down.+function cohortsOnly(rows: IRetentionCohortRow[]) {+ return rows+ .filter((row) => row.cohort_interval !== 'Weighted Average')+ .map(({ cohort_interval, sum, values }) => ({+ cohort_interval,+ sum,+ values,+ }));+}++describe('getRetentionCohort', () => {+ beforeAll(async () => {+ await setupRetentionFixtures(PROJECT_ID);+ });++ afterAll(async () => {+ await teardownRetentionFixtures(PROJECT_ID);+ });++ it('matches the blueprint for day interval, criteria "on"', async () => {+ const rows = await getRetentionCohort({+ projectId: PROJECT_ID,+ firstEvent: ['app_open'],+ secondEvent: ['app_open'],+ criteria: 'on',+ interval: 'day',+ startDate: day.start,+ endDate: day.end,+ });++ expect(cohortsOnly(rows)).toEqual(RETENTION_BLUEPRINT.dayOn);+ });++ it('matches the blueprint for day interval, criteria "on_or_after"', async () => {+ const rows = await getRetentionCohort({+ projectId: PROJECT_ID,+ firstEvent: ['app_open'],+ secondEvent: ['app_open'],+ criteria: 'on_or_after',+ interval: 'day',+ startDate: day.start,+ endDate: day.end,+ });++ expect(cohortsOnly(rows)).toEqual(RETENTION_BLUEPRINT.dayOnOrAfter);+ });++ it('counts retention across the year boundary (week interval)', async () => {+ // The old toWeek() implementation returned week-of-year (resets each Jan),+ // so a 2024-W52 -> 2025-W01 return produced a negative diff and was dropped.+ // This locks the year-aware behavior.+ const rows = await getRetentionCohort({+ projectId: PROJECT_ID,+ firstEvent: ['app_open'],+ secondEvent: ['app_open'],+ criteria: 'on',+ interval: 'week',+ startDate: week.start,+ endDate: week.end,+ });++ expect(cohortsOnly(rows)).toEqual(RETENTION_BLUEPRINT.weekOn);+ });++ it('prepends a weighted-average row that is 100% retained at interval 0', async () => {+ const rows = await getRetentionCohort({+ projectId: PROJECT_ID,+ firstEvent: ['app_open'],+ secondEvent: ['app_open'],+ criteria: 'on',+ interval: 'day',+ startDate: day.start,+ endDate: day.end,+ });++ expect(rows[0]?.cohort_interval).toBe('Weighted Average');+ expect(rows[0]?.percentages[0]).toBe(1);+ });++ it('assigns each user to a single first-touch cohort (no double-counting)', async () => {+ const rows = cohortsOnly(+ await getRetentionCohort({+ projectId: PROJECT_ID,+ firstEvent: ['app_open'],+ secondEvent: ['app_open'],+ criteria: 'on',+ interval: 'day',+ startDate: day.start,+ endDate: day.end,+ })+ );++ // Only D0 and D1 are real cohorts. There is NO D2 cohort: every user who+ // fired app_open on D2 (RU1, RU2, RU4) already belongs to an earlier cohort.+ expect(rows.map((row) => row.cohort_interval)).toEqual([day.d0, day.d1]);++ // Characterize the bug we fixed: the legacy "every-occurrence" cohorting+ // (no GROUP BY profile_id dedup) places recurring users in every day they+ // were active, producing a spurious D2 cohort and inflated sizes.+ const legacy = await chQuery<{ cohort_interval: string; size: number }>(`+ SELECT toDate(created_at) AS cohort_interval, COUNT(DISTINCT profile_id) AS size+ FROM ${TABLE_NAMES.cohort_events_mv}+ WHERE project_id = '${PROJECT_ID}'+ AND name = 'app_open'+ AND created_at BETWEEN toDate('${day.start}') AND toDate('${day.end}')+ GROUP BY cohort_interval+ ORDER BY cohort_interval ASC+ `);++ // 3 day-buckets, each with 3 distinct users => sizes sum to 9 for only 5+ // real users. First-touch collapses this to 3 + 2 = 5.+ expect(legacy.map((r) => r.size)).toEqual([3, 3, 3]);+ const firstTouchTotal = rows.reduce((acc, row) => acc + row.sum, 0);+ expect(firstTouchTotal).toBe(5);+ });++ it('supports any-event (active-user) retention when no event names given', async () => {+ const named = await getRetentionCohort({+ projectId: PROJECT_ID,+ firstEvent: ['app_open'],+ secondEvent: ['app_open'],+ criteria: 'on',+ interval: 'day',+ startDate: day.start,+ endDate: day.end,+ });+ const anyEvent = await getRetentionCohort({+ projectId: PROJECT_ID,+ criteria: 'on',+ interval: 'day',+ startDate: day.start,+ endDate: day.end,+ });++ // In the day window the only non-app_open events are RU1's purchase (D1)+ // and RU2's purchase (D2). Those users are already active via app_open on+ // those days, so "any event" yields the same cohort sizes as app_open.+ expect(cohortsOnly(anyEvent).map((r) => r.sum)).toEqual(+ cohortsOnly(named).map((r) => r.sum)+ );+ });++ it('applies a property filter via the raw-events fallback path', async () => {+ // country = US drops RU3 (SE) from the D0 cohort. This filter references an+ // event column not present in cohort_events_mv, so the engine must fall+ // back to the raw events table.+ const rows = await getRetentionCohort({+ projectId: PROJECT_ID,+ firstEvent: ['app_open'],+ secondEvent: ['app_open'],+ criteria: 'on',+ interval: 'day',+ startDate: day.start,+ endDate: day.end,+ filters: [{ name: 'country', operator: 'is', value: ['US'] }],+ });++ expect(cohortsOnly(rows)).toEqual(RETENTION_BLUEPRINT.countryUsOn);+ });++ it('scopes to a saved cohort via the fast MV path (inCohort)', async () => {+ // inCohort only needs profile_id, so this stays on cohort_events_mv. The+ // cohort contains RU1 and RU4, leaving one user in each of D0 and D1.+ const rows = await getRetentionCohort({+ projectId: PROJECT_ID,+ firstEvent: ['app_open'],+ secondEvent: ['app_open'],+ criteria: 'on',+ interval: 'day',+ startDate: day.start,+ endDate: day.end,+ filters: [+ {+ name: 'cohort',+ operator: 'inCohort',+ value: [],+ cohortIds: [RETENTION_FIXTURE.cohort.id],+ },+ ],+ });++ expect(cohortsOnly(rows)).toEqual(RETENTION_BLUEPRINT.cohortOn);+ });+});++describe('getRetentionSeries', () => {+ beforeAll(async () => {+ await setupRetentionFixtures(PROJECT_ID);+ });++ afterAll(async () => {+ await teardownRetentionFixtures(PROJECT_ID);+ });++ it('computes week-over-week active-user retention', async () => {+ const rows = await getRetentionSeries({ projectId: PROJECT_ID });+ expect(rows).toEqual(RETENTION_BLUEPRINT.weeklySeries);+ });+});++describe('processCohortData weighted average (pure)', () => {+ // Three weekly cohorts; reference date is 3 weeks after the oldest, so the+ // newer cohorts have not yet reached the later periods.+ const data = [+ {+ cohort_interval: '2024-01-07',+ total_first_event_count: 100,+ interval_0_user_count: 100,+ interval_1_user_count: 50,+ interval_2_user_count: 25,+ interval_3_user_count: 10,+ },+ {+ cohort_interval: '2024-01-14',+ total_first_event_count: 80,+ interval_0_user_count: 80,+ interval_1_user_count: 40,+ interval_2_user_count: 0, // genuine zero (mature) — must count+ interval_3_user_count: 0,+ },+ {+ cohort_interval: '2024-01-21',+ total_first_event_count: 60,+ interval_0_user_count: 60,+ interval_1_user_count: 30,+ interval_2_user_count: 0, // not yet mature — must be excluded+ interval_3_user_count: 0,+ },+ ];++ it('pools only mature cohorts and is internally consistent', () => {+ const rows = processCohortData(data, 3, 'week', '2024-01-28 00:00:00');+ const avg = rows[0]!;++ expect(avg.cohort_interval).toBe('Weighted Average');+ // Period 0 equals Total profiles and the curve starts at 100%.+ expect(avg.sum).toBe(80);+ expect(avg.values[0]).toBe(80);+ expect(avg.percentages[0]).toBe(1);+ // Period 2 pools C0+C1 only (C2 is immature). C1's genuine zero is included,+ // so it's 25/180 = 0.14 — NOT 25/100 = 0.25 (the old "exclude zeros" bug).+ expect(avg.percentages[2]).toBe(0.14);+ // Period 3 pools only C0 (C1, C2 immature): 10/100 = 0.10.+ expect(avg.percentages[3]).toBe(0.1);++ expect(avg.values).toEqual([80, 40, 11, 8]);+ expect(avg.percentages).toEqual([1, 0.5, 0.14, 0.1]);+ });++ it('treats all periods as mature when no reference date is given', () => {+ const rows = processCohortData(data, 3, 'week');+ const avg = rows[0]!;+ // Every cohort counts in every column: period 2 = 25/240, period 3 = 10/240.+ expect(avg.percentages[2]).toBe(round(25 / 240, 2));+ expect(avg.percentages[3]).toBe(round(10 / 240, 2));+ });+});diff --git a/packages/db/src/services/retention.service.ts b/packages/db/src/services/retention.service.tsindex b337ee8fe..5def2486b 100644--- a/packages/db/src/services/retention.service.ts+++ b/packages/db/src/services/retention.service.ts@@ -1,94 +1,47 @@+import { DateTime, round } from '@openpanel/common';+import type { IChartEventFilter } from '@openpanel/validation';+import { range } from 'ramda';import sqlstring from 'sqlstring';import { TABLE_NAMES, chQuery } from '../clickhouse/client';+import { getEventFiltersWhereClause } from './chart.service';type IGetWeekRetentionInput = {projectId: string;};-// https://www.geeksforgeeks.org/how-to-calculate-retention-rate-in-sql/-export function getRetentionCohortTable({ projectId }: IGetWeekRetentionInput) {+// Week-over-week retention graph: for each week, how many active users were+// also active the following week.+//+// Instead of self-joining the raw events table (O(events²) per profile), we+// first collapse events to one row per (profile, week) in `weekly_active`,+// then self-join that much smaller set to its next week. Same result, a+// fraction of the work on high-volume projects.+export function getRetentionSeries({ projectId }: IGetWeekRetentionInput) {const sql = `-WITH- m AS- (- SELECT- profile_id,- max(toWeek(created_at)) AS last_seen- FROM ${TABLE_NAMES.events}- WHERE (project_id = ${sqlstring.escape(projectId)}) AND (profile_id != device_id)- GROUP BY profile_id- ),- n AS- (+ WITH weekly_active AS (SELECT- profile_id,- min(toWeek(created_at)) AS first_seen+ profile_id,+ toStartOfWeek(created_at) AS weekFROM ${TABLE_NAMES.events}- WHERE (project_id = ${sqlstring.escape(projectId)}) AND (profile_id != device_id)- GROUP BY profile_id- ),- a AS- (- SELECT- m.profile_id,- m.last_seen,- n.first_seen,- m.last_seen - n.first_seen AS diff- FROM m, n- WHERE m.profile_id = n.profile_id- )-SELECT- first_seen,- SUM(multiIf(diff = 0, 1, 0)) AS period_0,- SUM(multiIf(diff = 1, 1, 0)) AS period_1,- SUM(multiIf(diff = 2, 1, 0)) AS period_2,- SUM(multiIf(diff = 3, 1, 0)) AS period_3,- SUM(multiIf(diff = 4, 1, 0)) AS period_4,- SUM(multiIf(diff = 5, 1, 0)) AS period_5,- SUM(multiIf(diff = 6, 1, 0)) AS period_6,- SUM(multiIf(diff = 7, 1, 0)) AS period_7,- SUM(multiIf(diff = 8, 1, 0)) AS period_8,- SUM(multiIf(diff = 9, 1, 0)) AS period_9-FROM a-GROUP BY first_seen-ORDER BY first_seen ASC- `;-- return chQuery<{- first_seen: number;- period_0: number;- period_1: number;- period_2: number;- period_3: number;- period_4: number;- period_5: number;- period_6: number;- period_7: number;- period_8: number;- period_9: number;- }>(sql);-}--// Retention graph-// https://www.sisense.com/blog/how-to-calculate-cohort-retention-in-sql/-export function getRetentionSeries({ projectId }: IGetWeekRetentionInput) {- const sql = `+ WHERE project_id = ${sqlstring.escape(projectId)}+ AND profile_id != device_id+ GROUP BY profile_id, week+ )SELECT- toStartOfWeek(events.created_at) AS date,- countDistinct(events.profile_id) AS active_users,- countDistinct(future_events.profile_id) AS retained_users,- (100 * (countDistinct(future_events.profile_id) / CAST(countDistinct(events.profile_id), 'float'))) AS retention- FROM ${TABLE_NAMES.events} as events- LEFT JOIN ${TABLE_NAMES.events} AS future_events ON- events.profile_id = future_events.profile_id- AND toStartOfWeek(events.created_at) = toStartOfWeek(future_events.created_at - toIntervalWeek(1))- AND future_events.profile_id != future_events.device_id- WHERE- project_id = ${sqlstring.escape(projectId)}- AND events.profile_id != events.device_id- GROUP BY 1- ORDER BY date ASC`;+ cur.week AS date,+ countDistinct(cur.profile_id) AS active_users,+ countDistinct(nxt.profile_id) AS retained_users,+ (100 * (countDistinct(nxt.profile_id) / CAST(countDistinct(cur.profile_id), 'Float64'))) AS retention+ FROM weekly_active AS cur+ LEFT JOIN weekly_active AS nxt+ ON cur.profile_id = nxt.profile_id+ AND nxt.week = cur.week + toIntervalWeek(1)+ GROUP BY date+ ORDER BY date ASC+ -- Unmatched LEFT JOIN rows must be NULL (not the empty-string default),+ -- otherwise countDistinct(nxt.profile_id) counts '' as a retained user.+ SETTINGS join_use_nulls = 1`;return chQuery<{date: string;@@ -180,8 +133,19 @@ export async function getWeeklyRetentionSeriesCore(projectId: string) {return getRetentionSeries({ projectId });}+// Weekly active-user retention cohort over the last 12 weeks, computed by the+// unified getRetentionCohort engine (replaces the legacy, broken+// getRetentionCohortTable). firstEvent/secondEvent are omitted so any+// identified activity counts toward the cohort.export async function getRetentionCohortCore(projectId: string) {- return getRetentionCohortTable({ projectId });+ const end = DateTime.now();+ const start = end.minus({ weeks: 12 });+ return getRetentionCohort({+ projectId,+ interval: 'week',+ startDate: start.toFormat('yyyy-MM-dd HH:mm:ss'),+ endDate: end.toFormat('yyyy-MM-dd HH:mm:ss'),+ });}export async function getEngagementCore(projectId: string) {@@ -216,3 +180,325 @@ export async function getEngagementCore(projectId: string) {distribution: raw,};}++// ---------------------------------------------------------------------------+// Cohort retention matrix+//+// This is the single source of truth for retention cohorts. It powers the+// dashboard retention chart (via the tRPC `cohort` procedure) as well as the+// MCP / agent / REST retention endpoints.+//+// Definition: every user is assigned to exactly ONE cohort, the interval of+// their FIRST `firstEvent` within the window (first-touch). For each cohort we+// count how many of those users performed `secondEvent` 0..N intervals later.+// - criteria 'on' -> active exactly k intervals after (=)+// - criteria 'on_or_after' -> active at least k intervals after (>=, cumulative)+// When `firstEvent` / `secondEvent` is empty the name filter is dropped and the+// query measures retention on ANY activity (active-user retention).+// ---------------------------------------------------------------------------++export type IRetentionInterval = 'minute' | 'hour' | 'day' | 'week' | 'month';+export type IRetentionCriteria = 'on' | 'on_or_after';++export type IGetRetentionCohortInput = {+ projectId: string;+ /** Event name(s) defining cohort entry. Empty/undefined => any event. */+ firstEvent?: string[];+ /** Event name(s) that count as "returned". Empty/undefined => any event. */+ secondEvent?: string[];+ criteria?: IRetentionCriteria;+ interval?: IRetentionInterval;+ /** ISO or `yyyy-MM-dd HH:mm:ss`. */+ startDate: string;+ /** ISO or `yyyy-MM-dd HH:mm:ss`. */+ endDate: string;+ /**+ * Property and/or cohort filters scoping the analysed events. Cohort+ * membership (inCohort/notInCohort) keeps the fast cohort_events_mv path;+ * any property/column filter falls back to the raw events table.+ */+ filters?: IChartEventFilter[];+};++export type IRetentionCohortRow = {+ cohort_interval: string;+ sum: number;+ values: number[];+ percentages: number[];+};++const SQL_START_OF: Record<IRetentionInterval, string> = {+ minute: 'toDate',+ hour: 'toDate',+ day: 'toDate',+ week: 'toStartOfWeek',+ month: 'toStartOfMonth',+};++const SQL_INTERVAL: Record<IRetentionInterval, string> = {+ minute: 'DAY',+ hour: 'DAY',+ day: 'DAY',+ week: 'WEEK',+ month: 'MONTH',+};++const LUXON_UNIT: Record<IRetentionInterval, 'days' | 'weeks' | 'months'> = {+ minute: 'days',+ hour: 'days',+ day: 'days',+ week: 'weeks',+ month: 'months',+};++// Normalize an ISO or `yyyy-MM-dd HH:mm:ss` string into ClickHouse date-time form.+function utc(date: string) {+ return date.replace('T', ' ').slice(0, 19);+}++// Number of `interval` buckets spanned by [startDate, endDate]; drives the+// number of retention columns (0..diffInterval).+function diffIntervalCount(+ startDate: string,+ endDate: string,+ interval: IRetentionInterval+) {+ const unit = LUXON_UNIT[interval];+ const start = DateTime.fromFormat(utc(startDate), 'yyyy-MM-dd HH:mm:ss', {+ zone: 'utc',+ });+ const end = DateTime.fromFormat(utc(endDate), 'yyyy-MM-dd HH:mm:ss', {+ zone: 'utc',+ });+ return Math.max(0, Math.floor(end.diff(start, unit).as(unit)));+}++// Build the `name = ... / name IN (...)` predicate; null means "any event".+function eventNameWhere(events: string[] | undefined): string | null {+ if (!events || events.length === 0) {+ return null;+ }+ if (events.length === 1) {+ return `name = ${sqlstring.escape(events[0])}`;+ }+ return `name IN (${events.map((event) => sqlstring.escape(event)).join(', ')})`;+}++export async function getRetentionCohort(input: IGetRetentionCohortInput) {+ const {+ projectId,+ firstEvent,+ secondEvent,+ criteria = 'on_or_after',+ interval = 'day',+ startDate,+ endDate,+ filters = [],+ } = input;++ const diffInterval = diffIntervalCount(startDate, endDate, interval);+ const sqlInterval = SQL_INTERVAL[interval];+ const sqlToStartOf = SQL_START_OF[interval];+ const countCriteria: '>=' | '=' = criteria === 'on_or_after' ? '>=' : '=';++ const start = utc(startDate);+ const end = utc(endDate);+ const escProject = sqlstring.escape(projectId);+ const firstWhere = eventNameWhere(firstEvent);+ const secondWhere = eventNameWhere(secondEvent);++ // Hybrid source: cohort_events_mv (skinny, fast) carries only+ // project_id/name/created_at/profile_id, so any filter referencing event+ // properties or columns forces a fallback to the raw events table. Cohort+ // membership filters only need profile_id, so they stay on the fast path.+ const needRawEvents = filters.some(+ (filter) =>+ filter.operator !== 'inCohort' && filter.operator !== 'notInCohort'+ );+ const source = needRawEvents ? TABLE_NAMES.events : TABLE_NAMES.cohort_events_mv;++ const baseConditions = [`project_id = ${escProject}`];+ if (needRawEvents) {+ // cohort_events_mv only stores identified-user rows; replicate that here.+ baseConditions.push('profile_id != device_id');+ }+ if (filters.length > 0) {+ baseConditions.push(+ ...Object.values(getEventFiltersWhereClause(filters, projectId))+ );+ }+ const baseWhere = baseConditions.join('\n AND ');++ const columns = range(0, diffInterval + 1);+ const usersSelect = columns+ .map(+ (index) =>+ `groupUniqArrayIf(profile_id, x_after_cohort ${countCriteria} ${index}) AS interval_${index}_users`+ )+ .join(',\n ');+ const countsSelect = columns+ .map(+ (index) =>+ `length(interval_${index}_users) AS interval_${index}_user_count`+ )+ .join(',\n ');++ const cohortQuery = `+ WITH+ cohort_users AS (+ SELECT+ profile_id AS userID,+ ${sqlToStartOf}(min(created_at)) AS cohort_interval+ FROM ${source}+ WHERE ${baseWhere}+ ${firstWhere ? `AND ${firstWhere}` : ''}+ AND created_at >= toDateTime('${start}')+ AND created_at <= toDateTime('${end}')+ GROUP BY profile_id+ ),+ last_event AS (+ SELECT+ profile_id,+ toDate(created_at) AS event_date+ FROM ${source}+ WHERE ${baseWhere}+ ${secondWhere ? `AND ${secondWhere}` : ''}+ AND created_at >= toDateTime('${start}')+ AND created_at <= toDateTime('${end}') + INTERVAL ${diffInterval} ${sqlInterval}+ ),+ retention_matrix AS (+ SELECT+ f.cohort_interval,+ l.profile_id,+ dateDiff('${sqlInterval}', f.cohort_interval, ${sqlToStartOf}(l.event_date)) AS x_after_cohort+ FROM cohort_users AS f+ INNER JOIN last_event AS l ON f.userID = l.profile_id+ WHERE l.event_date >= f.cohort_interval+ AND dateDiff('${sqlInterval}', f.cohort_interval, ${sqlToStartOf}(l.event_date)) <= ${diffInterval}+ ),+ interval_users AS (+ SELECT+ cohort_interval,+ ${usersSelect}+ FROM retention_matrix+ GROUP BY cohort_interval+ ),+ cohort_sizes AS (+ SELECT+ cohort_interval,+ COUNT(DISTINCT userID) AS total_first_event_count+ FROM cohort_users+ GROUP BY cohort_interval+ )+ SELECT+ interval_users.cohort_interval AS cohort_interval,+ cs.total_first_event_count AS total_first_event_count,+ ${countsSelect}+ FROM interval_users+ LEFT JOIN cohort_sizes AS cs ON interval_users.cohort_interval = cs.cohort_interval+ ORDER BY cohort_interval ASC+ `;++ const cohortData = await chQuery<{+ cohort_interval: string;+ total_first_event_count: number;+ [key: string]: number | string;+ }>(cohortQuery);++ // Reference point for cohort maturity: we only have return data up to "now",+ // so periods that haven't elapsed yet are excluded from the weighted average.+ const until = DateTime.utc().toFormat('yyyy-MM-dd HH:mm:ss');+ return processCohortData(cohortData, diffInterval, interval, until);+}++// Number of fully-elapsed periods between a cohort's start and the reference+// date. Periods beyond this haven't happened yet, so they carry no return data+// and must be excluded from the average. Fails open (treats everything as+// mature) when dates are missing/unparseable.+function maturePeriodCount(+ cohortInterval: string,+ until: string | undefined,+ interval: IRetentionInterval+): number {+ if (!until) {+ return Number.POSITIVE_INFINITY;+ }+ const unit = LUXON_UNIT[interval];+ const cohort = DateTime.fromFormat(cohortInterval.slice(0, 10), 'yyyy-MM-dd', {+ zone: 'utc',+ });+ const ref = DateTime.fromFormat(until, 'yyyy-MM-dd HH:mm:ss', { zone: 'utc' });+ if (!cohort.isValid || !ref.isValid) {+ return Number.POSITIVE_INFINITY;+ }+ return Math.floor(ref.diff(cohort, unit).as(unit));+}++// Shapes the raw ClickHouse matrix into per-cohort rows + a leading weighted-+// average row.+//+// The average is a maturity-aware pooled rate: for each period column it pools+// only the cohorts old enough to have actually reached that period+// (cohort_interval + N intervals <= `until`). This keeps not-yet-elapsed cells+// from being read as churn AND keeps genuine zeros in, so the late curve is not+// inflated. Counts are normalised to the representative (mean) cohort size, so+// the row is internally consistent: period 0 equals Total profiles and the+// curve starts at 100%.+export function processCohortData(+ data: Array<{+ cohort_interval: string;+ total_first_event_count: number;+ [key: string]: number | string;+ }>,+ diffInterval: number,+ interval: IRetentionInterval = 'day',+ until?: string+): IRetentionCohortRow[] {+ if (data.length === 0) {+ return [];+ }++ const columns = range(0, diffInterval + 1);+ const processed = data.map((row) => {+ const sum = row.total_first_event_count;+ const values = columns.map(+ (index) => (row[`interval_${index}_user_count`] || 0) as number+ );++ return {+ cohort_interval: row.cohort_interval,+ sum,+ values,+ percentages: values.map((value) => (sum > 0 ? round(value / sum, 2) : 0)),+ };+ });++ const maturePeriods = processed.map((row) =>+ maturePeriodCount(row.cohort_interval, until, interval)+ );+ const totalSize = processed.reduce((acc, row) => acc + row.sum, 0);+ const representativeSize = round(totalSize / processed.length, 0);++ const averageRow: IRetentionCohortRow = {+ cohort_interval: 'Weighted Average',+ sum: representativeSize,+ values: [],+ percentages: [],+ };++ for (const index of columns) {+ let matureSize = 0;+ let retained = 0;+ processed.forEach((row, i) => {+ if (index <= maturePeriods[i]!) {+ matureSize += row.sum;+ retained += row.values[index]!;+ }+ });+ const rate = matureSize > 0 ? retained / matureSize : 0;+ averageRow.percentages.push(round(rate, 2));+ averageRow.values.push(round(rate * representativeSize, 0));+ }++ return [averageRow, ...processed];+}diff --git a/packages/mcp/src/integration/tools.test.ts b/packages/mcp/src/integration/tools.test.tsindex 337a5dbf2..635a6c9b7 100644--- a/packages/mcp/src/integration/tools.test.ts+++ b/packages/mcp/src/integration/tools.test.ts@@ -660,7 +660,7 @@ describe('get_weekly_retention_series', () => {});describe('get_retention_cohort', () => {- it('returns array of cohort rows with period_0..period_9', async () => {+ it('returns weighted-average + per-cohort matrix rows', async () => {const server = makeServer();registerRetentionTools(server as any, CTX);const res = await server.invoke('get_retention_cohort', {@@ -668,8 +668,12 @@ describe('get_retention_cohort', () => {});expect(Array.isArray(res)).toBe(true);if (res.length > 0) {- expect(res[0]).toHaveProperty('first_seen');- expect(res[0]).toHaveProperty('period_0');+ expect(res[0]).toHaveProperty('cohort_interval');+ expect(res[0]).toHaveProperty('sum');+ expect(res[0]).toHaveProperty('values');+ expect(res[0]).toHaveProperty('percentages');+ // The first row is always the weighted-average summary.+ expect(res[0].cohort_interval).toBe('Weighted Average');}});});diff --git a/packages/mcp/src/tools/analytics/retention.ts b/packages/mcp/src/tools/analytics/retention.tsindex e831bab05..ac0627178 100644--- a/packages/mcp/src/tools/analytics/retention.ts+++ b/packages/mcp/src/tools/analytics/retention.ts@@ -1,4 +1,4 @@-import { getRetentionCohortTable } from '@openpanel/db';+import { getRetentionCohortCore } from '@openpanel/db';import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';import type { McpAuthContext } from '../../auth';import {@@ -14,14 +14,14 @@ export function registerRetentionTools() {server.tool('get_retention_cohort',- 'Get a weekly user retention cohort table. Shows what percentage of users who first visited in a given week returned in subsequent weeks. Useful for understanding long-term user engagement and product stickiness.',+ 'Get a weekly active-user retention cohort for the last 12 weeks. Returns one row per cohort (the week users were first seen), each with `cohort_interval`, `sum` (cohort size), `values` (retained user counts per following week) and `percentages` (retained share, 0-1). The leading "Weighted Average" row summarises all cohorts. Useful for understanding long-term user engagement and product stickiness.',{projectId: projectIdSchema(context),},async ({ projectId: inputProjectId }) =>withErrorHandling(async () => {const projectId = await resolveProjectId(context, inputProjectId);- return getRetentionCohortTable({ projectId });+ return getRetentionCohortCore(projectId);}),);}diff --git a/packages/trpc/src/routers/chart.ts b/packages/trpc/src/routers/chart.tsindex 70219b7c9..f5ee87e28 100644--- a/packages/trpc/src/routers/chart.ts+++ b/packages/trpc/src/routers/chart.ts@@ -1,4 +1,3 @@-import { round } from '@openpanel/common';import {AggregateChartEngine,ChartEngine,@@ -17,6 +16,7 @@ import {getProfilePropertySelect,getProfilesCached,getReportById,+ getRetentionCohort,getSelectPropertyKey,getSettingsForProject,type IServiceProfile,@@ -29,19 +29,14 @@ import {} from '@openpanel/db';import {type IChartEvent,+ zChartEventFilter,zChartSeries,zCriteria,zRange,zReportInput,zTimeInterval,} from '@openpanel/validation';-import {- differenceInDays,- differenceInMonths,- differenceInWeeks,- formatISO,-} from 'date-fns';-import { flatten, map, pipe, prop, range, sort, uniq } from 'ramda';+import { flatten, map, pipe, prop, sort, uniq } from 'ramda';import sqlstring from 'sqlstring';import { z } from 'zod';import { getProjectAccess } from '../access';@@ -53,13 +48,6 @@ import {publicProcedure,} from '../trpc';-function utc(date: string | Date) {- if (typeof date === 'string') {- return date.replace('T', ' ').slice(0, 19);- }- return formatISO(date).replace('T', ' ').slice(0, 19);-}-const cacher = cacheMiddleware(60);const chartProcedure = publicProcedure.use(@@ -624,6 +612,7 @@ export const chartRouter = createTRPCRouter({endDate: z.string().nullish(),interval: zTimeInterval.default('day'),range: zRange,+ filters: z.array(zChartEventFilter).optional(),shareId: z.string().optional(),id: z.string().optional(),})@@ -633,6 +622,10 @@ export const chartRouter = createTRPCRouter({let firstEvent = input.firstEvent;let secondEvent = input.secondEvent;let criteria = input.criteria;+ // Property/cohort filters scoping the retention audience. These live+ // alongside the event-name selector (filters[0]) on each retention+ // series; everything except that name selector is an audience filter.+ let filters = input.filters ?? [];const dateRange = ctx.report? (input.range ?? ctx.report.range): input.range;@@ -671,6 +664,9 @@ export const chartRouter = createTRPCRouter({firstEvent = extractedFirstEvent;secondEvent = extractedSecondEvent;+ filters = eventSeries.flatMap((serie) =>+ (serie.filters ?? []).filter((filter) => filter.name !== 'name')+ );}const { timezone } = await getSettingsForProject(projectId);@@ -682,116 +678,17 @@ export const chartRouter = createTRPCRouter({},timezone);- const diffInterval = {- minute: () => differenceInDays(dates.endDate, dates.startDate),- hour: () => differenceInDays(dates.endDate, dates.startDate),- day: () => differenceInDays(dates.endDate, dates.startDate),- week: () => differenceInWeeks(dates.endDate, dates.startDate),- month: () => differenceInMonths(dates.endDate, dates.startDate),- }[interval]();- const sqlInterval = {- minute: 'DAY',- hour: 'DAY',- day: 'DAY',- week: 'WEEK',- month: 'MONTH',- }[interval];-- const sqlToStartOf = {- minute: 'toDate',- hour: 'toDate',- day: 'toDate',- week: 'toStartOfWeek',- month: 'toStartOfMonth',- }[interval];-- const countCriteria = criteria === 'on_or_after' ? '>=' : '=';-- const usersSelect = range(0, diffInterval + 1)- .map(- (index) =>- `groupUniqArrayIf(profile_id, x_after_cohort ${countCriteria} ${index}) AS interval_${index}_users`- )- .join(',\n');-- const countsSelect = range(0, diffInterval + 1)- .map(- (index) =>- `length(interval_${index}_users) AS interval_${index}_user_count`- )- .join(',\n');-- const whereEventNameIs = (event: string[]) => {- if (event.length === 1) {- return `name = ${sqlstring.escape(event[0])}`;- }- return `name IN (${event.map((e) => sqlstring.escape(e)).join(',')})`;- };- const cohortQuery = `- WITH- cohort_users AS (- SELECT- profile_id AS userID,- project_id,- ${sqlToStartOf}(created_at) AS cohort_interval- FROM ${TABLE_NAMES.cohort_events_mv}- WHERE ${whereEventNameIs(firstEvent)}- AND project_id = ${sqlstring.escape(projectId)}- AND created_at BETWEEN toDate('${utc(dates.startDate)}') AND toDate('${utc(dates.endDate)}')- ),- last_event AS- (- SELECT- profile_id,- project_id,- toDate(created_at) AS event_date- FROM cohort_events_mv- WHERE ${whereEventNameIs(secondEvent)}- AND project_id = ${sqlstring.escape(projectId)}- AND created_at BETWEEN toDate('${utc(dates.startDate)}') AND toDate('${utc(dates.endDate)}') + INTERVAL ${diffInterval} ${sqlInterval}- ),- retention_matrix AS- (- SELECT- f.cohort_interval,- l.profile_id,- dateDiff('${sqlInterval}', f.cohort_interval, ${sqlToStartOf}(l.event_date)) AS x_after_cohort- FROM cohort_users AS f- INNER JOIN last_event AS l ON f.userID = l.profile_id- WHERE (l.event_date >= f.cohort_interval)- AND (l.event_date <= (f.cohort_interval + INTERVAL ${diffInterval} ${sqlInterval}))- ),- interval_users AS (- SELECT- cohort_interval,- ${usersSelect}- FROM retention_matrix- GROUP BY cohort_interval- ),- cohort_sizes AS (- SELECT- cohort_interval,- COUNT(DISTINCT userID) AS total_first_event_count- FROM cohort_users- GROUP BY cohort_interval- )- SELECT- cohort_interval,- cohort_sizes.total_first_event_count,- ${countsSelect}- FROM interval_users- LEFT JOIN cohort_sizes AS cs ON cohort_interval = cs.cohort_interval- ORDER BY cohort_interval ASC- `;-- const cohortData = await chQuery<{- cohort_interval: string;- total_first_event_count: number;- [key: string]: any;- }>(cohortQuery);-- return processCohortData(cohortData, diffInterval);+ return getRetentionCohort({+ projectId,+ firstEvent,+ secondEvent,+ criteria,+ interval,+ startDate: dates.startDate,+ endDate: dates.endDate,+ filters,+ });}),getProfiles: protectedProcedure@@ -1060,74 +957,3 @@ export const chartRouter = createTRPCRouter({return profiles;}),});--function processCohortData(- data: Array<{- cohort_interval: string;- total_first_event_count: number;- [key: string]: any;- }>,- diffInterval: number-) {- if (data.length === 0) {- return [];- }-- const processed = data.map((row) => {- const sum = row.total_first_event_count;- const values = range(0, diffInterval + 1).map(- (index) => (row[`interval_${index}_user_count`] || 0) as number- );-- return {- cohort_interval: row.cohort_interval,- sum,- values,- percentages: values.map((value) => (sum > 0 ? round(value / sum, 2) : 0)),- };- });-- const averageData: {- totalSum: number;- values: Array<{ sum: number; weightedSum: number }>;- percentages: Array<{ sum: number; weightedSum: number }>;- } = {- totalSum: 0,- values: range(0, diffInterval + 1).map(() => ({ sum: 0, weightedSum: 0 })),- percentages: range(0, diffInterval + 1).map(() => ({- sum: 0,- weightedSum: 0,- })),- };-- // Aggregate data for weighted averages, excluding zeros- processed.forEach((row) => {- averageData.totalSum += row.sum;- row.values.forEach((value, index) => {- if (value !== 0) {- averageData.values[index]!.sum += row.sum;- averageData.values[index]!.weightedSum += value * row.sum;- }- });- row.percentages.forEach((percentage, index) => {- if (percentage !== 0) {- averageData.percentages[index]!.sum += row.sum;- averageData.percentages[index]!.weightedSum += percentage * row.sum;- }- });- });-- // Calculate weighted average values, excluding zeros- const averageRow = {- cohort_interval: 'Weighted Average',- sum: round(averageData.totalSum / processed.length, 0),- percentages: averageData.percentages.map(({ sum, weightedSum }) =>- sum > 0 ? round(weightedSum / sum, 2) : 0- ),- values: averageData.values.map(({ sum, weightedSum }) =>- sum > 0 ? round(weightedSum / sum, 0) : 0- ),- };-- return [averageRow, ...processed];-}diff --git a/test/retention-fixtures.ts b/test/retention-fixtures.tsnew file mode 100644index 000000000..6579fb2ff--- /dev/null+++ b/test/retention-fixtures.ts@@ -0,0 +1,257 @@+/**+ * Deterministic ClickHouse fixture for retention-cohort tests.+ *+ * Unlike `test/fixtures.ts` (which anchors events to `new Date()`), this fixture+ * places every event on a FIXED absolute date so the expected cohort matrix is a+ * stable, hand-computed blueprint. It is the contract the retention engine must+ * satisfy.+ *+ * ---------------------------------------------------------------------------+ * Dataset (all events are name `app_open` unless noted; device_id != profile_id+ * so they survive the cohort_events_mv `profile_id != device_id` filter)+ * ---------------------------------------------------------------------------+ *+ * DAY scenario — window 2024-03-04 .. 2024-03-06 (D0, D1, D2)+ *+ * user app_open days purchase days first-touch cohort country+ * RU1 D0, D1, D2 D1 D0 US+ * RU2 D0, D2 D2 D0 US+ * RU3 D0 — D0 SE+ * RU4 D1, D2 — D1 US+ * RU5 D1 — D1 US+ *+ * => D0 cohort = {RU1,RU2,RU3} (size 3), D1 cohort = {RU4,RU5} (size 2)+ *+ * WEEK scenario — window 2024-12-29 .. 2025-01-06 (crosses the year boundary,+ * which is exactly what the old toWeek() implementation got wrong)+ *+ * WU1 app_open 2024-12-30 and 2025-01-06+ * WU2 app_open 2024-12-30 only+ *+ * => single cohort (week of 2024-12-29), size 2+ *+ * Use a per-suite projectId so suites can run concurrently.+ */++import { createClient } from '../packages/db/src/clickhouse/client';++// ---------------------------------------------------------------------------+// Well-known ids + absolute dates+// ---------------------------------------------------------------------------++export const RETENTION_FIXTURE = {+ users: {+ ru1: 'retention-ru1',+ ru2: 'retention-ru2',+ ru3: 'retention-ru3',+ ru4: 'retention-ru4',+ ru5: 'retention-ru5',+ wu1: 'retention-wu1',+ wu2: 'retention-wu2',+ },+ // DAY scenario boundaries (inclusive) used as query window+ day: {+ start: '2024-03-04 00:00:00',+ end: '2024-03-06 23:59:59',+ d0: '2024-03-04',+ d1: '2024-03-05',+ d2: '2024-03-06',+ },+ // WEEK scenario boundaries; cohort lands on the Sunday-aligned week start+ week: {+ start: '2024-12-29 00:00:00',+ end: '2025-01-06 23:59:59',+ cohort: '2024-12-29',+ },+ // Saved cohort used for the inCohort filter case (members: RU1, RU4)+ cohort: {+ id: 'retention-cohort-rc',+ members: ['retention-ru1', 'retention-ru4'],+ },+} as const;++/**+ * Hand-computed expected cohort matrices (per-cohort rows only, excluding the+ * leading "Weighted Average" row that processCohortData prepends).+ */+export const RETENTION_BLUEPRINT = {+ // firstEvent = secondEvent = app_open, interval day+ dayOn: [+ { cohort_interval: '2024-03-04', sum: 3, values: [3, 1, 2] },+ { cohort_interval: '2024-03-05', sum: 2, values: [2, 1, 0] },+ ],+ dayOnOrAfter: [+ { cohort_interval: '2024-03-04', sum: 3, values: [3, 2, 2] },+ { cohort_interval: '2024-03-05', sum: 2, values: [2, 1, 0] },+ ],+ // cross-year weekly self-retention, interval week, criteria on+ weekOn: [{ cohort_interval: '2024-12-29', sum: 2, values: [2, 1] }],+ // app_open self-retention, day, criteria on, filtered to country = US.+ // RU3 (country SE) drops out of the D0 cohort -> raw-events path.+ countryUsOn: [+ { cohort_interval: '2024-03-04', sum: 2, values: [2, 1, 2] },+ { cohort_interval: '2024-03-05', sum: 2, values: [2, 1, 0] },+ ],+ // app_open self-retention, day, criteria on, scoped to cohort {RU1, RU4}.+ // inCohort only needs profile_id -> stays on the fast cohort_events_mv path.+ cohortOn: [+ { cohort_interval: '2024-03-04', sum: 1, values: [1, 1, 1] },+ { cohort_interval: '2024-03-05', sum: 1, values: [1, 1, 0] },+ ],+ // Week-over-week active-user retention across the whole fixture.+ // - 2024-03-03: RU1-5 active, none active the next week -> 0% retained+ // - 2024-12-29: WU1+WU2 active, WU1 active next week -> 50% retained+ // - 2025-01-05: WU1 active, none after -> 0%+ weeklySeries: [+ { date: '2024-03-03', active_users: 5, retained_users: 0, retention: 0 },+ { date: '2024-12-29', active_users: 2, retained_users: 1, retention: 50 },+ { date: '2025-01-05', active_users: 1, retained_users: 0, retention: 0 },+ ],+} as const;++// ---------------------------------------------------------------------------+// Builders+// ---------------------------------------------------------------------------++type ChClient = ReturnType<typeof createClient>;++function getClient(): ChClient {+ const url = process.env.CLICKHOUSE_URL ?? 'http://localhost:8123';+ return createClient({ url });+}++let eventSeq = 0;++function buildEvent(+ projectId: string,+ profileId: string,+ name: string,+ createdAt: string,+ overrides: Record<string, unknown> = {}+) {+ eventSeq += 1;+ return {+ // Deterministic, collision-free uuid in the retention namespace+ id: `00000000-0000-4000-8000-${String(eventSeq).padStart(12, '0')}`,+ project_id: projectId,+ profile_id: profileId,+ device_id: `dev-${profileId}`,+ name,+ session_id: `sess-${profileId}`,+ created_at: createdAt,+ path: '/',+ origin: 'https://example.com',+ referrer: '',+ referrer_name: '',+ referrer_type: '',+ revenue: 0,+ duration: 0,+ properties: {},+ groups: [],+ country: 'US',+ city: '',+ region: '',+ sdk_name: 'web',+ sdk_version: '1.0.0',+ os: '',+ os_version: '',+ browser: 'Chrome',+ browser_version: '',+ device: 'desktop',+ brand: '',+ model: '',+ ...overrides,+ };+}++function buildEvents(projectId: string) {+ const { users, day } = RETENTION_FIXTURE;+ const dayAt = (date: string) => `${date} 12:00:00`;++ const us = { country: 'US' };+ const se = { country: 'SE' };++ return [+ // --- DAY scenario: app_open (self-retention) ---+ buildEvent(projectId, users.ru1, 'app_open', dayAt(day.d0), us),+ buildEvent(projectId, users.ru1, 'app_open', dayAt(day.d1), us),+ buildEvent(projectId, users.ru1, 'app_open', dayAt(day.d2), us),+ buildEvent(projectId, users.ru2, 'app_open', dayAt(day.d0), us),+ buildEvent(projectId, users.ru2, 'app_open', dayAt(day.d2), us),+ buildEvent(projectId, users.ru3, 'app_open', dayAt(day.d0), se),+ buildEvent(projectId, users.ru4, 'app_open', dayAt(day.d1), us),+ buildEvent(projectId, users.ru4, 'app_open', dayAt(day.d2), us),+ buildEvent(projectId, users.ru5, 'app_open', dayAt(day.d1), us),++ // --- DAY scenario: purchase (cross-event retention) ---+ buildEvent(projectId, users.ru1, 'purchase', dayAt(day.d1), us),+ buildEvent(projectId, users.ru2, 'purchase', dayAt(day.d2), us),++ // --- WEEK scenario: app_open crossing the 2024->2025 boundary ---+ // 2024-12-30 (Mon) falls in the Sunday-aligned week starting 2024-12-29.+ buildEvent(projectId, users.wu1, 'app_open', '2024-12-30 12:00:00'),+ buildEvent(projectId, users.wu1, 'app_open', '2025-01-06 12:00:00'),+ buildEvent(projectId, users.wu2, 'app_open', '2024-12-30 12:00:00'),+ ];+}++// ---------------------------------------------------------------------------+// Public API+// ---------------------------------------------------------------------------++function buildCohortMembers(projectId: string) {+ return RETENTION_FIXTURE.cohort.members.map((profileId) => ({+ project_id: projectId,+ cohort_id: RETENTION_FIXTURE.cohort.id,+ profile_id: profileId,+ matched_at: RETENTION_FIXTURE.day.start,+ matching_properties: {},+ version: 1,+ }));+}++async function deleteFixtures(client: ChClient, projectId: string) {+ await Promise.all([+ client.command({+ query: `DELETE FROM openpanel.events WHERE project_id = '${projectId}'`,+ }),+ // Materialized views are NOT touched by DELETE FROM events; mutate them too+ // so reruns stay clean.+ client.command({+ query: `ALTER TABLE openpanel.cohort_events_mv DELETE WHERE project_id = '${projectId}'`,+ }),+ client.command({+ query: `DELETE FROM openpanel.cohort_members WHERE project_id = '${projectId}'`,+ }),+ ]);+}++export async function setupRetentionFixtures(projectId: string): Promise<void> {+ const client = getClient();+ try {+ await deleteFixtures(client, projectId);+ await client.insert({+ table: 'openpanel.events',+ values: buildEvents(projectId),+ format: 'JSONEachRow',+ });+ await client.insert({+ table: 'openpanel.cohort_members',+ values: buildCohortMembers(projectId),+ format: 'JSONEachRow',+ });+ } finally {+ await client.close();+ }+}++export async function teardownRetentionFixtures(+ projectId: string+): Promise<void> {+ const client = getClient();+ try {+ await deleteFixtures(client, projectId);+ } finally {+ await client.close();+ }+}