OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.
ralph(M7-004): funnel + retention + sankey + conversion services into core (part 2)
ralph67c5595fb4b5
diff --git a/packages/core/src/index.ts b/packages/core/src/index.tsindex 5a9e8c648..5567d8eb1 100644--- a/packages/core/src/index.ts+++ b/packages/core/src/index.ts@@ -312,6 +312,63 @@ export {rewriteProfilePropertyRefs,transformPropertyKey,} from './modules/chart/chart.service';+// Dissolved from @openpanel/db's services/conversion.service.ts (M7-004) —+// reached through the chart module's own dispatch and the db shim.+export { getConversion } from './modules/chart/conversion.service';+// Dissolved from @openpanel/db's services/funnel.service.ts (M7-004) —+// packages/db's reports.service, apps/api's insights controller and the+// mcp/assistant tools call these. packages/db/src/services/funnel.service.ts+// stays a re-export shim.+export type {+ BuildFunnelBaseInput,+ FunnelStep,+} from './modules/chart/funnel.service';+export {+ buildFunnelBase,+ buildSessionsCte,+ EMPTY_BREAKDOWN_LABEL,+ getFunnel,+ getFunnelCore,+ getFunnelGroup,+ getFunnelProfileIds,+ toSeries,+} from './modules/chart/funnel.service';+// Dissolved from @openpanel/db's services/retention.service.ts (M7-004) —+// apps/api's insights controller and the mcp/assistant tools call these.+// packages/db/src/services/retention.service.ts stays a re-export shim.+export type {+ IGetRetentionCohortInput,+ IRetentionCohortRow,+ IRetentionCriteria,+ IRetentionInterval,+ IServiceRetentionRollingActiveUsers,+} from './modules/chart/retention.service';+export {+ getEngagementCore,+ getRetentionCohort,+ getRetentionCohortCore,+ getRetentionLastSeenSeries,+ getRetentionSeries,+ getRollingActiveUsers,+ getRollingActiveUsersCore,+ getWeeklyRetentionSeriesCore,+ processCohortData,+} from './modules/chart/retention.service';+// Dissolved from @openpanel/db's services/sankey.service.ts (M7-004) —+// apps/api's insights controller and the mcp/assistant tools call these.+// packages/db/src/services/sankey.service.ts stays a re-export shim.+export type {+ IGetSankeyInput,+ SankeyLink,+ SankeyNode,+ SankeyResult,+} from './modules/chart/sankey.service';+export {+ getRawWhereClause,+ getSankey,+ getUserFlowCore,+ zGetSankeyInput,+} from './modules/chart/sankey.service';// Dissolved from @openpanel/db's services/clients.service.ts (M6-002) —// packages/trpc's client router, apps/api's manage controller and mcp/utils// auth call these directly, the same way V1 reaches every other dissolveddiff --git a/packages/core/src/modules/assistant/src/report-runner.ts b/packages/core/src/modules/assistant/src/report-runner.tsindex 84bf1fe20..fcbc4a5c4 100644--- a/packages/core/src/modules/assistant/src/report-runner.ts+++ b/packages/core/src/modules/assistant/src/report-runner.ts@@ -19,8 +19,8 @@import {AggregateChartEngine,ChartEngine,- funnelService,getChartStartEndDate,+ getFunnel,getReportById,getSettingsForProject,} from '@openpanel/db';@@ -40,7 +40,7 @@ function reportUrl(/*** Execute a saved report by ID. Dispatches on chart type:- * - funnel → funnelService.getFunnel+ * - funnel → getFunnel* - metric → AggregateChartEngine.execute* - others → ChartEngine.execute*@@ -98,7 +98,7 @@ export async function runReport(input: {};if (report.chartType === 'funnel') {- return { ...meta, data: await funnelService.getFunnel(chartInput) };+ return { ...meta, data: await getFunnel(chartInput) };}if (report.chartType === 'metric') {return { ...meta, data: await AggregateChartEngine.execute(chartInput) };@@ -148,9 +148,7 @@ export async function runReportFromConfig(input: {if (input.config.chartType === 'funnel') {return {...meta,- data: await funnelService.getFunnel(- chartInput as Parameters<typeof funnelService.getFunnel>[0]- ),+ data: await getFunnel(chartInput as Parameters<typeof getFunnel>[0]),};}if (input.config.chartType === 'metric') {diff --git a/packages/core/src/modules/assistant/src/tools/dashboard.ts b/packages/core/src/modules/assistant/src/tools/dashboard.tsindex 0551dc420..4489fa793 100644--- a/packages/core/src/modules/assistant/src/tools/dashboard.ts+++ b/packages/core/src/modules/assistant/src/tools/dashboard.ts@@ -1,9 +1,9 @@import {AggregateChartEngine,ChartEngine,- funnelService,getChartStartEndDate,getDashboardById,+ getFunnel,getReportsByDashboardId,getSettingsForProject,} from '@openpanel/db';@@ -99,8 +99,8 @@ export const summarizeDashboard = chatTool(let data: unknown;if (report.chartType === 'funnel') {- data = await funnelService.getFunnel(- chartInput as Parameters<typeof funnelService.getFunnel>[0]+ data = await getFunnel(+ chartInput as Parameters<typeof getFunnel>[0]);} else if (report.chartType === 'metric') {data = await AggregateChartEngine.execute(chartInput);diff --git a/packages/core/src/modules/chart/chart.service.ts b/packages/core/src/modules/chart/chart.service.tsindex 2e052dae0..863112680 100644--- a/packages/core/src/modules/chart/chart.service.ts+++ b/packages/core/src/modules/chart/chart.service.ts@@ -5,15 +5,14 @@// src/chart.sql.proof.md; V1's chart.service and engine are re-export shims// onto this module and its router delegates here (DELEGATE PATTERN).//-// `funnel`, `conversion`, `sankey`, `cohort` and `getFunnelProfiles` still-// run V1's funnel/conversion/sankey/retention services — those queries are-// M7-004/M7-005's, and are reached lazily through @openpanel/db until then.+// The funnel, conversion, sankey and retention statements this module+// dispatches to live in the sibling `*.service.ts` files of this module+// (M7-004).//// db/ch access is lazy (`load*`), as in the sibling modules: the barrel pulls// this module into nearly every core test file, and constructing// @openpanel/db's clients at import time costs a pino-pretty worker per file.-import type { SqlFragment } from '@openpanel/db/src/clickhouse/sql';import {getChartPrevStartEndDate,getChartStartEndDate,@@ -29,7 +28,6 @@ import type {IReportInput,} from '@openpanel/validation';import { flatten, map, pipe, prop, sort, uniq } from 'ramda';-import sqlstring from 'sqlstring';import type { ServiceDeps } from '../../services';import { getEventMetasCached } from '../event/event.service';import { getSettingsForProject } from '../organization/organization.service';@@ -38,6 +36,10 @@ import {getProfilesCached,type IServiceProfile,} from '../profile/profile.service';+import { getConversion } from './conversion.service';+import { getFunnel, getFunnelProfileIds } from './funnel.service';+import { getRetentionCohort } from './retention.service';+import { getSankey } from './sankey.service';import {chartBucketProfilesQuery,eventFieldValuesQuery,@@ -58,6 +60,7 @@ import {isKnownEventField,normalizeEventField,} from './src/field-resolution';+import { runQuery } from './src/run-query';export {type ChartBucketProfilesInput,@@ -171,41 +174,10 @@ const FIXED_FILTER_PROPERTIES = ['profile.last_seen_at',];-function loadChClient() {- return import('@openpanel/db/src/clickhouse/client');-}--function loadFunnelService() {- return import('@openpanel/db/src/services/funnel.service');-}--function loadConversionService() {- return import('@openpanel/db/src/services/conversion.service');-}--function loadSankeyService() {- return import('@openpanel/db/src/services/sankey.service');-}--function loadRetentionService() {- return import('@openpanel/db/src/services/retention.service');-}-function loadReportsService() {return import('@openpanel/db/src/services/reports.service');}-async function runQuery<T extends object>(- statement: SqlFragment,- timezone?: string-): Promise<T[]> {- const { chQuery } = await loadChClient();- return chQuery<T>(- statement,- timezone ? { session_timezone: timezone } : undefined- );-}-async function getProfilesInBatches(ids: string[],projectId: string,@@ -443,7 +415,7 @@ export async function getChartPropertyValues(input: {};}-// --- funnel / conversion / sankey / cohort (V1 services, M7-004/M7-005) -----+// --- funnel / conversion / sankey / retention --------------------------------async function currentAndPreviousPeriod(chartInput: IReportInput) {const { timezone } = await getSettingsForProject(chartInput.projectId);@@ -453,14 +425,13 @@ async function currentAndPreviousPeriod(chartInput: IReportInput) {}export async function getFunnelChart(chartInput: IReportInput) {- const { funnelService } = await loadFunnelService();const { timezone, currentPeriod, previousPeriod } =await currentAndPreviousPeriod(chartInput);const [current, previous] = await Promise.all([- funnelService.getFunnel({ ...chartInput, ...currentPeriod, timezone }),+ getFunnel({ ...chartInput, ...currentPeriod, timezone }),chartInput.previous- ? funnelService.getFunnel({ ...chartInput, ...previousPeriod, timezone })+ ? getFunnel({ ...chartInput, ...previousPeriod, timezone }): Promise.resolve(null),]);@@ -468,20 +439,19 @@ export async function getFunnelChart(chartInput: IReportInput) {}export async function getConversionChart(chartInput: IReportInput) {- const { conversionService } = await loadConversionService();const { timezone, currentPeriod, previousPeriod } =await currentAndPreviousPeriod(chartInput);const interval = chartInput.interval;const [current, previous] = await Promise.all([- conversionService.getConversion({+ getConversion({...chartInput,...currentPeriod,interval,timezone,}),chartInput.previous- ? conversionService.getConversion({+ ? getConversion({...chartInput,...previousPeriod,interval,@@ -503,8 +473,7 @@ export async function getConversionChart(chartInput: IReportInput) {}export async function getSankeyChart(input: IReportInput) {- const [{ sankeyService }, { mergeGlobalFilters, onlyReportEvents }] =- await Promise.all([loadSankeyService(), loadReportsService()]);+ const { mergeGlobalFilters, onlyReportEvents } = await loadReportsService();const { timezone } = await getSettingsForProject(input.projectId);const currentPeriod = getChartStartEndDate(input, timezone);@@ -520,7 +489,7 @@ export async function getSankeyChart(input: IReportInput) {throw new Error('Start and end events are required');}- return sankeyService.getSankey({+ return getSankey({projectId: input.projectId,startDate: currentPeriod.startDate,endDate: currentPeriod.endDate,@@ -590,7 +559,6 @@ export async function getRetentionChart(report: NonNullable<IServiceReport> | null,input: RetentionChartInput) {- const { getRetentionCohort } = await loadRetentionService();const resolved = report ? retentionInputFromReport(report, input) : input;const { timezone } = await getSettingsForProject(resolved.projectId);@@ -671,15 +639,14 @@ export interface FunnelStepProfilesRequest {}/**- * Profiles at (or dropping off at) one funnel step. Runs on V1's clix funnel- * builder until the funnel module (M7-004) converts it: the CTE has to be the- * chart's own, or breakdown expressions referencing a `profile` / `cohort_<id>`- * alias this side never joined fail with UNKNOWN_IDENTIFIER.+ * Profiles at (or dropping off at) one funnel step. Built on the funnel+ * module's own base: the CTE has to be the chart's own, or breakdown+ * expressions referencing a `profile` / `cohort_<id>` alias this side never+ * joined fail with UNKNOWN_IDENTIFIER.*/export async function getFunnelStepProfiles(input: FunnelStepProfilesRequest): Promise<IServiceProfile[]> {- const { funnelService, EMPTY_BREAKDOWN_LABEL } = await loadFunnelService();const { timezone } = await getSettingsForProject(input.projectId);const {projectId,@@ -692,10 +659,8 @@ export async function getFunnelStepProfiles(breakdownValues = [],} = input;const { startDate, endDate } = getChartStartEndDate(input, timezone);- // stepIndex is 0-based, level is 1-based.- const targetLevel = stepIndex + 1;- const { query, breakdowns } = await funnelService.buildFunnelBase({+ const ids = await getFunnelProfileIds({projectId,startDate,endDate,@@ -704,41 +669,12 @@ export async function getFunnelStepProfiles(funnelWindow,funnelGroup,timezone,+ // stepIndex is 0-based, level is 1-based.+ targetLevel: stepIndex + 1,+ showDropoffs,+ breakdownValues,+ limit: FUNNEL_PROFILES_LIMIT,});-- // Same shape as the chart's `funnel` CTE: windowFunnel is already computed- // per primary key, so drop level=0 and select distinct profiles.- query.with('funnel', 'SELECT * FROM session_funnel WHERE level != 0');- query.select(['DISTINCT profile_id']).from('funnel');- if (showDropoffs) {- query.where('level', '=', targetLevel);- } else {- query.where('level', '>=', targetLevel);- }-- // The clicked row carries DISPLAY labels (trimmed, empty/null shown as- // EMPTY_BREAKDOWN_LABEL), so match against the same normalization.- // toString/ifNull keep the comparison valid for numeric and Nullable- // breakdown columns.- breakdowns.forEach((_, index) => {- const value = breakdownValues[index];- if (value === undefined) {- return;- }- const normalized = `trim(ifNull(toString(b_${index}), ''))`;- if (value === EMPTY_BREAKDOWN_LABEL) {- query.rawWhere(- `(${normalized} = '' OR ${normalized} = ${sqlstring.escape(EMPTY_BREAKDOWN_LABEL)})`- );- } else {- query.rawWhere(`${normalized} = ${sqlstring.escape(value)}`);- }- });-- query.limit(FUNNEL_PROFILES_LIMIT);-- const rows = (await query.execute()) as { profile_id: string }[];- const ids = rows.map((row) => row.profile_id).filter(Boolean);if (ids.length === 0) {return [];}diff --git a/packages/core/src/modules/chart/conversion.service.ts b/packages/core/src/modules/chart/conversion.service.tsnew file mode 100644index 000000000..a7a0f2875--- /dev/null+++ b/packages/core/src/modules/chart/conversion.service.ts@@ -0,0 +1,262 @@+// Ported from packages/db/src/services/conversion.service.ts (M7-004). The+// statement is a `sql` fragment from src/conversion.sql.ts, proven+// byte-equivalent to V1 in src/conversion.sql.proof.md; V1's+// conversion.service is a re-export shim onto this module (DELEGATE PATTERN).++import { NOT_SET_VALUE } from '@openpanel/constants';+import type { IReportInput } from '@openpanel/validation';+import { omit } from 'ramda';+import { fetchCohortsMetadata } from './src/chart-statement';+import { type ConversionGroup, conversionQuery } from './src/conversion.sql';+import {+ buildInlineCohortJoin,+ collectBreakdownCohortIds,+ extractCohortId,+ getSelectPropertyKey,+ isKnownEventField,+} from './src/field-resolution';+import { getEventFiltersWhereClause } from './src/filter-where';+import { runQuery } from './src/run-query';++// Lazy: packages/db's reports.service imports the funnel shim, which points+// back here — a static import would close the cycle at module load.+function loadReportsService() {+ return import('@openpanel/db/src/services/reports.service');+}++/** Default funnel window, in hours, when the report does not set one. */+const DEFAULT_FUNNEL_WINDOW_HOURS = 24;+const SECONDS_PER_HOUR = 3600;++/** Profile columns the join may expose, beyond the `properties` Map. */+const JOINABLE_PROFILE_COLUMNS = [+ 'email',+ 'first_name',+ 'last_name',+ 'created_at',+ 'last_seen_at',+];++const CONVERSION_STEP_COUNT = 2;++interface ConversionRow {+ event_day: string;+ total_first: number;+ conversions: number;+ conversion_rate_percentage: number;+ [key: string]: string | number;+}++interface ConversionSeriePoint {+ date: string;+ total: number;+ conversions: number;+ rate: number;+}++interface ConversionSerie {+ id: string;+ breakdowns: string[];+ data: ConversionSeriePoint[];+}++export type ConversionChartInput = Omit<+ IReportInput,+ 'range' | 'previous' | 'metric' | 'chartType'+> & { timezone: string };++/**+ * The profile columns the LEFT ANY JOIN must expose for the requested+ * breakdowns. Empty means no join.+ */+function profileJoinFields(profileBreakdowns: { name: string }[]): string[] {+ if (profileBreakdowns.length === 0) {+ return [];+ }+ const fields = new Set<string>(['id']);+ for (const breakdown of profileBreakdowns) {+ const fieldName = breakdown.name.replace('profile.', '').split('.')[0];+ if (fieldName === 'properties') {+ fields.add('properties');+ } else if (fieldName && JOINABLE_PROFILE_COLUMNS.includes(fieldName)) {+ fields.add(fieldName);+ }+ }+ return Array.from(fields);+}++export async function getConversion({+ projectId,+ startDate,+ endDate,+ options,+ series,+ globalFilters,+ breakdowns = [],+ limit,+ interval,+ timezone,+}: ConversionChartInput) {+ const { mergeGlobalFilters, onlyReportEvents } = await loadReportsService();+ const mergedSeries = mergeGlobalFilters(series, globalFilters);+ const funnelOptions = options?.type === 'funnel' ? options : undefined;+ const funnelGroup = funnelOptions?.funnelGroup;+ const funnelWindow =+ funnelOptions?.funnelWindow ?? DEFAULT_FUNNEL_WINDOW_HOURS;+ const group: ConversionGroup =+ funnelGroup === 'profile_id' ? 'profile_id' : 'session_id';++ // Same guard as the funnel service / getChartSql — drop breakdowns whose+ // name can't be resolved against the events schema.+ const knownBreakdowns = breakdowns.filter((breakdown) =>+ isKnownEventField(breakdown.name)+ );++ const cohortIds = collectBreakdownCohortIds(knownBreakdowns);+ const cohortMetadata = await fetchCohortsMetadata(cohortIds);+ const cohortJoins = cohortIds.map((id) =>+ buildInlineCohortJoin(id, projectId, 'events')+ );++ const breakdownExpressions = knownBreakdowns.map((breakdown) => {+ const cohortId = extractCohortId(breakdown.name);+ const cohortName = cohortId+ ? cohortMetadata.get(cohortId)?.name+ : undefined;+ return getSelectPropertyKey(+ breakdown.name,+ projectId,+ cohortId ?? undefined,+ cohortName+ );+ });+ const breakdownSelects = breakdownExpressions.map(+ (expression, index) => `${expression} as b_${index}`+ );++ const events = onlyReportEvents(mergedSeries);++ const anyBreakdownOnGroup = knownBreakdowns.some((breakdown) =>+ breakdown.name.startsWith('group.')+ );+ const anyFilterOnGroup = events.some((event) =>+ event.filters?.some((filter) => filter.name.startsWith('group.'))+ );++ if (events.length !== CONVERSION_STEP_COUNT) {+ throw new Error('events must be an array of two events');+ }++ if (!(startDate && endDate)) {+ throw new Error('startDate and endDate are required');+ }++ const eventA = events[0] as (typeof events)[number];+ const eventB = events[1] as (typeof events)[number];+ // Qualify with 'events' so event-level `properties[...]` becomes+ // `events.properties[...]` — required when the conversion query also joins+ // the profiles table (which exposes a `properties` column). Without the+ // qualifier ClickHouse fails with "ambiguous identifier 'properties'"+ // whenever a step filters on properties.X while a breakdown is on+ // profile.properties.Y.+ const whereA = Object.values(+ getEventFiltersWhereClause(eventA.filters, projectId, 'events')+ ).join(' AND ');+ const whereB = Object.values(+ getEventFiltersWhereClause(eventB.filters, projectId, 'events')+ ).join(' AND ');++ const results = await runQuery<ConversionRow>(+ conversionQuery({+ projectId,+ startDate,+ endDate,+ interval,+ group,+ funnelWindowSeconds: funnelWindow * SECONDS_PER_HOUR,+ firstEventName: eventA.name,+ secondEventName: eventB.name,+ firstEventWhere: whereA,+ secondEventWhere: whereB,+ breakdownSelects,+ breakdownExpressions,+ profileJoinFields: profileJoinFields(+ knownBreakdowns.filter((breakdown) =>+ breakdown.name.startsWith('profile.')+ )+ ),+ needsGroupArrayJoin: anyBreakdownOnGroup || anyFilterOnGroup,+ cohortJoins,+ }),+ timezone+ );++ return toSeries(results, knownBreakdowns, limit).map((serie, serieIndex) => ({+ ...serie,+ data: serie.data.map((point, index) => ({+ ...point,+ timestamp: new Date(point.date).getTime(),+ serieIndex,+ index,+ serie: omit(['data'], serie),+ })),+ }));+}++function toPoint(row: ConversionRow): ConversionSeriePoint {+ return {+ date: row.event_day,+ total: row.total_first,+ conversions: row.conversions,+ rate: row.conversion_rate_percentage,+ };+}++function toSeries(+ data: ConversionRow[],+ breakdowns: { name: string }[] = [],+ limit: number | undefined = undefined+) {+ if (breakdowns.length === 0) {+ return [+ {+ id: 'conversion',+ breakdowns: [],+ data: data.map(toPoint),+ },+ ];+ }++ const series = data.reduce(+ (acc, row) => {+ if (limit && Object.keys(acc).length >= limit) {+ return acc;+ }++ const key =+ breakdowns.map((_, index) => row[`b_${index}`]).join('|') ||+ NOT_SET_VALUE;+ if (!acc[key]) {+ acc[key] = {+ id: key,+ breakdowns: breakdowns.map(+ (_, index) => (row[`b_${index}`] || NOT_SET_VALUE) as string+ ),+ data: [],+ };+ }+ (acc[key] as ConversionSerie).data.push(toPoint(row));+ return acc;+ },+ {} as Record<string, ConversionSerie>+ );++ return Object.values(series).map((serie, serieIndex) => ({+ ...serie,+ data: serie.data.map((item, dataIndex) => ({+ ...item,+ dataIndex,+ serieIndex,+ })),+ }));+}diff --git a/packages/core/src/modules/chart/funnel.service.ts b/packages/core/src/modules/chart/funnel.service.tsnew file mode 100644index 000000000..f67b6cec1--- /dev/null+++ b/packages/core/src/modules/chart/funnel.service.ts@@ -0,0 +1,424 @@+// Ported from packages/db/src/services/funnel.service.ts (M7-004). Every+// ClickHouse statement is a `sql` fragment from src/funnel.sql.ts, proven+// byte-equivalent to V1 in src/funnel.sql.proof.md; V1's funnel.service is a+// re-export shim onto this module (DELEGATE PATTERN).++import { ifNaN } from '@openpanel/common';+import type {+ IChartBreakdown,+ IChartEvent,+ IReportInput,+} from '@openpanel/validation';+import { last, reverse } from 'ramda';+import { getSettingsForProject } from '../organization/organization.service';+import { fetchCohortsMetadata } from './src/chart-statement';+import { collectBreakdownCohortIds } from './src/field-resolution';+import {+ EMPTY_BREAKDOWN_LABEL,+ type FunnelBase,+ type FunnelGroup,+ type FunnelProfilesInput,+ funnelBase,+ funnelChartQuery,+ funnelProfilesQuery,+ funnelSessionsQuery,+ knownFunnelBreakdowns,+} from './src/funnel.sql';+import { runQuery } from './src/run-query';++// Lazy: packages/db's reports.service imports the funnel shim, which points+// back here — a static import would close the cycle at module load.+function loadReportsService() {+ return import('@openpanel/db/src/services/reports.service');+}++export {+ EMPTY_BREAKDOWN_LABEL,+ type FunnelBase,+ type FunnelProfilesInput,+ funnelProfilesQuery,+} from './src/funnel.sql';++/** Default funnel window, in hours, when the report does not set one. */+const DEFAULT_FUNNEL_WINDOW_HOURS = 24;+const MILLISECONDS_PER_HOUR = 3600 * 1000;+const PERCENT = 100;+const ROUND_TO_TWO_DECIMALS = 100;+const ROUND_RATE_TO_TWO_DECIMALS = 10_000;++export interface FunnelStep {+ event: IChartEvent & { displayName: string };+ count: number;+ percent: number;+ dropoffCount: number | null;+ dropoffPercent: number | null;+ previousCount: number;+ nextCount: number | null;+ isHighestDropoff: boolean;+}++interface FunnelRow {+ level: number;+ count: number;+ [key: string]: string | number;+}++interface FunnelSerieRow {+ id: string;+ breakdowns: string[];+ level: number;+ count: number;+}++function normalizeBreakdownValue(value: unknown): string {+ if (value === null || value === undefined || value === '') {+ return EMPTY_BREAKDOWN_LABEL;+ }+ const text = String(value).trim();+ return text === '' ? EMPTY_BREAKDOWN_LABEL : text;+}++/**+ * Returns the grouping strategy for the funnel: whether windowFunnel is+ * computed per session_id or per profile_id. profile_id correctly handles+ * cross-session funnel completions.+ */+export function getFunnelGroup(group?: string): FunnelGroup {+ return group === 'profile_id' ? 'profile_id' : 'session_id';+}++export interface BuildFunnelBaseInput {+ projectId: string;+ startDate: string;+ endDate: string;+ series: IReportInput['series'];+ globalFilters?: IReportInput['globalFilters'];+ breakdowns?: IChartBreakdown[];+ funnelWindow?: number;+ funnelGroup?: string;+ timezone: string;+}++/**+ * Everything the funnel chart and the funnel profile list share: the+ * normalized event series and breakdowns, and the `session_funnel` / `funnel`+ * CTEs with all of their joins wired up. Callers add their own projection.+ */+export async function buildFunnelBase({+ projectId,+ startDate,+ endDate,+ series,+ globalFilters,+ breakdowns: initialBreakdowns = [],+ funnelWindow = DEFAULT_FUNNEL_WINDOW_HOURS,+ funnelGroup,+ timezone,+}: BuildFunnelBaseInput): Promise<FunnelBase> {+ const { mergeGlobalFilters, onlyReportEvents } = await loadReportsService();+ const breakdowns = knownFunnelBreakdowns(initialBreakdowns);+ const eventSeries = onlyReportEvents(+ mergeGlobalFilters(series, globalFilters)+ );+ const cohortMetadata = await fetchCohortsMetadata(+ collectBreakdownCohortIds(breakdowns)+ );++ return funnelBase({+ projectId,+ startDate,+ endDate,+ eventSeries,+ breakdowns,+ funnelWindowMilliseconds: funnelWindow * MILLISECONDS_PER_HOUR,+ group: getFunnelGroup(funnelGroup),+ funnelGroup,+ cohortMetadata,+ timezone,+ });+}++export function buildSessionsCte(input: {+ projectId: string;+ startDate: string;+ endDate: string;+}) {+ return funnelSessionsQuery(input);+}++function fillFunnel(funnel: { level: number; count: number }[], steps: number) {+ const filled = Array.from({ length: steps }, (_, index) => {+ const level = index + 1;+ const matchingResult = funnel.find((result) => result.level === level);+ return {+ level,+ count: matchingResult ? matchingResult.count : 0,+ };+ });++ // Accumulate counts from the bottom of the funnel upwards.+ for (let index = filled.length - 1; index >= 0; index--) {+ const step = filled[index];+ const nextStep = filled[index + 1];+ if (step && nextStep) {+ step.count += nextStep.count;+ }+ }+ return filled.reverse();+}++export function toSeries(+ funnel: FunnelRow[],+ breakdowns: { name: string }[] = [],+ limit: number | undefined = undefined+): FunnelSerieRow[][] {+ if (breakdowns.length === 0) {+ return [+ funnel.map((row) => ({+ level: row.level,+ count: row.count,+ id: 'none',+ breakdowns: [],+ })),+ ];+ }++ // Group by breakdown values (normalize empty/null to "Not set").+ const series = funnel.reduce(+ (acc, row) => {+ if (limit && Object.keys(acc).length >= limit) {+ return acc;+ }++ const key = breakdowns+ .map((_, index) => normalizeBreakdownValue(row[`b_${index}`]))+ .join('|');+ if (!acc[key]) {+ acc[key] = [];+ }+ (acc[key] as FunnelSerieRow[]).push({+ id: key,+ breakdowns: breakdowns.map((_, index) =>+ normalizeBreakdownValue(row[`b_${index}`])+ ),+ level: row.level,+ count: row.count,+ });+ return acc;+ },+ {} as Record<string, FunnelSerieRow[]>+ );++ return Object.values(series);+}++function isHighestDropoff(+ step: { dropoffCount: number | null },+ index: number,+ list: { dropoffCount: number | null }[]+): boolean {+ if (!step?.dropoffCount) {+ return false;+ }+ const maxDropoff = Math.max(+ ...list.map((item) => item.dropoffCount || 0).filter((count) => count > 0)+ );+ return (+ step.dropoffCount === maxDropoff &&+ list.findIndex((item) => item.dropoffCount === maxDropoff) === index+ );+}++function toFunnelSteps(+ data: FunnelSerieRow[],+ eventSeries: IChartEvent[]+): { steps: FunnelStep[]; totalSessions: number } {+ const filledFunnel = fillFunnel(+ data.map((row) => ({ level: row.level, count: row.count })),+ eventSeries.length+ );+ const totalSessions = last(filledFunnel)?.count ?? 0;++ const steps = reverse(filledFunnel)+ .reduce(+ (acc, item, index, list) => {+ const previous = list[index - 1] ?? { count: totalSessions };+ const next = list[index + 1];+ const event = eventSeries[item.level - 1] as IChartEvent;+ acc.push({+ event: {+ ...event,+ displayName: event.displayName || event.name,+ },+ count: item.count,+ percent: (item.count / totalSessions) * PERCENT,+ dropoffCount: next ? item.count - next.count : null,+ dropoffPercent: next+ ? ((item.count - next.count) / item.count) * PERCENT+ : null,+ previousCount: previous.count,+ nextCount: next?.count ?? null,+ });+ return acc;+ },+ [] as Omit<FunnelStep, 'isHighestDropoff'>[]+ )+ .map((step, index, list) => ({+ ...step,+ percent: ifNaN(step.percent, 0),+ dropoffPercent: ifNaN(step.dropoffPercent, 0),+ isHighestDropoff: isHighestDropoff(step, index, list),+ }));++ return { steps, totalSessions };+}++export async function getFunnel({+ projectId,+ startDate,+ endDate,+ series,+ globalFilters,+ options,+ breakdowns: initialBreakdowns = [],+ limit,+ timezone = 'UTC',+}: IReportInput & { timezone: string; events?: IChartEvent[] }) {+ if (!(startDate && endDate)) {+ throw new Error('startDate and endDate are required');+ }++ const funnelOptions = options?.type === 'funnel' ? options : undefined;+ const base = await buildFunnelBase({+ projectId,+ startDate,+ endDate,+ series,+ globalFilters,+ breakdowns: initialBreakdowns,+ funnelWindow: funnelOptions?.funnelWindow,+ funnelGroup: funnelOptions?.funnelGroup,+ timezone,+ });++ const funnelData = await runQuery<FunnelRow>(+ funnelChartQuery(base),+ base.timezone+ );++ return toSeries(funnelData, base.breakdowns, limit)+ .map((data) => {+ const { steps, totalSessions } = toFunnelSteps(data, base.eventSeries);+ return {+ id: data[0]?.id ?? 'none',+ breakdowns: data[0]?.breakdowns ?? [],+ steps,+ totalSessions,+ lastStep: last(steps) as FunnelStep,+ mostDropoffsStep: steps.find(+ (step) => step.isHighestDropoff+ ) as FunnelStep,+ };+ })+ .sort((a, b) => {+ const aTotal = a.steps.reduce((acc, step) => acc + step.count, 0);+ const bTotal = b.steps.reduce((acc, step) => acc + step.count, 0);+ return bTotal - aTotal;+ });+}++export async function getFunnelCore(input: {+ projectId: string;+ startDate: string;+ endDate: string;+ steps: string[];+ windowHours?: number;+ groupBy?: FunnelGroup;+}) {+ const { timezone } = await getSettingsForProject(input.projectId);+ const eventSeries = input.steps.map((name, index) => ({+ id: String(index + 1),+ type: 'event' as const,+ name,+ displayName: name,+ segment: 'user' as const,+ filters: [],+ }));++ const result = await getFunnel({+ projectId: input.projectId,+ startDate: input.startDate,+ endDate: input.endDate,+ series: eventSeries,+ breakdowns: [],+ chartType: 'funnel',+ interval: 'day',+ range: 'custom',+ previous: false,+ metric: 'sum',+ options: {+ type: 'funnel',+ funnelWindow: input.windowHours ?? DEFAULT_FUNNEL_WINDOW_HOURS,+ funnelGroup: input.groupBy ?? 'session_id',+ },+ timezone,+ });++ const primarySeries = result[0];+ if (!primarySeries) {+ return {+ steps: [],+ totalUsers: 0,+ completedUsers: 0,+ overallConversionRate: 0,+ };+ }++ const steps = primarySeries.steps.map((step, index) => ({+ step: index + 1,+ eventName: step.event.displayName || step.event.name,+ users: step.count,+ conversionRateFromStart:+ Math.round(step.percent * ROUND_TO_TWO_DECIMALS) / ROUND_TO_TWO_DECIMALS,+ dropoffPercent:+ step.dropoffPercent === null+ ? null+ : Math.round(step.dropoffPercent * ROUND_TO_TWO_DECIMALS) /+ ROUND_TO_TWO_DECIMALS,+ isHighestDropoff: step.isHighestDropoff,+ }));++ const totalUsers = steps[0]?.users ?? 0;+ const completedUsers = steps.at(-1)?.users ?? 0;++ return {+ steps,+ totalUsers,+ completedUsers,+ overallConversionRate:+ totalUsers > 0+ ? Math.round(+ (completedUsers / totalUsers) * ROUND_RATE_TO_TWO_DECIMALS+ ) / ROUND_TO_TWO_DECIMALS+ : 0,+ };+}++export async function getFunnelProfileIds(+ input: BuildFunnelBaseInput &+ Omit<FunnelProfilesInput, 'breakdownValues'> & {+ breakdownValues: (string | undefined)[];+ }+): Promise<string[]> {+ const base = await buildFunnelBase(input);+ const rows = await runQuery<{ profile_id: string }>(+ funnelProfilesQuery(base, {+ targetLevel: input.targetLevel,+ showDropoffs: input.showDropoffs,+ breakdownValues: input.breakdownValues,+ limit: input.limit,+ }),+ base.timezone+ );+ return rows.map((row) => row.profile_id).filter(Boolean);+}diff --git a/packages/core/src/modules/chart/retention.service.ts b/packages/core/src/modules/chart/retention.service.tsnew file mode 100644index 000000000..f9c72e7c7--- /dev/null+++ b/packages/core/src/modules/chart/retention.service.ts@@ -0,0 +1,372 @@+// Ported from packages/db/src/services/retention.service.ts (M7-004). Every+// ClickHouse statement is a `sql` fragment from src/retention.sql.ts, proven+// byte-equivalent to V1 in src/retention.sql.proof.md; V1's retention.service+// is a re-export shim onto this module (DELEGATE PATTERN).+//+// The cohort matrix here is the single source of truth for retention cohorts:+// it powers the dashboard retention chart (via the tRPC `cohort` procedure) as+// well as the MCP / agent / REST retention endpoints.++import { DateTime, round } from '@openpanel/common';+import type { IChartEventFilter } from '@openpanel/validation';+import { range } from 'ramda';+import {+ type IRetentionCriteria,+ type IRetentionInterval,+ retentionCohortQuery,+ retentionLastSeenSeriesQuery,+ retentionSeriesQuery,+ rollingActiveUsersQuery,+} from './src/retention.sql';+import { runQuery } from './src/run-query';++export type {+ IRetentionCriteria,+ IRetentionInterval,+} from './src/retention.sql';++/** Days-since-last-seen buckets the engagement summary reports on. */+const ENGAGEMENT_ACTIVE_DAYS = 7;+const ENGAGEMENT_RECENT_DAYS = 14;+const ENGAGEMENT_LAPSING_DAYS = 30;+const ENGAGEMENT_INACTIVE_DAYS = 60;++/** getRetentionCohortCore's fixed window. */+const RETENTION_COHORT_WEEKS = 12;++const ROLLING_ACTIVE_USER_LABELS: Record<number, string> = {+ 1: 'DAU',+ 7: 'WAU',+ 30: 'MAU',+};++const RETENTION_PERCENTAGE_DECIMALS = 2;+const RETENTION_COUNT_DECIMALS = 0;++interface IGetWeekRetentionInput {+ projectId: string;+}++export interface IServiceRetentionRollingActiveUsers {+ date: string;+ users: number;+}++export interface IRetentionCohortRow {+ cohort_interval: string;+ sum: number;+ values: number[];+ percentages: number[];+}++export interface IGetRetentionCohortInput {+ projectId: string;+ /** Event name(s) defining cohort entry. Empty/undefined => any event. */+ firstEvent?: string[];+ /** Event name(s) that count as "returned". Empty/undefined => any event. */+ secondEvent?: string[];+ criteria?: IRetentionCriteria;+ interval?: IRetentionInterval;+ /** ISO or `yyyy-MM-dd HH:mm:ss`. */+ startDate: string;+ /** ISO or `yyyy-MM-dd HH:mm:ss`. */+ endDate: string;+ /**+ * Property and/or cohort filters scoping the analysed events. Cohort+ * membership (inCohort/notInCohort) keeps the fast cohort_events_mv path;+ * any property/column filter falls back to the raw events table.+ */+ filters?: IChartEventFilter[];+}++// Week-over-week retention graph: for each week, how many active users were+// also active the following week.+//+// Instead of self-joining the raw events table (O(events²) per profile), the+// statement first collapses events to one row per (profile, week), then+// self-joins that much smaller set to its next week.+export function getRetentionSeries({ projectId }: IGetWeekRetentionInput) {+ return runQuery<{+ date: string;+ active_users: number;+ retained_users: number;+ retention: number;+ }>(retentionSeriesQuery(projectId));+}++// https://medium.com/@andre_bodro/how-to-fast-calculating-mau-in-clickhouse-fd793559b229+// Rolling active users+export function getRollingActiveUsers({+ projectId,+ days,+}: IGetWeekRetentionInput & { days: number }) {+ return runQuery<IServiceRetentionRollingActiveUsers>(+ rollingActiveUsersQuery(projectId, days)+ );+}++export function getRetentionLastSeenSeries({+ projectId,+}: IGetWeekRetentionInput) {+ return runQuery<{+ days: number;+ users: number;+ }>(retentionLastSeenSeriesQuery(projectId));+}++export async function getRollingActiveUsersCore(input: {+ projectId: string;+ days: number;+}) {+ const data = await getRollingActiveUsers(input);+ return {+ window_days: input.days,+ label: ROLLING_ACTIVE_USER_LABELS[input.days] ?? `${input.days}d active`,+ series: data,+ };+}++export async function getWeeklyRetentionSeriesCore(projectId: string) {+ return getRetentionSeries({ projectId });+}++// Weekly active-user retention cohort over the last 12 weeks, computed by the+// unified getRetentionCohort engine. firstEvent/secondEvent are omitted so any+// identified activity counts toward the cohort.+export async function getRetentionCohortCore(projectId: string) {+ const end = DateTime.now();+ const start = end.minus({ weeks: RETENTION_COHORT_WEEKS });+ return getRetentionCohort({+ projectId,+ interval: 'week',+ startDate: start.toFormat('yyyy-MM-dd HH:mm:ss'),+ endDate: end.toFormat('yyyy-MM-dd HH:mm:ss'),+ });+}++export async function getEngagementCore(projectId: string) {+ const raw = await getRetentionLastSeenSeries({ projectId });++ let active_0_7 = 0;+ let active_8_14 = 0;+ let active_15_30 = 0;+ let active_31_60 = 0;+ let churned_60_plus = 0;++ for (const row of raw) {+ if (row.days <= ENGAGEMENT_ACTIVE_DAYS) {+ active_0_7 += row.users;+ } else if (row.days <= ENGAGEMENT_RECENT_DAYS) {+ active_8_14 += row.users;+ } else if (row.days <= ENGAGEMENT_LAPSING_DAYS) {+ active_15_30 += row.users;+ } else if (row.days <= ENGAGEMENT_INACTIVE_DAYS) {+ active_31_60 += row.users;+ } else {+ churned_60_plus += row.users;+ }+ }++ const total =+ active_0_7 + active_8_14 + active_15_30 + active_31_60 + churned_60_plus;++ return {+ summary: {+ total_identified_users: total,+ active_last_7_days: active_0_7,+ active_8_to_14_days: active_8_14,+ active_15_to_30_days: active_15_30,+ inactive_31_to_60_days: active_31_60,+ churned_60_plus_days: churned_60_plus,+ },+ distribution: raw,+ };+}++// ---------------------------------------------------------------------------+// Cohort retention matrix+//+// Definition: every user is assigned to exactly ONE cohort, the interval of+// their FIRST `firstEvent` within the window (first-touch). For each cohort we+// count how many of those users performed `secondEvent` 0..N intervals later.+// - criteria 'on' -> active exactly k intervals after (=)+// - criteria 'on_or_after' -> active at least k intervals after (>=, cumulative)+// When `firstEvent` / `secondEvent` is empty the name filter is dropped and the+// query measures retention on ANY activity (active-user retention).+// ---------------------------------------------------------------------------++const LUXON_UNIT: Record<IRetentionInterval, 'days' | 'weeks' | 'months'> = {+ minute: 'days',+ hour: 'days',+ day: 'days',+ week: 'weeks',+ month: 'months',+};++const CLICKHOUSE_DATE_TIME_LENGTH = 'yyyy-MM-dd HH:mm:ss'.length;+const ISO_DATE_LENGTH = 'yyyy-MM-dd'.length;++// Normalize an ISO or `yyyy-MM-dd HH:mm:ss` string into ClickHouse date-time form.+function utc(date: string) {+ return date.replace('T', ' ').slice(0, CLICKHOUSE_DATE_TIME_LENGTH);+}++// Number of `interval` buckets spanned by [startDate, endDate]; drives the+// number of retention columns (0..diffInterval).+function diffIntervalCount(+ startDate: string,+ endDate: string,+ interval: IRetentionInterval+) {+ const unit = LUXON_UNIT[interval];+ const start = DateTime.fromFormat(utc(startDate), 'yyyy-MM-dd HH:mm:ss', {+ zone: 'utc',+ });+ const end = DateTime.fromFormat(utc(endDate), 'yyyy-MM-dd HH:mm:ss', {+ zone: 'utc',+ });+ return Math.max(0, Math.floor(end.diff(start, unit).as(unit)));+}++export async function getRetentionCohort(input: IGetRetentionCohortInput) {+ const {+ projectId,+ firstEvent,+ secondEvent,+ criteria = 'on_or_after',+ interval = 'day',+ startDate,+ endDate,+ filters = [],+ } = input;++ const diffInterval = diffIntervalCount(startDate, endDate, interval);++ const cohortData = await runQuery<{+ cohort_interval: string;+ total_first_event_count: number;+ [key: string]: number | string;+ }>(+ retentionCohortQuery({+ projectId,+ firstEvent,+ secondEvent,+ criteria,+ interval,+ start: utc(startDate),+ end: utc(endDate),+ diffInterval,+ filters,+ })+ );++ // Reference point for cohort maturity: we only have return data up to "now",+ // so periods that haven't elapsed yet are excluded from the weighted average.+ const until = DateTime.utc().toFormat('yyyy-MM-dd HH:mm:ss');+ return processCohortData(cohortData, diffInterval, interval, until);+}++// Number of fully-elapsed periods between a cohort's start and the reference+// date. Periods beyond this haven't happened yet, so they carry no return data+// and must be excluded from the average. Fails open (treats everything as+// mature) when dates are missing/unparseable.+function maturePeriodCount(+ cohortInterval: string,+ until: string | undefined,+ interval: IRetentionInterval+): number {+ if (!until) {+ return Number.POSITIVE_INFINITY;+ }+ const unit = LUXON_UNIT[interval];+ const cohort = DateTime.fromFormat(+ cohortInterval.slice(0, ISO_DATE_LENGTH),+ 'yyyy-MM-dd',+ {+ zone: 'utc',+ }+ );+ const ref = DateTime.fromFormat(until, 'yyyy-MM-dd HH:mm:ss', {+ zone: 'utc',+ });+ if (!(cohort.isValid && ref.isValid)) {+ return Number.POSITIVE_INFINITY;+ }+ return Math.floor(ref.diff(cohort, unit).as(unit));+}++// Shapes the raw ClickHouse matrix into per-cohort rows + a leading weighted-+// average row.+//+// The average is a maturity-aware pooled rate: for each period column it pools+// only the cohorts old enough to have actually reached that period+// (cohort_interval + N intervals <= `until`). This keeps not-yet-elapsed cells+// from being read as churn AND keeps genuine zeros in, so the late curve is not+// inflated. Counts are normalised to the representative (mean) cohort size, so+// the row is internally consistent: period 0 equals Total profiles and the+// curve starts at 100%.+export function processCohortData(+ data: Array<{+ cohort_interval: string;+ total_first_event_count: number;+ [key: string]: number | string;+ }>,+ diffInterval: number,+ interval: IRetentionInterval = 'day',+ until?: string+): IRetentionCohortRow[] {+ if (data.length === 0) {+ return [];+ }++ const columns = range(0, diffInterval + 1);+ const processed = data.map((row) => {+ const sum = row.total_first_event_count;+ const values = columns.map(+ (index) => (row[`interval_${index}_user_count`] || 0) as number+ );++ return {+ cohort_interval: row.cohort_interval,+ sum,+ values,+ percentages: values.map((value) =>+ sum > 0 ? round(value / sum, RETENTION_PERCENTAGE_DECIMALS) : 0+ ),+ };+ });++ const maturePeriods = processed.map((row) =>+ maturePeriodCount(row.cohort_interval, until, interval)+ );+ const totalSize = processed.reduce((acc, row) => acc + row.sum, 0);+ const representativeSize = round(+ totalSize / processed.length,+ RETENTION_COUNT_DECIMALS+ );++ const averageRow: IRetentionCohortRow = {+ cohort_interval: 'Weighted Average',+ sum: representativeSize,+ values: [],+ percentages: [],+ };++ for (const index of columns) {+ let matureSize = 0;+ let retained = 0;+ processed.forEach((row, i) => {+ if (index <= (maturePeriods[i] as number)) {+ matureSize += row.sum;+ retained += row.values[index] as number;+ }+ });+ const rate = matureSize > 0 ? retained / matureSize : 0;+ averageRow.percentages.push(round(rate, RETENTION_PERCENTAGE_DECIMALS));+ averageRow.values.push(+ round(rate * representativeSize, RETENTION_COUNT_DECIMALS)+ );+ }++ return [averageRow, ...processed];+}diff --git a/packages/core/src/modules/chart/sankey.service.ts b/packages/core/src/modules/chart/sankey.service.tsnew file mode 100644index 000000000..77b84c3b4--- /dev/null+++ b/packages/core/src/modules/chart/sankey.service.ts@@ -0,0 +1,504 @@+// Ported from packages/db/src/services/sankey.service.ts (M7-004). Every+// ClickHouse statement is a `sql` fragment from src/sankey.sql.ts, proven+// byte-equivalent to V1 in src/sankey.sql.proof.md; V1's sankey.service is a+// re-export shim onto this module (DELEGATE PATTERN).++import { chartColors } from '@openpanel/constants';+import { type IChartEventFilter, zChartEvent } from '@openpanel/validation';+import { z } from 'zod';+import { getSettingsForProject } from '../organization/organization.service';+import { getEventFiltersWhereClause } from './src/filter-where';+import { runQuery } from './src/run-query';+import {+ type SankeyEvent,+ type SankeyPathsInput,+ sankeyBetweenTopEntriesQuery,+ sankeyBetweenTransitionsQuery,+ sankeyTopEntriesQuery,+ sankeyTransitionsQuery,+} from './src/sankey.sql';++const DEFAULT_SANKEY_STEPS = 5;+const MIN_SANKEY_STEPS = 2;+const MAX_SANKEY_STEPS = 10;++/** Destinations kept per node when the flow branches. */+const TOP_DESTINATIONS_PER_NODE = 3;+/** Links carrying less than this share of the entry sessions are dropped. */+const MIN_LINK_PERCENT = 0.25;+const PERCENT = 100;+const FIRST_STEP = 1;++export const zGetSankeyInput = z.object({+ projectId: z.string(),+ startDate: z.string(),+ endDate: z.string(),+ steps: z+ .number()+ .min(MIN_SANKEY_STEPS)+ .max(MAX_SANKEY_STEPS)+ .default(DEFAULT_SANKEY_STEPS),+ mode: z.enum(['between', 'after', 'before']),+ startEvent: zChartEvent,+ endEvent: zChartEvent.optional(),+ exclude: z.array(z.string()).default([]),+ include: z.array(z.string()).optional(),+});++export type IGetSankeyInput = z.infer<typeof zGetSankeyInput> & {+ timezone: string;+};++export interface SankeyNode {+ id: string;+ label: string;+ nodeColor: string;+ percentage?: number;+ value?: number;+ step?: number;+}++export interface SankeyLink {+ source: string;+ target: string;+ value: number;+}++export interface SankeyResult {+ nodes: SankeyNode[];+ links: SankeyLink[];+}++/** Every node the builder emits carries all three; the public type keeps V1's optionals. */+type ResolvedSankeyNode = SankeyNode & {+ percentage: number;+ value: number;+ step: number;+};++interface Transition {+ source: string;+ target: string;+ step: number;+ value: number;+}++interface TopEntry {+ entry_event: string;+ count: number;+}++/**+ * The sessions table names the first page `entry_path` / `entry_origin` and+ * flattens `properties.__query.utm_*` into `utm_*` columns, so a filter+ * written against the events vocabulary has to be renamed before it compiles+ * for that table.+ */+export function getRawWhereClause(+ type: 'events' | 'sessions',+ filters: IChartEventFilter[]+): string {+ const where = getEventFiltersWhereClause(+ filters.map((filter) => {+ if (type !== 'sessions') {+ return filter;+ }+ if (filter.name === 'path') {+ return { ...filter, name: 'entry_path' };+ }+ if (filter.name === 'origin') {+ return { ...filter, name: 'entry_origin' };+ }+ if (filter.name.startsWith('properties.__query.utm_')) {+ return {+ ...filter,+ name: filter.name.replace('properties.__query.utm_', 'utm_'),+ };+ }+ return filter;+ })+ );++ return Object.values(where).join(' AND ');+}++function toSankeyEvent(+ event: z.infer<typeof zChartEvent> | undefined+): SankeyEvent | undefined {+ return event+ ? {+ name: event.name,+ whereClause: getRawWhereClause('events', event.filters),+ }+ : undefined;+}++export async function getSankey({+ projectId,+ startDate,+ endDate,+ steps = DEFAULT_SANKEY_STEPS,+ mode,+ startEvent,+ endEvent,+ exclude = [],+ include,+ timezone,+}: IGetSankeyInput): Promise<SankeyResult> {+ const colors = chartColors.map((color) => color.main);+ const pathsInput: SankeyPathsInput = {+ projectId,+ startDate,+ endDate,+ steps,+ mode,+ startEvent: toSankeyEvent(startEvent),+ endEvent: toSankeyEvent(endEvent),+ include,+ exclude,+ };+ const isBetween = mode === 'between' && !!startEvent && !!endEvent;+ const start = pathsInput.startEvent as SankeyEvent;+ const end = pathsInput.endEvent as SankeyEvent;++ const topEntries = await runQuery<TopEntry>(+ isBetween+ ? sankeyBetweenTopEntriesQuery(pathsInput, start, end)+ : sankeyTopEntriesQuery(pathsInput),+ timezone+ );++ if (topEntries.length === 0) {+ return { nodes: [], links: [] };+ }++ const topEntryEvents = topEntries.map((entry) => entry.entry_event);+ const totalSessions = topEntries.reduce((sum, entry) => sum + entry.count, 0);++ const transitions = await runQuery<Transition>(+ isBetween+ ? sankeyBetweenTransitionsQuery(pathsInput, start, end, topEntryEvents)+ : sankeyTransitionsQuery(pathsInput, topEntryEvents),+ timezone+ );++ return buildSankeyFromTransitions(+ transitions,+ topEntries,+ totalSessions,+ steps,+ colors+ );+}++interface NodeData {+ event: string;+ value: number;+ step: number;+ color: string;+}++/** Node ids are per-step so a repeated event does not collapse the flow. */+function getNodeId(event: string, step: number) {+ return `${event}::step${step}`;+}++function groupTransitionsByStep(transitions: Transition[]) {+ const byStep = new Map<number, Transition[]>();+ for (const transition of transitions) {+ const bucket = byStep.get(transition.step);+ if (bucket) {+ bucket.push(transition);+ } else {+ byStep.set(transition.step, [transition]);+ }+ }+ return byStep;+}++/** Walks the flow step by step, keeping the top destinations of each node. */+function expandFlow(+ transitions: Transition[],+ topEntries: TopEntry[],+ steps: number,+ colors: string[]+): { nodes: Map<string, NodeData>; links: SankeyLink[] } {+ const nodes = new Map<string, NodeData>();+ const links: SankeyLink[] = [];+ const transitionsByStep = groupTransitionsByStep(transitions);++ const activeNodes = new Map<string, string>();+ topEntries.forEach((entry, index) => {+ const nodeId = getNodeId(entry.entry_event, FIRST_STEP);+ nodes.set(nodeId, {+ event: entry.entry_event,+ value: entry.count,+ step: FIRST_STEP,+ color: colors[index % colors.length] as string,+ });+ activeNodes.set(entry.entry_event, nodeId);+ });++ for (let step = FIRST_STEP; step < steps; step++) {+ const stepTransitions = transitionsByStep.get(step) || [];+ const nextActiveNodes = new Map<string, string>();++ for (const [sourceEvent, sourceNodeId] of activeNodes) {+ const fromSource = stepTransitions+ .filter((transition) => transition.source === sourceEvent)+ .sort((a, b) => b.value - a.value)+ .slice(0, TOP_DESTINATIONS_PER_NODE);++ for (const transition of fromSource) {+ if (transition.source === transition.target) {+ continue;+ }+ const targetNodeId = getNodeId(transition.target, step + 1);+ links.push({+ source: sourceNodeId,+ target: targetNodeId,+ value: transition.value,+ });++ const existing = nodes.get(targetNodeId);+ if (existing) {+ existing.value += transition.value;+ } else {+ nodes.set(targetNodeId, {+ event: transition.target,+ value: transition.value,+ step: step + 1,+ color:+ nodes.get(sourceNodeId)?.color ||+ (colors[nodes.size % colors.length] as string),+ });+ }+ nextActiveNodes.set(transition.target, targetNodeId);+ }+ }++ activeNodes.clear();+ for (const [event, nodeId] of nextActiveNodes) {+ activeNodes.set(event, nodeId);+ }+ if (activeNodes.size === 0) {+ break;+ }+ }++ return { nodes, links };+}++/** Merges the terminal nodes that carry the same event name into one. */+function mergeFinalNodes(+ finalNodes: ResolvedSankeyNode[],+ validLinks: SankeyLink[],+ totalSessions: number+): SankeyResult {+ const nodesWithOutgoing = new Set(validLinks.map((link) => link.source));+ const finalNodeIds = new Set(+ finalNodes+ .filter((node) => !nodesWithOutgoing.has(node.id))+ .map((node) => node.id)+ );++ const finalNodesByEvent = new Map<string, ResolvedSankeyNode[]>();+ for (const node of finalNodes) {+ if (finalNodeIds.has(node.id)) {+ const bucket = finalNodesByEvent.get(node.label);+ if (bucket) {+ bucket.push(node);+ } else {+ finalNodesByEvent.set(node.label, [node]);+ }+ }+ }++ const nodeIdRemap = new Map<string, string>();+ const mergedNodes = new Map<string, ResolvedSankeyNode>();+ finalNodesByEvent.forEach((nodesToMerge, eventName) => {+ if (nodesToMerge.length <= 1) {+ return;+ }+ const maxStep = Math.max(...nodesToMerge.map((node) => node.step || 0));+ const totalValue = nodesToMerge.reduce(+ (sum, node) => sum + (node.value || 0),+ 0+ );+ const mergedNodeId = `${eventName}::final`;+ const firstNode = nodesToMerge[0] as ResolvedSankeyNode;+ mergedNodes.set(mergedNodeId, {+ id: mergedNodeId,+ label: eventName,+ nodeColor: firstNode.nodeColor,+ percentage: (totalValue / totalSessions) * PERCENT,+ value: totalValue,+ step: maxStep,+ });+ for (const node of nodesToMerge) {+ nodeIdRemap.set(node.id, mergedNodeId);+ }+ });++ const remappedLinks = validLinks.map((link) => ({+ source: nodeIdRemap.get(link.source) || link.source,+ target: nodeIdRemap.get(link.target) || link.target,+ value: link.value,+ }));++ const mergedOldNodeIds = new Set(nodeIdRemap.keys());+ const remainingNodes = finalNodes.filter(+ (node) => !(finalNodeIds.has(node.id) || mergedOldNodeIds.has(node.id))+ );+ const allNodes = [...remainingNodes, ...mergedNodes.values()].sort((a, b) => {+ if (a.step !== b.step) {+ return a.step - b.step;+ }+ return b.value - a.value;+ });++ // Links can now point at the same merged target; sum them.+ const linkMap = new Map<string, number>();+ for (const link of remappedLinks) {+ const key = `${link.source}->${link.target}`;+ linkMap.set(key, (linkMap.get(key) || 0) + link.value);+ }++ const nodeIds = new Set(allNodes.map((node) => node.id));+ const links: SankeyLink[] = [];+ for (const [key, value] of linkMap) {+ const parts = key.split('->');+ if (parts.length !== 2) {+ continue;+ }+ const source = parts[0] as string;+ const target = parts[1] as string;+ if (nodeIds.has(source) && nodeIds.has(target)) {+ links.push({ source, target, value });+ }+ }++ return { nodes: allNodes, links };+}++function buildSankeyFromTransitions(+ transitions: Transition[],+ topEntries: TopEntry[],+ totalSessions: number,+ steps: number,+ colors: string[]+): SankeyResult {+ if (transitions.length === 0) {+ return { nodes: [], links: [] };+ }++ const { nodes, links } = expandFlow(transitions, topEntries, steps, colors);++ const minLinkValue = Math.ceil((totalSessions * MIN_LINK_PERCENT) / PERCENT);+ const filteredLinks = links.filter((link) => link.value >= minLinkValue);++ const referencedNodeIds = new Set<string>();+ for (const link of filteredLinks) {+ referencedNodeIds.add(link.source);+ referencedNodeIds.add(link.target);+ }++ const nodeValuesFromLinks = new Map<string, number>();+ for (const link of filteredLinks) {+ nodeValuesFromLinks.set(+ link.target,+ (nodeValuesFromLinks.get(link.target) || 0) + link.value+ );+ }++ // An entry node only survives if it still has an outgoing link.+ nodes.forEach((nodeData, nodeId) => {+ if (+ nodeData.step === FIRST_STEP &&+ !filteredLinks.some((link) => link.source === nodeId)+ ) {+ referencedNodeIds.delete(nodeId);+ }+ });++ const finalNodes: ResolvedSankeyNode[] = Array.from(nodes.entries())+ .filter(([id]) => referencedNodeIds.has(id))+ .map(([id, data]) => {+ const value =+ data.step === FIRST_STEP+ ? data.value+ : nodeValuesFromLinks.get(id) || data.value;+ return {+ id,+ label: data.event,+ nodeColor: data.color,+ percentage: (value / totalSessions) * PERCENT,+ value,+ step: data.step,+ };+ })+ .sort((a, b) => {+ if (a.step !== b.step) {+ return a.step - b.step;+ }+ return b.value - a.value;+ });++ const nodeIds = new Set(finalNodes.map((node) => node.id));+ const validLinks = filteredLinks.filter(+ (link) => nodeIds.has(link.source) && nodeIds.has(link.target)+ );++ return mergeFinalNodes(finalNodes, validLinks, totalSessions);+}++function toChartEvent(name: string) {+ return {+ id: name,+ name,+ displayName: name,+ type: 'event' as const,+ segment: 'event' as const,+ filters: [],+ };+}++export async function getUserFlowCore(input: {+ projectId: string;+ startDate: string;+ endDate: string;+ startEvent: string;+ endEvent?: string;+ mode: 'after' | 'before' | 'between';+ steps?: number;+ exclude?: string[];+ include?: string[];+}) {+ if (input.mode === 'between' && !input.endEvent) {+ throw new Error('endEvent is required when mode is "between"');+ }++ const { timezone } = await getSettingsForProject(input.projectId);+ const result = await getSankey({+ projectId: input.projectId,+ startDate: input.startDate,+ endDate: input.endDate,+ steps: input.steps ?? DEFAULT_SANKEY_STEPS,+ mode: input.mode,+ startEvent: toChartEvent(input.startEvent),+ endEvent: input.endEvent ? toChartEvent(input.endEvent) : undefined,+ exclude: input.exclude ?? [],+ include: input.include,+ timezone,+ });++ return {+ mode: input.mode,+ startEvent: input.startEvent,+ endEvent: input.endEvent,+ node_count: result.nodes.length,+ link_count: result.links.length,+ nodes: result.nodes,+ links: result.links,+ };+}diff --git a/packages/core/src/modules/chart/src/conversion.sql.proof.md b/packages/core/src/modules/chart/src/conversion.sql.proof.mdnew file mode 100644index 000000000..40497b085--- /dev/null+++ b/packages/core/src/modules/chart/src/conversion.sql.proof.md@@ -0,0 +1,152 @@+# conversion.sql.ts — V1 -> V2 result-set proof (M7-004)++The conversion statement was executed twice against the same data — once as V1 (the `git HEAD` `packages/db/src/services/conversion.service.ts`, `c4760ac2`, captured by wrapping the shared ClickHouse client) and once as V2 (the fragment the core service sends) — through one `@clickhouse/client` with `format: 'JSON'` and the same `session_timezone`. The two responses were compared on `data` (every row, every column, as JSON) and `meta` (column names and types). Both sides were driven by their own service, so the inputs, the breakdown resolution and the cohort lookups are the real ones.++- **Date**: 2026-09-04. **Data**: local prod-copy `openpanel` (319,499,000 events, static). Projects: `secure-privacy`, `verdict`, `bayse` (America/New_York).+- **Machine**: single-node ClickHouse on a 4-vCPU box — timings are directional only; production is 2 shards x 2 replicas (`docs/ENVIRONMENT.md`). The statement contains no `IN (subquery)`; the profile, group and cohort joins keep V1's exact `LEFT ANY JOIN` / `ARRAY JOIN` shape, so no `IN` / `GLOBAL IN` decision is made or unmade here.++- **Verdict**: 12 cases, 0 not identical.++### conversionQuery — no breakdown, day (secure-privacy)++**IDENTICAL** — rows V1/V2 = 7/7; rows_read V1/V2 = 106489/106489; wall V1/V2 = 38 ms / 42 ms; clickhouse_settings: {"session_timezone":"UTC"} (both).++```sql+-- V1+SELECT event_day, uniqExact(session_id) AS total_first, countIf(steps >= 2) AS conversions, round(100.0 * countIf(steps >= 2) / uniqExact(session_id), 2) AS conversion_rate_percentage FROM ( (SELECT session_id, any(toStartOfDay(created_at)) as event_day, windowFunnel(86400)( toDateTime(created_at), events.name = 'session_start', events.name = 'session_end' ) as steps FROM events WHERE project_id = 'secure-privacy' AND events.name IN ('session_start', 'session_end') AND created_at BETWEEN toDateTime('2026-07-06 00:00:00') AND toDateTime('2026-07-12 23:59:59') GROUP BY session_id) ) WHERE steps > 0 GROUP BY event_day ORDER BY event_day ASC+-- V2+SELECT event_day, uniqExact(session_id) AS total_first, countIf(steps >= {p1:UInt64}) AS conversions, round(100.0 * countIf(steps >= {p2:UInt64}) / uniqExact(session_id), {p3:UInt64}) AS conversion_rate_percentage FROM ( (SELECT session_id, any(toStartOfDay(created_at)) as event_day, windowFunnel({p4:UInt64})( toDateTime(created_at), events.name = {p5:String}, events.name = {p6:String} ) as steps FROM events WHERE project_id = {p7:String} AND events.name IN {p8:Array(String)} AND created_at BETWEEN toDateTime({p9:String}) AND toDateTime({p10:String}) GROUP BY session_id) ) WHERE steps > 0 GROUP BY event_day ORDER BY event_day ASC+-- V2 params: {"p1":2,"p2":2,"p3":2,"p4":86400,"p5":"session_start","p6":"session_end","p7":"secure-privacy","p8":["session_start","session_end"],"p9":"2026-07-06 00:00:00","p10":"2026-07-12 23:59:59"}+```++### conversionQuery — breakdown country, day++**IDENTICAL** — rows V1/V2 = 463/463; rows_read V1/V2 = 106489/106489; wall V1/V2 = 55 ms / 38 ms; clickhouse_settings: {"session_timezone":"UTC"} (both).++```sql+-- V1+SELECT event_day, b_0, uniqExact(session_id) AS total_first, countIf(steps >= 2) AS conversions, round(100.0 * countIf(steps >= 2) / uniqExact(session_id), 2) AS conversion_rate_percentage FROM ( (SELECT session_id, any(toStartOfDay(created_at)) as event_day, country as b_0, windowFunnel(86400)( toDateTime(created_at), events.name = 'session_start', events.name = 'session_end' ) as steps FROM events WHERE project_id = 'secure-privacy' AND events.name IN ('session_start', 'session_end') AND created_at BETWEEN toDateTime('2026-07-06 00:00:00') AND toDateTime('2026-07-12 23:59:59') GROUP BY session_id, country) ) WHERE steps > 0 GROUP BY event_day, b_0 ORDER BY event_day ASC, b_0 ASC+-- V2+SELECT event_day, b_0, uniqExact(session_id) AS total_first, countIf(steps >= {p1:UInt64}) AS conversions, round(100.0 * countIf(steps >= {p2:UInt64}) / uniqExact(session_id), {p3:UInt64}) AS conversion_rate_percentage FROM ( (SELECT session_id, any(toStartOfDay(created_at)) as event_day, country as b_0, windowFunnel({p4:UInt64})( toDateTime(created_at), events.name = {p5:String}, events.name = {p6:String} ) as steps FROM events WHERE project_id = {p7:String} AND events.name IN {p8:Array(String)} AND created_at BETWEEN toDateTime({p9:String}) AND toDateTime({p10:String}) GROUP BY session_id, country) ) WHERE steps > 0 GROUP BY event_day, b_0 ORDER BY event_day ASC, b_0 ASC+-- V2 params: {"p1":2,"p2":2,"p3":2,"p4":86400,"p5":"session_start","p6":"session_end","p7":"secure-privacy","p8":["session_start","session_end"],"p9":"2026-07-06 00:00:00","p10":"2026-07-12 23:59:59"}+```++### conversionQuery — profile_id group, week interval, limit 3++**IDENTICAL** — rows V1/V2 = 19/19; rows_read V1/V2 = 155590/155590; wall V1/V2 = 50 ms / 65 ms; clickhouse_settings: {"session_timezone":"UTC"} (both).++```sql+-- V1+SELECT event_day, b_0, uniqExact(profile_id) AS total_first, countIf(steps >= 2) AS conversions, round(100.0 * countIf(steps >= 2) / uniqExact(profile_id), 2) AS conversion_rate_percentage FROM ( (SELECT profile_id, any(toStartOfWeek(toDateTime(created_at))) as event_day, os as b_0, windowFunnel(259200)( toDateTime(created_at), events.name = 'session_start', events.name = 'screen_view' ) as steps FROM events WHERE project_id = 'secure-privacy' AND events.name IN ('session_start', 'screen_view') AND created_at BETWEEN toDateTime('2026-06-01 00:00:00') AND toDateTime('2026-07-12 23:59:59') GROUP BY profile_id, os) ) WHERE steps > 0 GROUP BY event_day, b_0 ORDER BY event_day ASC, b_0 ASC+-- V2+SELECT event_day, b_0, uniqExact(profile_id) AS total_first, countIf(steps >= {p1:UInt64}) AS conversions, round(100.0 * countIf(steps >= {p2:UInt64}) / uniqExact(profile_id), {p3:UInt64}) AS conversion_rate_percentage FROM ( (SELECT profile_id, any(toStartOfWeek(toDateTime(created_at))) as event_day, os as b_0, windowFunnel({p4:UInt64})( toDateTime(created_at), events.name = {p5:String}, events.name = {p6:String} ) as steps FROM events WHERE project_id = {p7:String} AND events.name IN {p8:Array(String)} AND created_at BETWEEN toDateTime({p9:String}) AND toDateTime({p10:String}) GROUP BY profile_id, os) ) WHERE steps > 0 GROUP BY event_day, b_0 ORDER BY event_day ASC, b_0 ASC+-- V2 params: {"p1":2,"p2":2,"p3":2,"p4":259200,"p5":"session_start","p6":"screen_view","p7":"secure-privacy","p8":["session_start","screen_view"],"p9":"2026-06-01 00:00:00","p10":"2026-07-12 23:59:59"}+```++### conversionQuery — step filters (country is US) + breakdown, hour++**IDENTICAL** — rows V1/V2 = 49/49; rows_read V1/V2 = 24572/24572; wall V1/V2 = 16 ms / 17 ms; clickhouse_settings: {"session_timezone":"UTC"} (both).++```sql+-- V1+SELECT event_day, b_0, uniqExact(session_id) AS total_first, countIf(steps >= 2) AS conversions, round(100.0 * countIf(steps >= 2) / uniqExact(session_id), 2) AS conversion_rate_percentage FROM ( (SELECT session_id, any(toStartOfHour(created_at)) as event_day, device as b_0, windowFunnel(86400)( toDateTime(created_at), (events.name = 'screen_view' AND country = 'US'), events.name = 'session_end' ) as steps FROM events WHERE project_id = 'secure-privacy' AND events.name IN ('screen_view', 'session_end') AND created_at BETWEEN toDateTime('2026-07-06 00:00:00') AND toDateTime('2026-07-07 00:00:00') GROUP BY session_id, device) ) WHERE steps > 0 GROUP BY event_day, b_0 ORDER BY event_day ASC, b_0 ASC+-- V2+SELECT event_day, b_0, uniqExact(session_id) AS total_first, countIf(steps >= {p1:UInt64}) AS conversions, round(100.0 * countIf(steps >= {p2:UInt64}) / uniqExact(session_id), {p3:UInt64}) AS conversion_rate_percentage FROM ( (SELECT session_id, any(toStartOfHour(created_at)) as event_day, device as b_0, windowFunnel({p4:UInt64})( toDateTime(created_at), (events.name = {p5:String} AND country = 'US'), events.name = {p6:String} ) as steps FROM events WHERE project_id = {p7:String} AND events.name IN {p8:Array(String)} AND created_at BETWEEN toDateTime({p9:String}) AND toDateTime({p10:String}) GROUP BY session_id, device) ) WHERE steps > 0 GROUP BY event_day, b_0 ORDER BY event_day ASC, b_0 ASC+-- V2 params: {"p1":2,"p2":2,"p3":2,"p4":86400,"p5":"screen_view","p6":"session_end","p7":"secure-privacy","p8":["screen_view","session_end"],"p9":"2026-07-06 00:00:00","p10":"2026-07-07 00:00:00"}+```++### conversionQuery — property breakdown + properties filter, day++**IDENTICAL** — rows V1/V2 = 4/4; rows_read V1/V2 = 65534/65534; wall V1/V2 = 29 ms / 48 ms; clickhouse_settings: {"session_timezone":"UTC"} (both).++```sql+-- V1+SELECT event_day, b_0, uniqExact(session_id) AS total_first, countIf(steps >= 2) AS conversions, round(100.0 * countIf(steps >= 2) / uniqExact(session_id), 2) AS conversion_rate_percentage FROM ( (SELECT session_id, any(toStartOfDay(created_at)) as event_day, properties['__path'] as b_0, windowFunnel(86400)( toDateTime(created_at), (events.name = 'screen_view' AND events.properties['__query.utm_source'] != ''), events.name = 'session_end' ) as steps FROM events WHERE project_id = 'secure-privacy' AND events.name IN ('screen_view', 'session_end') AND created_at BETWEEN toDateTime('2026-07-06 00:00:00') AND toDateTime('2026-07-09 23:59:59') GROUP BY session_id, properties['__path']) ) WHERE steps > 0 GROUP BY event_day, b_0 ORDER BY event_day ASC, b_0 ASC+-- V2+SELECT event_day, b_0, uniqExact(session_id) AS total_first, countIf(steps >= {p1:UInt64}) AS conversions, round(100.0 * countIf(steps >= {p2:UInt64}) / uniqExact(session_id), {p3:UInt64}) AS conversion_rate_percentage FROM ( (SELECT session_id, any(toStartOfDay(created_at)) as event_day, properties['__path'] as b_0, windowFunnel({p4:UInt64})( toDateTime(created_at), (events.name = {p5:String} AND events.properties['__query.utm_source'] != ''), events.name = {p6:String} ) as steps FROM events WHERE project_id = {p7:String} AND events.name IN {p8:Array(String)} AND created_at BETWEEN toDateTime({p9:String}) AND toDateTime({p10:String}) GROUP BY session_id, properties['__path']) ) WHERE steps > 0 GROUP BY event_day, b_0 ORDER BY event_day ASC, b_0 ASC+-- V2 params: {"p1":2,"p2":2,"p3":2,"p4":86400,"p5":"screen_view","p6":"session_end","p7":"secure-privacy","p8":["screen_view","session_end"],"p9":"2026-07-06 00:00:00","p10":"2026-07-09 23:59:59"}+```++### conversionQuery — profile breakdown -> profiles LEFT ANY JOIN, day++**IDENTICAL** — rows V1/V2 = 15/15; rows_read V1/V2 = 376510/376510; wall V1/V2 = 72 ms / 66 ms; clickhouse_settings: {"session_timezone":"UTC"} (both).++```sql+-- V1+SELECT event_day, b_0, uniqExact(session_id) AS total_first, countIf(steps >= 2) AS conversions, round(100.0 * countIf(steps >= 2) / uniqExact(session_id), 2) AS conversion_rate_percentage FROM ( (SELECT session_id, any(toStartOfDay(created_at)) as event_day, profile.email as b_0, windowFunnel(86400)( toDateTime(created_at), events.name = 'session_start', events.name = 'session_end' ) as steps FROM events LEFT ANY JOIN ( SELECT id, email FROM profiles FINAL WHERE project_id = 'secure-privacy' ) as profile ON profile.id = profile_id WHERE project_id = 'secure-privacy' AND events.name IN ('session_start', 'session_end') AND created_at BETWEEN toDateTime('2026-07-06 00:00:00') AND toDateTime('2026-07-09 23:59:59') GROUP BY session_id, profile.email) ) WHERE steps > 0 GROUP BY event_day, b_0 ORDER BY event_day ASC, b_0 ASC+-- V2+SELECT event_day, b_0, uniqExact(session_id) AS total_first, countIf(steps >= {p1:UInt64}) AS conversions, round(100.0 * countIf(steps >= {p2:UInt64}) / uniqExact(session_id), {p3:UInt64}) AS conversion_rate_percentage FROM ( (SELECT session_id, any(toStartOfDay(created_at)) as event_day, profile.email as b_0, windowFunnel({p4:UInt64})( toDateTime(created_at), events.name = {p5:String}, events.name = {p6:String} ) as steps FROM events LEFT ANY JOIN ( SELECT id, email FROM profiles FINAL WHERE project_id = {p7:String} ) as profile ON profile.id = profile_id WHERE project_id = {p8:String} AND events.name IN {p9:Array(String)} AND created_at BETWEEN toDateTime({p10:String}) AND toDateTime({p11:String}) GROUP BY session_id, profile.email) ) WHERE steps > 0 GROUP BY event_day, b_0 ORDER BY event_day ASC, b_0 ASC+-- V2 params: {"p1":2,"p2":2,"p3":2,"p4":86400,"p5":"session_start","p6":"session_end","p7":"secure-privacy","p8":"secure-privacy","p9":["session_start","session_end"],"p10":"2026-07-06 00:00:00","p11":"2026-07-09 23:59:59"}+```++### conversionQuery — profile.properties breakdown -> properties Map join, day++**IDENTICAL** — rows V1/V2 = 4/4; rows_read V1/V2 = 376510/376510; wall V1/V2 = 96 ms / 101 ms; clickhouse_settings: {"session_timezone":"UTC"} (both).++```sql+-- V1+SELECT event_day, b_0, uniqExact(session_id) AS total_first, countIf(steps >= 2) AS conversions, round(100.0 * countIf(steps >= 2) / uniqExact(session_id), 2) AS conversion_rate_percentage FROM ( (SELECT session_id, any(toStartOfDay(created_at)) as event_day, profile.properties['plan'] as b_0, windowFunnel(86400)( toDateTime(created_at), events.name = 'session_start', events.name = 'session_end' ) as steps FROM events LEFT ANY JOIN ( SELECT id, properties FROM profiles FINAL WHERE project_id = 'secure-privacy' ) as profile ON profile.id = profile_id WHERE project_id = 'secure-privacy' AND events.name IN ('session_start', 'session_end') AND created_at BETWEEN toDateTime('2026-07-06 00:00:00') AND toDateTime('2026-07-09 23:59:59') GROUP BY session_id, profile.properties['plan']) ) WHERE steps > 0 GROUP BY event_day, b_0 ORDER BY event_day ASC, b_0 ASC+-- V2+SELECT event_day, b_0, uniqExact(session_id) AS total_first, countIf(steps >= {p1:UInt64}) AS conversions, round(100.0 * countIf(steps >= {p2:UInt64}) / uniqExact(session_id), {p3:UInt64}) AS conversion_rate_percentage FROM ( (SELECT session_id, any(toStartOfDay(created_at)) as event_day, profile.properties['plan'] as b_0, windowFunnel({p4:UInt64})( toDateTime(created_at), events.name = {p5:String}, events.name = {p6:String} ) as steps FROM events LEFT ANY JOIN ( SELECT id, properties FROM profiles FINAL WHERE project_id = {p7:String} ) as profile ON profile.id = profile_id WHERE project_id = {p8:String} AND events.name IN {p9:Array(String)} AND created_at BETWEEN toDateTime({p10:String}) AND toDateTime({p11:String}) GROUP BY session_id, profile.properties['plan']) ) WHERE steps > 0 GROUP BY event_day, b_0 ORDER BY event_day ASC, b_0 ASC+-- V2 params: {"p1":2,"p2":2,"p3":2,"p4":86400,"p5":"session_start","p6":"session_end","p7":"secure-privacy","p8":"secure-privacy","p9":["session_start","session_end"],"p10":"2026-07-06 00:00:00","p11":"2026-07-09 23:59:59"}+```++### conversionQuery — group breakdown -> ARRAY JOIN groups, day++**IDENTICAL** — rows V1/V2 = 14/14; rows_read V1/V2 = 68893/68893; wall V1/V2 = 35 ms / 43 ms; clickhouse_settings: {"session_timezone":"UTC"} (both).++```sql+-- V1+SELECT event_day, b_0, uniqExact(session_id) AS total_first, countIf(steps >= 2) AS conversions, round(100.0 * countIf(steps >= 2) / uniqExact(session_id), 2) AS conversion_rate_percentage FROM ( (SELECT session_id, any(toStartOfDay(created_at)) as event_day, _group_id as b_0, windowFunnel(86400)( toDateTime(created_at), events.name = 'session_start', events.name = 'session_end' ) as steps FROM events ARRAY JOIN groups AS _group_id LEFT ANY JOIN (SELECT id, name, type, properties FROM groups FINAL WHERE project_id = 'secure-privacy') AS _g ON _g.id = _group_id WHERE project_id = 'secure-privacy' AND events.name IN ('session_start', 'session_end') AND created_at BETWEEN toDateTime('2026-07-06 00:00:00') AND toDateTime('2026-07-09 23:59:59') GROUP BY session_id, _group_id) ) WHERE steps > 0 GROUP BY event_day, b_0 ORDER BY event_day ASC, b_0 ASC+-- V2+SELECT event_day, b_0, uniqExact(session_id) AS total_first, countIf(steps >= {p1:UInt64}) AS conversions, round(100.0 * countIf(steps >= {p2:UInt64}) / uniqExact(session_id), {p3:UInt64}) AS conversion_rate_percentage FROM ( (SELECT session_id, any(toStartOfDay(created_at)) as event_day, _group_id as b_0, windowFunnel({p4:UInt64})( toDateTime(created_at), events.name = {p5:String}, events.name = {p6:String} ) as steps FROM events ARRAY JOIN groups AS _group_id LEFT ANY JOIN (SELECT id, name, type, properties FROM groups FINAL WHERE project_id = {p7:String}) AS _g ON _g.id = _group_id WHERE project_id = {p8:String} AND events.name IN {p9:Array(String)} AND created_at BETWEEN toDateTime({p10:String}) AND toDateTime({p11:String}) GROUP BY session_id, _group_id) ) WHERE steps > 0 GROUP BY event_day, b_0 ORDER BY event_day ASC, b_0 ASC+-- V2 params: {"p1":2,"p2":2,"p3":2,"p4":86400,"p5":"session_start","p6":"session_end","p7":"secure-privacy","p8":"secure-privacy","p9":["session_start","session_end"],"p10":"2026-07-06 00:00:00","p11":"2026-07-09 23:59:59"}+```++### conversionQuery — unknown breakdown dropped, minute interval++**IDENTICAL** — rows V1/V2 = 116/116; rows_read V1/V2 = 16381/16381; wall V1/V2 = 15 ms / 15 ms; clickhouse_settings: {"session_timezone":"UTC"} (both).++```sql+-- V1+SELECT event_day, uniqExact(session_id) AS total_first, countIf(steps >= 2) AS conversions, round(100.0 * countIf(steps >= 2) / uniqExact(session_id), 2) AS conversion_rate_percentage FROM ( (SELECT session_id, any(toStartOfMinute(created_at)) as event_day, windowFunnel(86400)( toDateTime(created_at), events.name = 'session_start', events.name = 'session_end' ) as steps FROM events WHERE project_id = 'secure-privacy' AND events.name IN ('session_start', 'session_end') AND created_at BETWEEN toDateTime('2026-07-06 00:00:00') AND toDateTime('2026-07-06 02:00:00') GROUP BY session_id) ) WHERE steps > 0 GROUP BY event_day ORDER BY event_day ASC+-- V2+SELECT event_day, uniqExact(session_id) AS total_first, countIf(steps >= {p1:UInt64}) AS conversions, round(100.0 * countIf(steps >= {p2:UInt64}) / uniqExact(session_id), {p3:UInt64}) AS conversion_rate_percentage FROM ( (SELECT session_id, any(toStartOfMinute(created_at)) as event_day, windowFunnel({p4:UInt64})( toDateTime(created_at), events.name = {p5:String}, events.name = {p6:String} ) as steps FROM events WHERE project_id = {p7:String} AND events.name IN {p8:Array(String)} AND created_at BETWEEN toDateTime({p9:String}) AND toDateTime({p10:String}) GROUP BY session_id) ) WHERE steps > 0 GROUP BY event_day ORDER BY event_day ASC+-- V2 params: {"p1":2,"p2":2,"p3":2,"p4":86400,"p5":"session_start","p6":"session_end","p7":"secure-privacy","p8":["session_start","session_end"],"p9":"2026-07-06 00:00:00","p10":"2026-07-06 02:00:00"}+```++### conversionQuery — month interval, verdict++**IDENTICAL** — rows V1/V2 = 1/1; rows_read V1/V2 = 24600197/24600197; wall V1/V2 = 626 ms / 576 ms; clickhouse_settings: {"session_timezone":"UTC"} (both).++```sql+-- V1+SELECT event_day, uniqExact(session_id) AS total_first, countIf(steps >= 2) AS conversions, round(100.0 * countIf(steps >= 2) / uniqExact(session_id), 2) AS conversion_rate_percentage FROM ( (SELECT session_id, any(toStartOfMonth(toDateTime(created_at))) as event_day, windowFunnel(86400)( toDateTime(created_at), events.name = 'view_lobby_item', events.name = 'view_picks' ) as steps FROM events WHERE project_id = 'verdict' AND events.name IN ('view_lobby_item', 'view_picks') AND created_at BETWEEN toDateTime('2026-05-01 00:00:00') AND toDateTime('2026-07-31 23:59:59') GROUP BY session_id) ) WHERE steps > 0 GROUP BY event_day ORDER BY event_day ASC+-- V2+SELECT event_day, uniqExact(session_id) AS total_first, countIf(steps >= {p1:UInt64}) AS conversions, round(100.0 * countIf(steps >= {p2:UInt64}) / uniqExact(session_id), {p3:UInt64}) AS conversion_rate_percentage FROM ( (SELECT session_id, any(toStartOfMonth(toDateTime(created_at))) as event_day, windowFunnel({p4:UInt64})( toDateTime(created_at), events.name = {p5:String}, events.name = {p6:String} ) as steps FROM events WHERE project_id = {p7:String} AND events.name IN {p8:Array(String)} AND created_at BETWEEN toDateTime({p9:String}) AND toDateTime({p10:String}) GROUP BY session_id) ) WHERE steps > 0 GROUP BY event_day ORDER BY event_day ASC+-- V2 params: {"p1":2,"p2":2,"p3":2,"p4":86400,"p5":"view_lobby_item","p6":"view_picks","p7":"verdict","p8":["view_lobby_item","view_picks"],"p9":"2026-05-01 00:00:00","p10":"2026-07-31 23:59:59"}+```++### conversionQuery — non-UTC session timezone (America/New_York), day++**IDENTICAL** — rows V1/V2 = 7/7; rows_read V1/V2 = 7945887/7945887; wall V1/V2 = 145 ms / 143 ms; clickhouse_settings: {"session_timezone":"America/New_York"} (both).++```sql+-- V1+SELECT event_day, uniqExact(session_id) AS total_first, countIf(steps >= 2) AS conversions, round(100.0 * countIf(steps >= 2) / uniqExact(session_id), 2) AS conversion_rate_percentage FROM ( (SELECT session_id, any(toStartOfDay(created_at)) as event_day, windowFunnel(86400)( toDateTime(created_at), events.name = 'session_start', events.name = 'session_end' ) as steps FROM events WHERE project_id = 'bayse' AND events.name IN ('session_start', 'session_end') AND created_at BETWEEN toDateTime('2026-07-06 00:00:00') AND toDateTime('2026-07-12 23:59:59') GROUP BY session_id) ) WHERE steps > 0 GROUP BY event_day ORDER BY event_day ASC+-- V2+SELECT event_day, uniqExact(session_id) AS total_first, countIf(steps >= {p1:UInt64}) AS conversions, round(100.0 * countIf(steps >= {p2:UInt64}) / uniqExact(session_id), {p3:UInt64}) AS conversion_rate_percentage FROM ( (SELECT session_id, any(toStartOfDay(created_at)) as event_day, windowFunnel({p4:UInt64})( toDateTime(created_at), events.name = {p5:String}, events.name = {p6:String} ) as steps FROM events WHERE project_id = {p7:String} AND events.name IN {p8:Array(String)} AND created_at BETWEEN toDateTime({p9:String}) AND toDateTime({p10:String}) GROUP BY session_id) ) WHERE steps > 0 GROUP BY event_day ORDER BY event_day ASC+-- V2 params: {"p1":2,"p2":2,"p3":2,"p4":86400,"p5":"session_start","p6":"session_end","p7":"bayse","p8":["session_start","session_end"],"p9":"2026-07-06 00:00:00","p10":"2026-07-12 23:59:59"}+```++### conversionQuery — globalFilters merged into both steps, day++**IDENTICAL** — rows V1/V2 = 4/4; rows_read V1/V2 = 65534/65534; wall V1/V2 = 25 ms / 35 ms; clickhouse_settings: {"session_timezone":"UTC"} (both).++```sql+-- V1+SELECT event_day, uniqExact(session_id) AS total_first, countIf(steps >= 2) AS conversions, round(100.0 * countIf(steps >= 2) / uniqExact(session_id), 2) AS conversion_rate_percentage FROM ( (SELECT session_id, any(toStartOfDay(created_at)) as event_day, windowFunnel(86400)( toDateTime(created_at), (events.name = 'session_start' AND os = 'iOS'), (events.name = 'session_end' AND os = 'iOS') ) as steps FROM events WHERE project_id = 'secure-privacy' AND events.name IN ('session_start', 'session_end') AND created_at BETWEEN toDateTime('2026-07-06 00:00:00') AND toDateTime('2026-07-09 23:59:59') GROUP BY session_id) ) WHERE steps > 0 GROUP BY event_day ORDER BY event_day ASC+-- V2+SELECT event_day, uniqExact(session_id) AS total_first, countIf(steps >= {p1:UInt64}) AS conversions, round(100.0 * countIf(steps >= {p2:UInt64}) / uniqExact(session_id), {p3:UInt64}) AS conversion_rate_percentage FROM ( (SELECT session_id, any(toStartOfDay(created_at)) as event_day, windowFunnel({p4:UInt64})( toDateTime(created_at), (events.name = {p5:String} AND os = 'iOS'), (events.name = {p6:String} AND os = 'iOS') ) as steps FROM events WHERE project_id = {p7:String} AND events.name IN {p8:Array(String)} AND created_at BETWEEN toDateTime({p9:String}) AND toDateTime({p10:String}) GROUP BY session_id) ) WHERE steps > 0 GROUP BY event_day ORDER BY event_day ASC+-- V2 params: {"p1":2,"p2":2,"p3":2,"p4":86400,"p5":"session_start","p6":"session_end","p7":"secure-privacy","p8":["session_start","session_end"],"p9":"2026-07-06 00:00:00","p10":"2026-07-09 23:59:59"}+```diff --git a/packages/core/src/modules/chart/src/conversion.sql.ts b/packages/core/src/modules/chart/src/conversion.sql.tsnew file mode 100644index 000000000..0ee5a8d42--- /dev/null+++ b/packages/core/src/modules/chart/src/conversion.sql.ts@@ -0,0 +1,157 @@+// The conversion statement as a pure `sql` fragment (ADR-013). Converted 1:1+// from packages/db/src/services/conversion.service.ts (M7-004): the SQL text+// is V1's clix output, with the project id, event names, dates, the funnel+// window and the interval bucket bound as `{pN:Type}` parameters instead of+// inlined literals. Result-set proof: conversion.sql.proof.md.+//+// The field resolver and filter compiler still render text (see compiled.ts);+// their output — breakdown expressions, per-step filter clauses and the+// cohort joins — is spliced, everything else is bound.+//+// Cluster note (docs/ENVIRONMENT.md): `events`, `profiles` and `groups` are+// Distributed on Cloud. The profile / group / cohort joins keep V1's exact+// LEFT ANY JOIN shape and run under the client's+// `distributed_product_mode: 'allow'` as before; no `IN (subquery)` is+// introduced or removed.++import { type SqlFragment, sql } from '@openpanel/db/src/clickhouse/sql';+import type { IInterval } from '@openpanel/validation';+import { compiledText } from './compiled';+import { CHART_TABLE } from './field-resolution';++/** windowFunnel counts a conversion when both steps fired. */+const CONVERTED_STEP_COUNT = 2;+const CONVERSION_RATE_DECIMALS = 2;++/** The two grouping keys the funnel may be computed per — a closed set. */+const CONVERSION_GROUPS = ['session_id', 'profile_id'] as const;+export type ConversionGroup = (typeof CONVERSION_GROUPS)[number];++/** V1 `clix.toStartOf(created_at, interval)` — no timezone argument. */+function intervalBucket(interval: IInterval): SqlFragment {+ switch (interval) {+ case 'minute':+ return sql`toStartOfMinute(created_at)`;+ case 'hour':+ return sql`toStartOfHour(created_at)`;+ case 'week':+ return sql`toStartOfWeek(toDateTime(created_at))`;+ case 'month':+ return sql`toStartOfMonth(toDateTime(created_at))`;+ default:+ return sql`toStartOfDay(created_at)`;+ }+}++export interface ConversionQueryInput {+ projectId: string;+ startDate: string;+ endDate: string;+ interval: IInterval;+ group: ConversionGroup;+ /** Funnel window in seconds. */+ funnelWindowSeconds: number;+ firstEventName: string;+ secondEventName: string;+ /** Compiled filter text per step (`''` when the step has no filters). */+ firstEventWhere: string;+ secondEventWhere: string;+ /** Compiled `<expr> as b_<index>` breakdown selects. */+ breakdownSelects: string[];+ /** Compiled breakdown expressions, as written in the inner GROUP BY. */+ breakdownExpressions: string[];+ /** Profile columns to select in the LEFT ANY JOIN, or [] for no join. */+ profileJoinFields: string[];+ needsGroupArrayJoin: boolean;+ /** Compiled cohort join clauses. */+ cohortJoins: string[];+}++function stepCondition(name: string, where: string): SqlFragment {+ return where+ ? sql`(events.name = ${sql.string(name)} AND ${compiledText(where)})`+ : sql`events.name = ${sql.string(name)}`;+}++function profileJoin(fields: string[], projectId: string): SqlFragment {+ if (fields.length === 0) {+ return sql.empty;+ }+ // Simple column names (not aliased) so profile.properties works directly.+ return sql`LEFT ANY JOIN (+ SELECT ${sql.join(fields.map((field) => sql.id(field)))}+ FROM ${sql.id(CHART_TABLE.profiles)} FINAL+ WHERE project_id = ${sql.string(projectId)}+ ) as profile ON profile.id = profile_id`;+}++function groupJoin(needed: boolean, projectId: string): SqlFragment {+ return needed+ ? sql`ARRAY JOIN groups AS _group_id LEFT ANY JOIN (SELECT id, name, type, properties FROM ${sql.id(CHART_TABLE.groups)} FINAL WHERE project_id = ${sql.string(projectId)}) AS _g ON _g.id = _group_id`+ : sql.empty;+}++/** V1 `ConversionService.getConversion`: one windowFunnel scan per bucket. */+export function conversionQuery(input: ConversionQueryInput): SqlFragment {+ const {+ projectId,+ startDate,+ endDate,+ interval,+ group,+ funnelWindowSeconds,+ firstEventName,+ secondEventName,+ firstEventWhere,+ secondEventWhere,+ breakdownSelects,+ breakdownExpressions,+ profileJoinFields,+ needsGroupArrayJoin,+ cohortJoins,+ } = input;++ const groupKey = sql.id(group, CONVERSION_GROUPS);+ const breakdownAliases = breakdownSelects.map((_, index) =>+ compiledText(`b_${index}`)+ );+ const outerKeys = [sql`event_day`, ...breakdownAliases];+ const converted = sql`countIf(steps >= ${sql.uint64(CONVERTED_STEP_COUNT)})`;++ const outerSelect = sql.join([+ sql`event_day`,+ ...breakdownAliases,+ sql`uniqExact(${groupKey}) AS total_first`,+ sql`${converted} AS conversions`,+ sql`round(100.0 * ${converted} / uniqExact(${groupKey}), ${sql.uint64(CONVERSION_RATE_DECIMALS)}) AS conversion_rate_percentage`,+ ]);++ const innerBreakdownSelects =+ breakdownSelects.length > 0+ ? sql`${sql.join(breakdownSelects.map(compiledText))},`+ : sql.empty;+ const innerBreakdownGroupBy =+ breakdownExpressions.length > 0+ ? sql`, ${sql.join(breakdownExpressions.map(compiledText))}`+ : sql.empty;++ return sql`SELECT ${outerSelect} FROM (+ (SELECT+ ${groupKey},+ any(${intervalBucket(interval)}) as event_day,+ ${innerBreakdownSelects}+ windowFunnel(${sql.uint64(funnelWindowSeconds)})(+ toDateTime(created_at),+ ${stepCondition(firstEventName, firstEventWhere)},+ ${stepCondition(secondEventName, secondEventWhere)}+ ) as steps+ FROM ${sql.id(CHART_TABLE.events)}+ ${profileJoin(profileJoinFields, projectId)}+ ${groupJoin(needsGroupArrayJoin, projectId)}+ ${sql.join(cohortJoins.map(compiledText), ' ')}+ WHERE project_id = ${sql.string(projectId)}+ AND events.name IN ${sql.array('String', [firstEventName, secondEventName])}+ AND created_at BETWEEN toDateTime(${sql.string(startDate)}) AND toDateTime(${sql.string(endDate)})+ GROUP BY ${groupKey}${innerBreakdownGroupBy})+ ) WHERE steps > 0 GROUP BY ${sql.join(outerKeys)} ORDER BY ${sql.join(outerKeys.map((key) => sql`${key} ASC`))}`;+}diff --git a/packages/core/src/modules/chart/src/field-resolution.ts b/packages/core/src/modules/chart/src/field-resolution.tsindex beb8424cb..d29e48636 100644--- a/packages/core/src/modules/chart/src/field-resolution.ts+++ b/packages/core/src/modules/chart/src/field-resolution.ts@@ -24,6 +24,8 @@ export const CHART_TABLE = {cohortMembers: 'cohort_members',eventNamesMv: 'distinct_event_names_mv',eventPropertyValuesMv: 'event_property_values_mv',+ dauMv: 'dau_mv',+ cohortEventsMv: 'cohort_events_mv',} as const;// Top-level columns on the events table. Derived from the migration indiff --git a/packages/core/src/modules/chart/src/funnel.sql.proof.md b/packages/core/src/modules/chart/src/funnel.sql.proof.mdnew file mode 100644index 000000000..066f80a9d--- /dev/null+++ b/packages/core/src/modules/chart/src/funnel.sql.proof.md@@ -0,0 +1,261 @@+# funnel.sql.ts — V1 -> V2 result-set proof (M7-004)++Both funnel statements were executed twice against the same data — once as V1 (the `git HEAD` `packages/db/src/services/funnel.service.ts` plus the `getFunnelProfiles` composition from `chart.service.ts`, `c4760ac2`, captured by wrapping the shared ClickHouse client) and once as V2 (the fragment the core builders return) — through one `@clickhouse/client` with `format: 'JSON'` and the same `session_timezone`. The two responses were compared on `data` (every row, every column, as JSON) and `meta` (column names and types).++- **Date**: 2026-09-04. **Data**: local prod-copy `openpanel` (319,499,000 events, static). Projects: `secure-privacy`, `verdict`, `bayse` (America/New_York), `dream-mate` (Australia/Sydney).+- **Machine**: single-node ClickHouse on a 4-vCPU box — timings are directional only; production is 2 shards x 2 replicas (`docs/ENVIRONMENT.md`). The statements contain no `IN (subquery)`; the profiles `LEFT JOIN`, the groups `ARRAY JOIN` and the cohort joins keep V1's exact shape, so no `IN` / `GLOBAL IN` decision is made or unmade here.+- **Note on `LIMIT 1000`**: the profile-list cases that return 1000 rows are capped by the statement's own `LIMIT` (V1's `FUNNEL_PROFILES_LIMIT`), and `SELECT DISTINCT ... LIMIT` carries no `ORDER BY` on either side; those cases came back with identical row *sequences* anyway, which is why they read IDENTICAL rather than IDENTICAL AS SET.++- **Verdict**: 21 cases, 0 not identical.++### funnelChartQuery — two steps, no breakdown (golden insights-funnel-two-steps shape)++**IDENTICAL** — rows V1/V2 = 2/2; rows_read V1/V2 = 7470531/7470531; wall V1/V2 = 198 ms / 199 ms; clickhouse_settings: {"session_timezone":"UTC"} (both).++```sql+-- V1+WITH session_funnel AS (SELECT session_id, windowFunnel(86400000, 'strict_increase')(toUInt64(toUnixTimestamp64Milli(created_at)), events.name = 'view_lobby_item', events.name = 'view_picks') AS level, argMax(profile_id, created_at) AS profile_id FROM events WHERE project_id = 'verdict' AND created_at BETWEEN toDateTime('2026-07-06 00:00:00') AND toDateTime('2026-07-12 23:59:59') AND events.name IN ('view_lobby_item', 'view_picks') AND ((events.name = 'view_lobby_item') OR (events.name = 'view_picks')) GROUP BY session_id), funnel AS (SELECT * FROM session_funnel WHERE level != 0) SELECT level, count() as count FROM funnel GROUP BY level ORDER BY level DESC+-- V2+WITH session_funnel AS (SELECT session_id, windowFunnel({p1:UInt64}, 'strict_increase')(toUInt64(toUnixTimestamp64Milli(created_at)), events.name = {p2:String}, events.name = {p3:String}) AS level, argMax(profile_id, created_at) AS profile_id FROM events WHERE project_id = {p4:String} AND created_at BETWEEN toDateTime({p5:String}) AND toDateTime({p6:String}) AND events.name IN {p7:Array(String)} AND ((events.name = {p8:String}) OR (events.name = {p9:String})) GROUP BY session_id), funnel AS (SELECT * FROM session_funnel WHERE level != 0) SELECT level, count() as count FROM funnel GROUP BY level ORDER BY level DESC+-- V2 params: {"p1":86400000,"p2":"view_lobby_item","p3":"view_picks","p4":"verdict","p5":"2026-07-06 00:00:00","p6":"2026-07-12 23:59:59","p7":["view_lobby_item","view_picks"],"p8":"view_lobby_item","p9":"view_picks"}+```++### funnelChartQuery — three steps, profile_id group, 72h window (golden shape)++**IDENTICAL** — rows V1/V2 = 2/2; rows_read V1/V2 = 106489/106489; wall V1/V2 = 34 ms / 36 ms; clickhouse_settings: {"session_timezone":"UTC"} (both).++```sql+-- V1+WITH session_funnel AS (SELECT profile_id, windowFunnel(259200000, 'strict_increase')(toUInt64(toUnixTimestamp64Milli(created_at)), events.name = 'session_start', events.name = 'screen_view', events.name = 'session_end') AS level FROM events WHERE project_id = 'secure-privacy' AND created_at BETWEEN toDateTime('2026-07-06 00:00:00') AND toDateTime('2026-07-12 23:59:59') AND events.name IN ('session_start', 'screen_view', 'session_end') AND ((events.name = 'session_start') OR (events.name = 'screen_view') OR (events.name = 'session_end')) GROUP BY profile_id), funnel AS (SELECT * FROM session_funnel WHERE level != 0) SELECT level, count() as count FROM funnel GROUP BY level ORDER BY level DESC+-- V2+WITH session_funnel AS (SELECT profile_id, windowFunnel({p1:UInt64}, 'strict_increase')(toUInt64(toUnixTimestamp64Milli(created_at)), events.name = {p2:String}, events.name = {p3:String}, events.name = {p4:String}) AS level FROM events WHERE project_id = {p5:String} AND created_at BETWEEN toDateTime({p6:String}) AND toDateTime({p7:String}) AND events.name IN {p8:Array(String)} AND ((events.name = {p9:String}) OR (events.name = {p10:String}) OR (events.name = {p11:String})) GROUP BY profile_id), funnel AS (SELECT * FROM session_funnel WHERE level != 0) SELECT level, count() as count FROM funnel GROUP BY level ORDER BY level DESC+-- V2 params: {"p1":259200000,"p2":"session_start","p3":"screen_view","p4":"session_end","p5":"secure-privacy","p6":"2026-07-06 00:00:00","p7":"2026-07-12 23:59:59","p8":["session_start","screen_view","session_end"],"p9":"session_start","p10":"screen_view","p11":"session_end"}+```++### funnelChartQuery — breakdown country (argMinIf attribution)++**IDENTICAL** — rows V1/V2 = 118/118; rows_read V1/V2 = 106489/106489; wall V1/V2 = 43 ms / 28 ms; clickhouse_settings: {"session_timezone":"UTC"} (both).++```sql+-- V1+WITH session_funnel AS (SELECT session_id, windowFunnel(86400000, 'strict_increase')(toUInt64(toUnixTimestamp64Milli(created_at)), events.name = 'session_start', events.name = 'screen_view') AS level, argMax(profile_id, created_at) AS profile_id, argMinIf(country, created_at, events.name = 'session_start') as b_0 FROM events WHERE project_id = 'secure-privacy' AND created_at BETWEEN toDateTime('2026-07-06 00:00:00') AND toDateTime('2026-07-12 23:59:59') AND events.name IN ('session_start', 'screen_view') AND ((events.name = 'session_start') OR (events.name = 'screen_view')) GROUP BY session_id), funnel AS (SELECT * FROM session_funnel WHERE level != 0) SELECT level, b_0, count() as count FROM funnel GROUP BY level, b_0 ORDER BY level DESC+-- V2+WITH session_funnel AS (SELECT session_id, windowFunnel({p1:UInt64}, 'strict_increase')(toUInt64(toUnixTimestamp64Milli(created_at)), events.name = {p2:String}, events.name = {p3:String}) AS level, argMax(profile_id, created_at) AS profile_id, argMinIf(country, created_at, events.name = {p4:String}) as b_0 FROM events WHERE project_id = {p5:String} AND created_at BETWEEN toDateTime({p6:String}) AND toDateTime({p7:String}) AND events.name IN {p8:Array(String)} AND ((events.name = {p9:String}) OR (events.name = {p10:String})) GROUP BY session_id), funnel AS (SELECT * FROM session_funnel WHERE level != 0) SELECT level, b_0, count() as count FROM funnel GROUP BY level, b_0 ORDER BY level DESC+-- V2 params: {"p1":86400000,"p2":"session_start","p3":"screen_view","p4":"session_start","p5":"secure-privacy","p6":"2026-07-06 00:00:00","p7":"2026-07-12 23:59:59","p8":["session_start","screen_view"],"p9":"session_start","p10":"screen_view"}+```++### funnelChartQuery — two breakdowns (country + os)++**IDENTICAL** — rows V1/V2 = 284/284; rows_read V1/V2 = 65534/65534; wall V1/V2 = 24 ms / 24 ms; clickhouse_settings: {"session_timezone":"UTC"} (both).++```sql+-- V1+WITH session_funnel AS (SELECT session_id, windowFunnel(86400000, 'strict_increase')(toUInt64(toUnixTimestamp64Milli(created_at)), events.name = 'session_start', events.name = 'session_end') AS level, argMax(profile_id, created_at) AS profile_id, argMinIf(country, created_at, events.name = 'session_start') as b_0, argMinIf(os, created_at, events.name = 'session_start') as b_1 FROM events WHERE project_id = 'secure-privacy' AND created_at BETWEEN toDateTime('2026-07-06 00:00:00') AND toDateTime('2026-07-09 23:59:59') AND events.name IN ('session_start', 'session_end') AND ((events.name = 'session_start') OR (events.name = 'session_end')) GROUP BY session_id), funnel AS (SELECT * FROM session_funnel WHERE level != 0) SELECT level, b_0, b_1, count() as count FROM funnel GROUP BY level, b_0, b_1 ORDER BY level DESC+-- V2+WITH session_funnel AS (SELECT session_id, windowFunnel({p1:UInt64}, 'strict_increase')(toUInt64(toUnixTimestamp64Milli(created_at)), events.name = {p2:String}, events.name = {p3:String}) AS level, argMax(profile_id, created_at) AS profile_id, argMinIf(country, created_at, events.name = {p4:String}) as b_0, argMinIf(os, created_at, events.name = {p5:String}) as b_1 FROM events WHERE project_id = {p6:String} AND created_at BETWEEN toDateTime({p7:String}) AND toDateTime({p8:String}) AND events.name IN {p9:Array(String)} AND ((events.name = {p10:String}) OR (events.name = {p11:String})) GROUP BY session_id), funnel AS (SELECT * FROM session_funnel WHERE level != 0) SELECT level, b_0, b_1, count() as count FROM funnel GROUP BY level, b_0, b_1 ORDER BY level DESC+-- V2 params: {"p1":86400000,"p2":"session_start","p3":"session_end","p4":"session_start","p5":"session_start","p6":"secure-privacy","p7":"2026-07-06 00:00:00","p8":"2026-07-09 23:59:59","p9":["session_start","session_end"],"p10":"session_start","p11":"session_end"}+```++### funnelChartQuery — step filters + step pre-filter++**IDENTICAL** — rows V1/V2 = 2/2; rows_read V1/V2 = 73724/73724; wall V1/V2 = 21 ms / 42 ms; clickhouse_settings: {"session_timezone":"UTC"} (both).++```sql+-- V1+WITH session_funnel AS (SELECT session_id, windowFunnel(86400000, 'strict_increase')(toUInt64(toUnixTimestamp64Milli(created_at)), country = 'US' AND events.name = 'screen_view', events.name = 'session_end') AS level, argMax(profile_id, created_at) AS profile_id FROM events WHERE project_id = 'secure-privacy' AND created_at BETWEEN toDateTime('2026-07-06 00:00:00') AND toDateTime('2026-07-09 23:59:59') AND events.name IN ('screen_view', 'session_end') AND ((country = 'US' AND events.name = 'screen_view') OR (events.name = 'session_end')) GROUP BY session_id), funnel AS (SELECT * FROM session_funnel WHERE level != 0) SELECT level, count() as count FROM funnel GROUP BY level ORDER BY level DESC+-- V2+WITH session_funnel AS (SELECT session_id, windowFunnel({p1:UInt64}, 'strict_increase')(toUInt64(toUnixTimestamp64Milli(created_at)), country = 'US' AND events.name = {p2:String}, events.name = {p3:String}) AS level, argMax(profile_id, created_at) AS profile_id FROM events WHERE project_id = {p4:String} AND created_at BETWEEN toDateTime({p5:String}) AND toDateTime({p6:String}) AND events.name IN {p7:Array(String)} AND ((country = 'US' AND events.name = {p8:String}) OR (events.name = {p9:String})) GROUP BY session_id), funnel AS (SELECT * FROM session_funnel WHERE level != 0) SELECT level, count() as count FROM funnel GROUP BY level ORDER BY level DESC+-- V2 params: {"p1":86400000,"p2":"screen_view","p3":"session_end","p4":"secure-privacy","p5":"2026-07-06 00:00:00","p6":"2026-07-09 23:59:59","p7":["screen_view","session_end"],"p8":"screen_view","p9":"session_end"}+```++### funnelChartQuery — profile.* filter -> profiles LEFT JOIN++**IDENTICAL** — rows V1/V2 = 1/1; rows_read V1/V2 = 376510/376510; wall V1/V2 = 76 ms / 68 ms; clickhouse_settings: {"session_timezone":"UTC"} (both).++```sql+-- V1+WITH session_funnel AS (SELECT session_id, windowFunnel(86400000, 'strict_increase')(toUInt64(toUnixTimestamp64Milli(created_at)), events.name = 'session_start', events.name = 'session_end') AS level, argMax(profile_id, created_at) AS profile_id FROM events LEFT JOIN (SELECT id, email FROM profiles FINAL WHERE project_id = 'secure-privacy') as profile ON profile.id = events.profile_id WHERE project_id = 'secure-privacy' AND created_at BETWEEN toDateTime('2026-07-06 00:00:00') AND toDateTime('2026-07-09 23:59:59') AND events.name IN ('session_start', 'session_end') AND ((events.name = 'session_start') OR (events.name = 'session_end')) GROUP BY session_id), funnel AS (SELECT * FROM session_funnel WHERE level != 0) SELECT level, count() as count FROM funnel GROUP BY level ORDER BY level DESC+-- V2+WITH session_funnel AS (SELECT session_id, windowFunnel({p1:UInt64}, 'strict_increase')(toUInt64(toUnixTimestamp64Milli(created_at)), events.name = {p2:String}, events.name = {p3:String}) AS level, argMax(profile_id, created_at) AS profile_id FROM events LEFT JOIN (SELECT id, email FROM profiles FINAL WHERE project_id = {p4:String}) as profile ON profile.id = events.profile_id WHERE project_id = {p5:String} AND created_at BETWEEN toDateTime({p6:String}) AND toDateTime({p7:String}) AND events.name IN {p8:Array(String)} AND ((events.name = {p9:String}) OR (events.name = {p10:String})) GROUP BY session_id), funnel AS (SELECT * FROM session_funnel WHERE level != 0) SELECT level, count() as count FROM funnel GROUP BY level ORDER BY level DESC+-- V2 params: {"p1":86400000,"p2":"session_start","p3":"session_end","p4":"secure-privacy","p5":"secure-privacy","p6":"2026-07-06 00:00:00","p7":"2026-07-09 23:59:59","p8":["session_start","session_end"],"p9":"session_start","p10":"session_end"}+```++### funnelChartQuery — profile.properties breakdown -> narrowed properties CTE++**IDENTICAL** — rows V1/V2 = 1/1; rows_read V1/V2 = 376510/376510; wall V1/V2 = 103 ms / 102 ms; clickhouse_settings: {"session_timezone":"UTC"} (both).++```sql+-- V1+WITH session_funnel AS (SELECT session_id, windowFunnel(86400000, 'strict_increase')(toUInt64(toUnixTimestamp64Milli(created_at)), events.name = 'session_start', events.name = 'session_end') AS level, argMax(profile_id, created_at) AS profile_id, argMinIf(`profile.properties.plan`, created_at, events.name = 'session_start') as b_0 FROM events LEFT JOIN (SELECT id, properties['plan'] as `profile.properties.plan` FROM profiles FINAL WHERE project_id = 'secure-privacy') as profile ON profile.id = events.profile_id WHERE project_id = 'secure-privacy' AND created_at BETWEEN toDateTime('2026-07-06 00:00:00') AND toDateTime('2026-07-09 23:59:59') AND events.name IN ('session_start', 'session_end') AND ((events.name = 'session_start') OR (events.name = 'session_end')) GROUP BY session_id), funnel AS (SELECT * FROM session_funnel WHERE level != 0) SELECT level, b_0, count() as count FROM funnel GROUP BY level, b_0 ORDER BY level DESC+-- V2+WITH session_funnel AS (SELECT session_id, windowFunnel({p1:UInt64}, 'strict_increase')(toUInt64(toUnixTimestamp64Milli(created_at)), events.name = {p2:String}, events.name = {p3:String}) AS level, argMax(profile_id, created_at) AS profile_id, argMinIf(`profile.properties.plan`, created_at, events.name = {p4:String}) as b_0 FROM events LEFT JOIN (SELECT id, properties['plan'] as `profile.properties.plan` FROM profiles FINAL WHERE project_id = {p5:String}) as profile ON profile.id = events.profile_id WHERE project_id = {p6:String} AND created_at BETWEEN toDateTime({p7:String}) AND toDateTime({p8:String}) AND events.name IN {p9:Array(String)} AND ((events.name = {p10:String}) OR (events.name = {p11:String})) GROUP BY session_id), funnel AS (SELECT * FROM session_funnel WHERE level != 0) SELECT level, b_0, count() as count FROM funnel GROUP BY level, b_0 ORDER BY level DESC+-- V2 params: {"p1":86400000,"p2":"session_start","p3":"session_end","p4":"session_start","p5":"secure-privacy","p6":"secure-privacy","p7":"2026-07-06 00:00:00","p8":"2026-07-09 23:59:59","p9":["session_start","session_end"],"p10":"session_start","p11":"session_end"}+```++### funnelChartQuery — group.* breakdown -> ARRAY JOIN + per-row GROUP BY++**IDENTICAL** — rows V1/V2 = 8/8; rows_read V1/V2 = 68893/68893; wall V1/V2 = 33 ms / 36 ms; clickhouse_settings: {"session_timezone":"UTC"} (both).++```sql+-- V1+WITH _g AS (SELECT id, name, type, properties FROM groups FINAL WHERE project_id = 'secure-privacy'), session_funnel AS (SELECT session_id, windowFunnel(86400000, 'strict_increase')(toUInt64(toUnixTimestamp64Milli(created_at)), events.name = 'session_start', events.name = 'session_end') AS level, argMax(profile_id, created_at) AS profile_id, _group_id as b_0 FROM events ARRAY JOIN groups AS _group_id LEFT ANY JOIN _g ON _g.id = _group_id WHERE project_id = 'secure-privacy' AND created_at BETWEEN toDateTime('2026-07-06 00:00:00') AND toDateTime('2026-07-09 23:59:59') AND events.name IN ('session_start', 'session_end') AND ((events.name = 'session_start') OR (events.name = 'session_end')) GROUP BY session_id, b_0), funnel AS (SELECT * FROM session_funnel WHERE level != 0) SELECT level, b_0, count() as count FROM funnel GROUP BY level, b_0 ORDER BY level DESC+-- V2+WITH _g AS (SELECT id, name, type, properties FROM groups FINAL WHERE project_id = {p1:String}), session_funnel AS (SELECT session_id, windowFunnel({p2:UInt64}, 'strict_increase')(toUInt64(toUnixTimestamp64Milli(created_at)), events.name = {p3:String}, events.name = {p4:String}) AS level, argMax(profile_id, created_at) AS profile_id, _group_id as b_0 FROM events ARRAY JOIN groups AS _group_id LEFT ANY JOIN _g ON _g.id = _group_id WHERE project_id = {p5:String} AND created_at BETWEEN toDateTime({p6:String}) AND toDateTime({p7:String}) AND events.name IN {p8:Array(String)} AND ((events.name = {p9:String}) OR (events.name = {p10:String})) GROUP BY session_id, b_0), funnel AS (SELECT * FROM session_funnel WHERE level != 0) SELECT level, b_0, count() as count FROM funnel GROUP BY level, b_0 ORDER BY level DESC+-- V2 params: {"p1":"secure-privacy","p2":86400000,"p3":"session_start","p4":"session_end","p5":"secure-privacy","p6":"2026-07-06 00:00:00","p7":"2026-07-09 23:59:59","p8":["session_start","session_end"],"p9":"session_start","p10":"session_end"}+```++### funnelChartQuery — funnelGroup "group" forces the groups ARRAY JOIN++**IDENTICAL** — rows V1/V2 = 1/1; rows_read V1/V2 = 68893/68893; wall V1/V2 = 27 ms / 38 ms; clickhouse_settings: {"session_timezone":"UTC"} (both).++```sql+-- V1+WITH _g AS (SELECT id, name, type, properties FROM groups FINAL WHERE project_id = 'secure-privacy'), session_funnel AS (SELECT session_id, windowFunnel(86400000, 'strict_increase')(toUInt64(toUnixTimestamp64Milli(created_at)), events.name = 'session_start', events.name = 'session_end') AS level, argMax(profile_id, created_at) AS profile_id FROM events ARRAY JOIN groups AS _group_id LEFT ANY JOIN _g ON _g.id = _group_id WHERE project_id = 'secure-privacy' AND created_at BETWEEN toDateTime('2026-07-06 00:00:00') AND toDateTime('2026-07-09 23:59:59') AND events.name IN ('session_start', 'session_end') AND ((events.name = 'session_start') OR (events.name = 'session_end')) GROUP BY session_id), funnel AS (SELECT * FROM session_funnel WHERE level != 0) SELECT level, count() as count FROM funnel GROUP BY level ORDER BY level DESC+-- V2+WITH _g AS (SELECT id, name, type, properties FROM groups FINAL WHERE project_id = {p1:String}), session_funnel AS (SELECT session_id, windowFunnel({p2:UInt64}, 'strict_increase')(toUInt64(toUnixTimestamp64Milli(created_at)), events.name = {p3:String}, events.name = {p4:String}) AS level, argMax(profile_id, created_at) AS profile_id FROM events ARRAY JOIN groups AS _group_id LEFT ANY JOIN _g ON _g.id = _group_id WHERE project_id = {p5:String} AND created_at BETWEEN toDateTime({p6:String}) AND toDateTime({p7:String}) AND events.name IN {p8:Array(String)} AND ((events.name = {p9:String}) OR (events.name = {p10:String})) GROUP BY session_id), funnel AS (SELECT * FROM session_funnel WHERE level != 0) SELECT level, count() as count FROM funnel GROUP BY level ORDER BY level DESC+-- V2 params: {"p1":"secure-privacy","p2":86400000,"p3":"session_start","p4":"session_end","p5":"secure-privacy","p6":"2026-07-06 00:00:00","p7":"2026-07-09 23:59:59","p8":["session_start","session_end"],"p9":"session_start","p10":"session_end"}+```++### funnelChartQuery — properties.* filter + properties.* breakdown++**IDENTICAL** — rows V1/V2 = 2/2; rows_read V1/V2 = 65534/65534; wall V1/V2 = 29 ms / 29 ms; clickhouse_settings: {"session_timezone":"UTC"} (both).++```sql+-- V1+WITH session_funnel AS (SELECT session_id, windowFunnel(86400000, 'strict_increase')(toUInt64(toUnixTimestamp64Milli(created_at)), events.properties['__query.utm_source'] != '' AND events.name = 'screen_view', events.name = 'session_end') AS level, argMax(profile_id, created_at) AS profile_id, argMinIf(properties['__path'], created_at, events.properties['__query.utm_source'] != '' AND events.name = 'screen_view') as b_0 FROM events WHERE project_id = 'secure-privacy' AND created_at BETWEEN toDateTime('2026-07-06 00:00:00') AND toDateTime('2026-07-09 23:59:59') AND events.name IN ('screen_view', 'session_end') AND ((events.properties['__query.utm_source'] != '' AND events.name = 'screen_view') OR (events.name = 'session_end')) GROUP BY session_id), funnel AS (SELECT * FROM session_funnel WHERE level != 0) SELECT level, b_0, count() as count FROM funnel GROUP BY level, b_0 ORDER BY level DESC+-- V2+WITH session_funnel AS (SELECT session_id, windowFunnel({p1:UInt64}, 'strict_increase')(toUInt64(toUnixTimestamp64Milli(created_at)), events.properties['__query.utm_source'] != '' AND events.name = {p2:String}, events.name = {p3:String}) AS level, argMax(profile_id, created_at) AS profile_id, argMinIf(properties['__path'], created_at, events.properties['__query.utm_source'] != '' AND events.name = {p4:String}) as b_0 FROM events WHERE project_id = {p5:String} AND created_at BETWEEN toDateTime({p6:String}) AND toDateTime({p7:String}) AND events.name IN {p8:Array(String)} AND ((events.properties['__query.utm_source'] != '' AND events.name = {p9:String}) OR (events.name = {p10:String})) GROUP BY session_id), funnel AS (SELECT * FROM session_funnel WHERE level != 0) SELECT level, b_0, count() as count FROM funnel GROUP BY level, b_0 ORDER BY level DESC+-- V2 params: {"p1":86400000,"p2":"screen_view","p3":"session_end","p4":"screen_view","p5":"secure-privacy","p6":"2026-07-06 00:00:00","p7":"2026-07-09 23:59:59","p8":["screen_view","session_end"],"p9":"screen_view","p10":"session_end"}+```++### funnelChartQuery — unknown + all-cohorts breakdowns dropped++**IDENTICAL** — rows V1/V2 = 1/1; rows_read V1/V2 = 65534/65534; wall V1/V2 = 33 ms / 34 ms; clickhouse_settings: {"session_timezone":"UTC"} (both).++```sql+-- V1+WITH session_funnel AS (SELECT session_id, windowFunnel(86400000, 'strict_increase')(toUInt64(toUnixTimestamp64Milli(created_at)), events.name = 'session_start', events.name = 'session_end') AS level, argMax(profile_id, created_at) AS profile_id FROM events WHERE project_id = 'secure-privacy' AND created_at BETWEEN toDateTime('2026-07-06 00:00:00') AND toDateTime('2026-07-09 23:59:59') AND events.name IN ('session_start', 'session_end') AND ((events.name = 'session_start') OR (events.name = 'session_end')) GROUP BY session_id), funnel AS (SELECT * FROM session_funnel WHERE level != 0) SELECT level, count() as count FROM funnel GROUP BY level ORDER BY level DESC+-- V2+WITH session_funnel AS (SELECT session_id, windowFunnel({p1:UInt64}, 'strict_increase')(toUInt64(toUnixTimestamp64Milli(created_at)), events.name = {p2:String}, events.name = {p3:String}) AS level, argMax(profile_id, created_at) AS profile_id FROM events WHERE project_id = {p4:String} AND created_at BETWEEN toDateTime({p5:String}) AND toDateTime({p6:String}) AND events.name IN {p7:Array(String)} AND ((events.name = {p8:String}) OR (events.name = {p9:String})) GROUP BY session_id), funnel AS (SELECT * FROM session_funnel WHERE level != 0) SELECT level, count() as count FROM funnel GROUP BY level ORDER BY level DESC+-- V2 params: {"p1":86400000,"p2":"session_start","p3":"session_end","p4":"secure-privacy","p5":"2026-07-06 00:00:00","p6":"2026-07-09 23:59:59","p7":["session_start","session_end"],"p8":"session_start","p9":"session_end"}+```++### funnelChartQuery — globalFilters merged into every step++**IDENTICAL** — rows V1/V2 = 1/1; rows_read V1/V2 = 81916/81916; wall V1/V2 = 26 ms / 28 ms; clickhouse_settings: {"session_timezone":"UTC"} (both).++```sql+-- V1+WITH session_funnel AS (SELECT session_id, windowFunnel(86400000, 'strict_increase')(toUInt64(toUnixTimestamp64Milli(created_at)), os = 'iOS' AND events.name = 'session_start', os = 'iOS' AND events.name = 'session_end') AS level, argMax(profile_id, created_at) AS profile_id FROM events WHERE project_id = 'secure-privacy' AND created_at BETWEEN toDateTime('2026-07-06 00:00:00') AND toDateTime('2026-07-09 23:59:59') AND events.name IN ('session_start', 'session_end') AND ((os = 'iOS' AND events.name = 'session_start') OR (os = 'iOS' AND events.name = 'session_end')) GROUP BY session_id), funnel AS (SELECT * FROM session_funnel WHERE level != 0) SELECT level, count() as count FROM funnel GROUP BY level ORDER BY level DESC+-- V2+WITH session_funnel AS (SELECT session_id, windowFunnel({p1:UInt64}, 'strict_increase')(toUInt64(toUnixTimestamp64Milli(created_at)), os = 'iOS' AND events.name = {p2:String}, os = 'iOS' AND events.name = {p3:String}) AS level, argMax(profile_id, created_at) AS profile_id FROM events WHERE project_id = {p4:String} AND created_at BETWEEN toDateTime({p5:String}) AND toDateTime({p6:String}) AND events.name IN {p7:Array(String)} AND ((os = 'iOS' AND events.name = {p8:String}) OR (os = 'iOS' AND events.name = {p9:String})) GROUP BY session_id), funnel AS (SELECT * FROM session_funnel WHERE level != 0) SELECT level, count() as count FROM funnel GROUP BY level ORDER BY level DESC+-- V2 params: {"p1":86400000,"p2":"session_start","p3":"session_end","p4":"secure-privacy","p5":"2026-07-06 00:00:00","p6":"2026-07-09 23:59:59","p7":["session_start","session_end"],"p8":"session_start","p9":"session_end"}+```++### funnelChartQuery — non-UTC session timezone (America/New_York)++**IDENTICAL** — rows V1/V2 = 2/2; rows_read V1/V2 = 7945887/7945887; wall V1/V2 = 146 ms / 161 ms; clickhouse_settings: {"session_timezone":"America/New_York"} (both).++```sql+-- V1+WITH session_funnel AS (SELECT session_id, windowFunnel(86400000, 'strict_increase')(toUInt64(toUnixTimestamp64Milli(created_at)), events.name = 'session_start', events.name = 'screen_view') AS level, argMax(profile_id, created_at) AS profile_id FROM events WHERE project_id = 'bayse' AND created_at BETWEEN toDateTime('2026-07-06 00:00:00') AND toDateTime('2026-07-12 23:59:59') AND events.name IN ('session_start', 'screen_view') AND ((events.name = 'session_start') OR (events.name = 'screen_view')) GROUP BY session_id), funnel AS (SELECT * FROM session_funnel WHERE level != 0) SELECT level, count() as count FROM funnel GROUP BY level ORDER BY level DESC+-- V2+WITH session_funnel AS (SELECT session_id, windowFunnel({p1:UInt64}, 'strict_increase')(toUInt64(toUnixTimestamp64Milli(created_at)), events.name = {p2:String}, events.name = {p3:String}) AS level, argMax(profile_id, created_at) AS profile_id FROM events WHERE project_id = {p4:String} AND created_at BETWEEN toDateTime({p5:String}) AND toDateTime({p6:String}) AND events.name IN {p7:Array(String)} AND ((events.name = {p8:String}) OR (events.name = {p9:String})) GROUP BY session_id), funnel AS (SELECT * FROM session_funnel WHERE level != 0) SELECT level, count() as count FROM funnel GROUP BY level ORDER BY level DESC+-- V2 params: {"p1":86400000,"p2":"session_start","p3":"screen_view","p4":"bayse","p5":"2026-07-06 00:00:00","p6":"2026-07-12 23:59:59","p7":["session_start","screen_view"],"p8":"session_start","p9":"screen_view"}+```++### funnelChartQuery — Sydney DST window, 168h (golden insights-funnel-syd-dst shape)++**IDENTICAL** — rows V1/V2 = 2/2; rows_read V1/V2 = 7962156/7962156; wall V1/V2 = 217 ms / 229 ms; clickhouse_settings: {"session_timezone":"Australia/Sydney"} (both).++```sql+-- V1+WITH session_funnel AS (SELECT session_id, windowFunnel(604800000, 'strict_increase')(toUInt64(toUnixTimestamp64Milli(created_at)), events.name = 'screen_view', events.name = 'send_message') AS level, argMax(profile_id, created_at) AS profile_id FROM events WHERE project_id = 'dream-mate' AND created_at BETWEEN toDateTime('2026-08-01 00:00:00') AND toDateTime('2026-10-31 23:59:59') AND events.name IN ('screen_view', 'send_message') AND ((events.name = 'screen_view') OR (events.name = 'send_message')) GROUP BY session_id), funnel AS (SELECT * FROM session_funnel WHERE level != 0) SELECT level, count() as count FROM funnel GROUP BY level ORDER BY level DESC+-- V2+WITH session_funnel AS (SELECT session_id, windowFunnel({p1:UInt64}, 'strict_increase')(toUInt64(toUnixTimestamp64Milli(created_at)), events.name = {p2:String}, events.name = {p3:String}) AS level, argMax(profile_id, created_at) AS profile_id FROM events WHERE project_id = {p4:String} AND created_at BETWEEN toDateTime({p5:String}) AND toDateTime({p6:String}) AND events.name IN {p7:Array(String)} AND ((events.name = {p8:String}) OR (events.name = {p9:String})) GROUP BY session_id), funnel AS (SELECT * FROM session_funnel WHERE level != 0) SELECT level, count() as count FROM funnel GROUP BY level ORDER BY level DESC+-- V2 params: {"p1":604800000,"p2":"screen_view","p3":"send_message","p4":"dream-mate","p5":"2026-08-01 00:00:00","p6":"2026-10-31 23:59:59","p7":["screen_view","send_message"],"p8":"screen_view","p9":"send_message"}+```++### funnelChartQuery — four steps, limit 3, breakdown device++**IDENTICAL** — rows V1/V2 = 3/3; rows_read V1/V2 = 73724/73724; wall V1/V2 = 31 ms / 27 ms; clickhouse_settings: {"session_timezone":"UTC"} (both).++```sql+-- V1+WITH session_funnel AS (SELECT session_id, windowFunnel(86400000, 'strict_increase')(toUInt64(toUnixTimestamp64Milli(created_at)), events.name = 'session_start', events.name = 'screen_view', events.name = 'screen_view', events.name = 'session_end') AS level, argMax(profile_id, created_at) AS profile_id, argMinIf(device, created_at, events.name = 'session_start') as b_0 FROM events WHERE project_id = 'secure-privacy' AND created_at BETWEEN toDateTime('2026-07-06 00:00:00') AND toDateTime('2026-07-09 23:59:59') AND events.name IN ('session_start', 'screen_view', 'screen_view', 'session_end') AND ((events.name = 'session_start') OR (events.name = 'screen_view') OR (events.name = 'screen_view') OR (events.name = 'session_end')) GROUP BY session_id), funnel AS (SELECT * FROM session_funnel WHERE level != 0) SELECT level, b_0, count() as count FROM funnel GROUP BY level, b_0 ORDER BY level DESC+-- V2+WITH session_funnel AS (SELECT session_id, windowFunnel({p1:UInt64}, 'strict_increase')(toUInt64(toUnixTimestamp64Milli(created_at)), events.name = {p2:String}, events.name = {p3:String}, events.name = {p4:String}, events.name = {p5:String}) AS level, argMax(profile_id, created_at) AS profile_id, argMinIf(device, created_at, events.name = {p6:String}) as b_0 FROM events WHERE project_id = {p7:String} AND created_at BETWEEN toDateTime({p8:String}) AND toDateTime({p9:String}) AND events.name IN {p10:Array(String)} AND ((events.name = {p11:String}) OR (events.name = {p12:String}) OR (events.name = {p13:String}) OR (events.name = {p14:String})) GROUP BY session_id), funnel AS (SELECT * FROM session_funnel WHERE level != 0) SELECT level, b_0, count() as count FROM funnel GROUP BY level, b_0 ORDER BY level DESC+-- V2 params: {"p1":86400000,"p2":"session_start","p3":"screen_view","p4":"screen_view","p5":"session_end","p6":"session_start","p7":"secure-privacy","p8":"2026-07-06 00:00:00","p9":"2026-07-09 23:59:59","p10":["session_start","screen_view","screen_view","session_end"],"p11":"session_start","p12":"screen_view","p13":"screen_view","p14":"session_end"}+```++### funnelProfilesQuery — level >= 2, no breakdown++**IDENTICAL** — rows V1/V2 = 1000/1000; rows_read V1/V2 = 65534/65534; wall V1/V2 = 49 ms / 32 ms; clickhouse_settings: {"session_timezone":"UTC"} (both).++```sql+-- V1+WITH session_funnel AS (SELECT session_id, windowFunnel(86400000, 'strict_increase')(toUInt64(toUnixTimestamp64Milli(created_at)), events.name = 'session_start', events.name = 'session_end') AS level, argMax(profile_id, created_at) AS profile_id FROM events WHERE project_id = 'secure-privacy' AND created_at BETWEEN toDateTime('2026-07-06 00:00:00') AND toDateTime('2026-07-09 23:59:59') AND events.name IN ('session_start', 'session_end') AND ((events.name = 'session_start') OR (events.name = 'session_end')) GROUP BY session_id), funnel AS (SELECT * FROM session_funnel WHERE level != 0) SELECT DISTINCT profile_id FROM funnel WHERE level >= 2 LIMIT 1000+-- V2+WITH session_funnel AS (SELECT session_id, windowFunnel({p1:UInt64}, 'strict_increase')(toUInt64(toUnixTimestamp64Milli(created_at)), events.name = {p2:String}, events.name = {p3:String}) AS level, argMax(profile_id, created_at) AS profile_id FROM events WHERE project_id = {p4:String} AND created_at BETWEEN toDateTime({p5:String}) AND toDateTime({p6:String}) AND events.name IN {p7:Array(String)} AND ((events.name = {p8:String}) OR (events.name = {p9:String})) GROUP BY session_id), funnel AS (SELECT * FROM session_funnel WHERE level != 0) SELECT DISTINCT profile_id FROM funnel WHERE level >= {p10:UInt64} LIMIT {p11:UInt64}+-- V2 params: {"p1":86400000,"p2":"session_start","p3":"session_end","p4":"secure-privacy","p5":"2026-07-06 00:00:00","p6":"2026-07-09 23:59:59","p7":["session_start","session_end"],"p8":"session_start","p9":"session_end","p10":2,"p11":1000}+```++### funnelProfilesQuery — level = 1 dropoffs (rare second step), no breakdown++**IDENTICAL** — rows V1/V2 = 1000/1000; rows_read V1/V2 = 65534/65534; wall V1/V2 = 29 ms / 43 ms; clickhouse_settings: {"session_timezone":"UTC"} (both).++```sql+-- V1+WITH session_funnel AS (SELECT session_id, windowFunnel(86400000, 'strict_increase')(toUInt64(toUnixTimestamp64Milli(created_at)), events.name = 'session_start', events.name = 'link_out') AS level, argMax(profile_id, created_at) AS profile_id FROM events WHERE project_id = 'secure-privacy' AND created_at BETWEEN toDateTime('2026-07-06 00:00:00') AND toDateTime('2026-07-09 23:59:59') AND events.name IN ('session_start', 'link_out') AND ((events.name = 'session_start') OR (events.name = 'link_out')) GROUP BY session_id), funnel AS (SELECT * FROM session_funnel WHERE level != 0) SELECT DISTINCT profile_id FROM funnel WHERE level = 1 LIMIT 1000+-- V2+WITH session_funnel AS (SELECT session_id, windowFunnel({p1:UInt64}, 'strict_increase')(toUInt64(toUnixTimestamp64Milli(created_at)), events.name = {p2:String}, events.name = {p3:String}) AS level, argMax(profile_id, created_at) AS profile_id FROM events WHERE project_id = {p4:String} AND created_at BETWEEN toDateTime({p5:String}) AND toDateTime({p6:String}) AND events.name IN {p7:Array(String)} AND ((events.name = {p8:String}) OR (events.name = {p9:String})) GROUP BY session_id), funnel AS (SELECT * FROM session_funnel WHERE level != 0) SELECT DISTINCT profile_id FROM funnel WHERE level = {p10:UInt64} LIMIT {p11:UInt64}+-- V2 params: {"p1":86400000,"p2":"session_start","p3":"link_out","p4":"secure-privacy","p5":"2026-07-06 00:00:00","p6":"2026-07-09 23:59:59","p7":["session_start","link_out"],"p8":"session_start","p9":"link_out","p10":1,"p11":1000}+```++### funnelProfilesQuery — level >= 2, breakdown value "US"++**IDENTICAL** — rows V1/V2 = 1000/1000; rows_read V1/V2 = 65534/65534; wall V1/V2 = 45 ms / 31 ms; clickhouse_settings: {"session_timezone":"UTC"} (both).++```sql+-- V1+WITH session_funnel AS (SELECT session_id, windowFunnel(86400000, 'strict_increase')(toUInt64(toUnixTimestamp64Milli(created_at)), events.name = 'session_start', events.name = 'session_end') AS level, argMax(profile_id, created_at) AS profile_id, argMinIf(country, created_at, events.name = 'session_start') as b_0 FROM events WHERE project_id = 'secure-privacy' AND created_at BETWEEN toDateTime('2026-07-06 00:00:00') AND toDateTime('2026-07-09 23:59:59') AND events.name IN ('session_start', 'session_end') AND ((events.name = 'session_start') OR (events.name = 'session_end')) GROUP BY session_id), funnel AS (SELECT * FROM session_funnel WHERE level != 0) SELECT DISTINCT profile_id FROM funnel WHERE level >= 2 AND trim(ifNull(toString(b_0), '')) = 'US' LIMIT 1000+-- V2+WITH session_funnel AS (SELECT session_id, windowFunnel({p1:UInt64}, 'strict_increase')(toUInt64(toUnixTimestamp64Milli(created_at)), events.name = {p2:String}, events.name = {p3:String}) AS level, argMax(profile_id, created_at) AS profile_id, argMinIf(country, created_at, events.name = {p4:String}) as b_0 FROM events WHERE project_id = {p5:String} AND created_at BETWEEN toDateTime({p6:String}) AND toDateTime({p7:String}) AND events.name IN {p8:Array(String)} AND ((events.name = {p9:String}) OR (events.name = {p10:String})) GROUP BY session_id), funnel AS (SELECT * FROM session_funnel WHERE level != 0) SELECT DISTINCT profile_id FROM funnel WHERE level >= {p11:UInt64} AND trim(ifNull(toString(b_0), '')) = {p12:String} LIMIT {p13:UInt64}+-- V2 params: {"p1":86400000,"p2":"session_start","p3":"session_end","p4":"session_start","p5":"secure-privacy","p6":"2026-07-06 00:00:00","p7":"2026-07-09 23:59:59","p8":["session_start","session_end"],"p9":"session_start","p10":"session_end","p11":2,"p12":"US","p13":1000}+```++### funnelProfilesQuery — level >= 2, breakdown value "Not set" (empty-bucket match)++**IDENTICAL** — rows V1/V2 = 1000/1000; rows_read V1/V2 = 65534/65534; wall V1/V2 = 51 ms / 56 ms; clickhouse_settings: {"session_timezone":"UTC"} (both).++```sql+-- V1+WITH session_funnel AS (SELECT session_id, windowFunnel(86400000, 'strict_increase')(toUInt64(toUnixTimestamp64Milli(created_at)), events.name = 'session_start', events.name = 'session_end') AS level, argMax(profile_id, created_at) AS profile_id, argMinIf(properties['__query.utm_source'], created_at, events.name = 'session_start') as b_0 FROM events WHERE project_id = 'secure-privacy' AND created_at BETWEEN toDateTime('2026-07-06 00:00:00') AND toDateTime('2026-07-09 23:59:59') AND events.name IN ('session_start', 'session_end') AND ((events.name = 'session_start') OR (events.name = 'session_end')) GROUP BY session_id), funnel AS (SELECT * FROM session_funnel WHERE level != 0) SELECT DISTINCT profile_id FROM funnel WHERE level >= 2 AND (trim(ifNull(toString(b_0), '')) = '' OR trim(ifNull(toString(b_0), '')) = 'Not set') LIMIT 1000+-- V2+WITH session_funnel AS (SELECT session_id, windowFunnel({p1:UInt64}, 'strict_increase')(toUInt64(toUnixTimestamp64Milli(created_at)), events.name = {p2:String}, events.name = {p3:String}) AS level, argMax(profile_id, created_at) AS profile_id, argMinIf(properties['__query.utm_source'], created_at, events.name = {p4:String}) as b_0 FROM events WHERE project_id = {p5:String} AND created_at BETWEEN toDateTime({p6:String}) AND toDateTime({p7:String}) AND events.name IN {p8:Array(String)} AND ((events.name = {p9:String}) OR (events.name = {p10:String})) GROUP BY session_id), funnel AS (SELECT * FROM session_funnel WHERE level != 0) SELECT DISTINCT profile_id FROM funnel WHERE level >= {p11:UInt64} AND (trim(ifNull(toString(b_0), '')) = '' OR trim(ifNull(toString(b_0), '')) = {p12:String}) LIMIT {p13:UInt64}+-- V2 params: {"p1":86400000,"p2":"session_start","p3":"session_end","p4":"session_start","p5":"secure-privacy","p6":"2026-07-06 00:00:00","p7":"2026-07-09 23:59:59","p8":["session_start","session_end"],"p9":"session_start","p10":"session_end","p11":2,"p12":"Not set","p13":1000}+```++### funnelProfilesQuery — profile.properties breakdown, profile_id group++**IDENTICAL** — rows V1/V2 = 1000/1000; rows_read V1/V2 = 376510/376510; wall V1/V2 = 87 ms / 67 ms; clickhouse_settings: {"session_timezone":"UTC"} (both).++```sql+-- V1+WITH session_funnel AS (SELECT profile_id, windowFunnel(86400000, 'strict_increase')(toUInt64(toUnixTimestamp64Milli(created_at)), events.name = 'session_start', events.name = 'session_end') AS level, argMinIf(`profile.properties.plan`, created_at, events.name = 'session_start') as b_0 FROM events LEFT JOIN (SELECT id, properties['plan'] as `profile.properties.plan` FROM profiles FINAL WHERE project_id = 'secure-privacy') as profile ON profile.id = events.profile_id WHERE project_id = 'secure-privacy' AND created_at BETWEEN toDateTime('2026-07-06 00:00:00') AND toDateTime('2026-07-09 23:59:59') AND events.name IN ('session_start', 'session_end') AND ((events.name = 'session_start') OR (events.name = 'session_end')) GROUP BY profile_id), funnel AS (SELECT * FROM session_funnel WHERE level != 0) SELECT DISTINCT profile_id FROM funnel WHERE level >= 2 LIMIT 1000+-- V2+WITH session_funnel AS (SELECT profile_id, windowFunnel({p1:UInt64}, 'strict_increase')(toUInt64(toUnixTimestamp64Milli(created_at)), events.name = {p2:String}, events.name = {p3:String}) AS level, argMinIf(`profile.properties.plan`, created_at, events.name = {p4:String}) as b_0 FROM events LEFT JOIN (SELECT id, properties['plan'] as `profile.properties.plan` FROM profiles FINAL WHERE project_id = {p5:String}) as profile ON profile.id = events.profile_id WHERE project_id = {p6:String} AND created_at BETWEEN toDateTime({p7:String}) AND toDateTime({p8:String}) AND events.name IN {p9:Array(String)} AND ((events.name = {p10:String}) OR (events.name = {p11:String})) GROUP BY profile_id), funnel AS (SELECT * FROM session_funnel WHERE level != 0) SELECT DISTINCT profile_id FROM funnel WHERE level >= {p12:UInt64} LIMIT {p13:UInt64}+-- V2 params: {"p1":86400000,"p2":"session_start","p3":"session_end","p4":"session_start","p5":"secure-privacy","p6":"secure-privacy","p7":"2026-07-06 00:00:00","p8":"2026-07-09 23:59:59","p9":["session_start","session_end"],"p10":"session_start","p11":"session_end","p12":2,"p13":1000}+```++### funnelSessionsQuery — buildSessionsCte++**IDENTICAL** — rows V1/V2 = 100/100; rows_read V1/V2 = 24553/24553; wall V1/V2 = 8 ms / 9 ms; clickhouse_settings: {"session_timezone":"UTC"} (both).++```sql+-- V1+SELECT profile_id as pid, id as sid FROM sessions WHERE project_id = 'secure-privacy' AND created_at BETWEEN toDateTime('2026-07-06 00:00:00') AND toDateTime('2026-07-07 00:00:00') LIMIT 100+-- V2+SELECT profile_id as pid, id as sid FROM sessions WHERE project_id = {p1:String} AND created_at BETWEEN toDateTime({p2:String}) AND toDateTime({p3:String}) LIMIT 100+-- V2 params: {"p1":"secure-privacy","p2":"2026-07-06 00:00:00","p3":"2026-07-07 00:00:00"}+```diff --git a/packages/core/src/modules/chart/src/funnel.sql.test.ts b/packages/core/src/modules/chart/src/funnel.sql.test.tsnew file mode 100644index 000000000..b6a5f2dc3--- /dev/null+++ b/packages/core/src/modules/chart/src/funnel.sql.test.ts@@ -0,0 +1,436 @@+/**+ * SQL-shape tests for the funnel statements. Ported from+ * packages/db/src/services/funnel-sql.test.ts (M7-004).+ *+ * Strategy, same as chart.sql.test.ts: render the statement, then run+ * `EXPLAIN <sql>` with its bound params against the isolated `openpanel_test`+ * ClickHouse (pinned by test/preload.ts). EXPLAIN parses and resolves columns+ * without executing, so a breakdown expression referencing a join alias that+ * was never added fails here as UNKNOWN_IDENTIFIER — which is exactly the+ * class of bug that made funnel "View Users" return "No users found" for+ * profile-property and cohort breakdowns.+ *+ * Text assertions look at the rendered `query` with placeholder NUMBERS+ * normalised away: a fragment reused in two clauses renders `{p3:String}` in+ * one and `{p11:String}` in the other, and the property under test is that the+ * same condition appears twice, not what its slots were numbered.+ *+ * The statements' only Postgres read (the breakdown's cohort names) is stubbed,+ * so this file is independent of whichever partial `prisma-client` mock a+ * module-root service test left behind.+ */++import { afterAll, beforeAll, describe, expect, it, mock } from 'bun:test';+import type { SqlFragment } from '@openpanel/db/src/clickhouse/sql';+import type { IChartBreakdown, IReportInput } from '@openpanel/validation';++const COHORT_ID = 'aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee';+const cohortFindMany = mock(async () => [+ { id: COHORT_ID, name: 'Power users' },+]);+// Plain-object snapshot, not the live binding — restoring from the namespace+// import in afterAll would just re-apply the mock.+const actualPrismaClient = await import('@openpanel/db/src/prisma-client');+const realPrismaClient = { ...actualPrismaClient };+mock.module('@openpanel/db/src/prisma-client', () => ({+ ...realPrismaClient,+ db: { cohort: { findMany: cohortFindMany } },+}));++afterAll(() => {+ mock.module('@openpanel/db/src/prisma-client', () => realPrismaClient);+});++let ch: typeof import('@openpanel/db/src/clickhouse/client').ch;+let buildFunnelBase: typeof import('../funnel.service').buildFunnelBase;+let funnelChartQuery: typeof import('./funnel.sql').funnelChartQuery;+let funnelProfilesQuery: typeof import('./funnel.sql').funnelProfilesQuery;+let funnelStepConditions: typeof import('./funnel.sql').funnelStepConditions;+let onlyReportEvents: typeof import('@openpanel/db/src/services/reports.service').onlyReportEvents;++const PROJECT_ID = 'test-sql-validation';+const START = '2026-04-14 00:00:00';+const END = '2026-05-15 00:00:00';+const FUNNEL_WINDOW_HOURS = 24;+const PROFILES_LIMIT = 1000;+const TARGET_LEVEL = 2;++type ISeriesItem = IReportInput['series'][number];++const event = (overrides: Partial<ISeriesItem> = {}): ISeriesItem =>+ ({+ id: 'A',+ type: 'event',+ name: 'screen_view',+ segment: 'event',+ filters: [],+ ...overrides,+ }) as ISeriesItem;++const breakdown = (name: string): IChartBreakdown => ({ id: name, name });++const SERIES = [event({ id: 'A' }), event({ id: 'B', name: 'sign_up' })];++/** `{p12:String}` → `{p:String}`; slot numbering is not the contract. */+const PLACEHOLDER_NUMBER = /\{p\d+:/g;+function normalize(query: string): string {+ return query.replace(PLACEHOLDER_NUMBER, '{p:');+}++interface Rendered {+ sql: string;+ params: Record<string, unknown>;+}++function render(fragment: SqlFragment): Rendered {+ const { query, query_params } = fragment.toStatement();+ return { sql: normalize(query), params: query_params };+}++async function explain(fragment: SqlFragment): Promise<void> {+ const { query, query_params } = fragment.toStatement();+ await ch.command({ query: `EXPLAIN ${query}`, query_params });+}++interface BaseOverrides {+ series?: ISeriesItem[];+ breakdowns?: IChartBreakdown[];+ funnelGroup?: string;+}++function baseInput(overrides: BaseOverrides = {}) {+ return {+ projectId: PROJECT_ID,+ startDate: START,+ endDate: END,+ series: overrides.series ?? SERIES,+ breakdowns: overrides.breakdowns ?? [],+ funnelWindow: FUNNEL_WINDOW_HOURS,+ funnelGroup: overrides.funnelGroup,+ timezone: 'UTC',+ };+}++/** Mirrors what getFunnelStepProfiles builds on top of the shared base. */+async function profilesStatement(overrides: BaseOverrides = {}) {+ const base = await buildFunnelBase(baseInput(overrides));+ return funnelProfilesQuery(base, {+ targetLevel: TARGET_LEVEL,+ showDropoffs: false,+ breakdownValues: [],+ limit: PROFILES_LIMIT,+ });+}++/** Mirrors what the funnel chart builds on top of the shared base. */+async function chartStatement(overrides: BaseOverrides = {}) {+ const base = await buildFunnelBase(baseInput(overrides));+ return funnelChartQuery(base);+}++const chartSql = async (breakdowns: IChartBreakdown[]) =>+ render(await chartStatement({ breakdowns })).sql;++const originalNonStrict = process.env.FUNNEL_NON_STRICT_ORDERING;+function setNonStrictOrdering(value: string | undefined) {+ if (value === undefined) {+ process.env.FUNNEL_NON_STRICT_ORDERING = undefined;+ delete process.env.FUNNEL_NON_STRICT_ORDERING;+ return;+ }+ process.env.FUNNEL_NON_STRICT_ORDERING = value;+}++beforeAll(async () => {+ ({ ch } = await import('@openpanel/db/src/clickhouse/client'));+ ({ buildFunnelBase } = await import('../funnel.service'));+ ({ funnelChartQuery, funnelProfilesQuery, funnelStepConditions } =+ await import('./funnel.sql'));+ ({ onlyReportEvents } = await import(+ '@openpanel/db/src/services/reports.service'+ ));+});++describe('funnel.sql / buildFunnelBase — profile breakdowns', () => {+ it('adds the profiles join for a profile.properties breakdown', async () => {+ const statement = await profilesStatement({+ breakdowns: [breakdown('profile.properties.plan')],+ });+ const { sql } = render(statement);++ // The breakdown is narrowed to a scalar alias selected by the profile+ // join, so both the join and the aliased column must exist.+ expect(sql).toContain("properties['plan'] as `profile.properties.plan`");+ expect(sql).toMatch(/as profile/);+ expect(sql).toContain('profile.id = events.profile_id');+ await explain(statement);+ });++ it('selects the properties column, not a bare `properties` field', async () => {+ const statement = await profilesStatement({+ breakdowns: [breakdown('profile.properties.plan')],+ });+ expect(render(statement).sql).toMatch(+ /SELECT [^)]*properties[^)]*FROM profiles/+ );+ await explain(statement);+ });++ it('adds the profiles join for a top-level profile breakdown', async () => {+ const statement = await profilesStatement({+ breakdowns: [breakdown('profile.email')],+ });+ expect(render(statement).sql).toContain('profile.id = events.profile_id');+ await explain(statement);+ });++ it('adds the join for a profile filter with no profile breakdown', async () => {+ const statement = await profilesStatement({+ series: [+ event({+ filters: [+ {+ id: 'f',+ name: 'profile.email',+ operator: 'is',+ value: ['[email protected]'],+ },+ ],+ }),+ event({ id: 'B', name: 'sign_up' }),+ ],+ });+ expect(render(statement).sql).toContain('profile.id = events.profile_id');+ await explain(statement);+ });+});++describe('funnel.sql / buildFunnelBase — cohort breakdowns', () => {+ it('adds the cohort join for a cohort breakdown', async () => {+ const statement = await profilesStatement({+ breakdowns: [breakdown(`cohort:${COHORT_ID}`)],+ });+ const { sql } = render(statement);++ // The breakdown renders `cohort_<id>.profile_id`, so its join must exist.+ const alias = `cohort_${COHORT_ID.replace(/-/g, '_')}`;+ expect(sql).toContain(`${alias}.profile_id`);+ expect(sql).toContain(`AS ${alias}`);+ expect(sql).toContain('Power users');+ await explain(statement);+ });+});++describe('funnel.sql / buildFunnelBase — unknown breakdowns', () => {+ it('drops the all-cohorts breakdown, which the funnel cannot render', async () => {+ // Bare `cohort` is the chart's all-cohorts breakdown. The funnel has no+ // equivalent, and inlining it produced `cohort as b_0 FROM events` —+ // UNKNOWN_IDENTIFIER for the chart and the profile list alike.+ const statement = await profilesStatement({+ breakdowns: [breakdown('cohort')],+ });+ expect(render(statement).sql).not.toMatch(/\bcohort as b_0\b/);+ await explain(statement);+ });++ it('drops an unrecognised breakdown name entirely', async () => {+ const statement = await profilesStatement({+ breakdowns: [breakdown('not_a_real_column')],+ });+ expect(render(statement).sql).not.toContain('not_a_real_column');+ await explain(statement);+ });++ it('drops unsupported breakdowns identically for chart and profiles', async () => {+ const breakdowns = [+ breakdown('cohort'),+ breakdown('profile.properties.plan'),+ ];+ const chart = await chartStatement({ breakdowns });+ const profiles = await profilesStatement({ breakdowns });++ // Both sides must resolve the same breakdown to the same b_N index,+ // otherwise the breakdownValues filter targets the wrong column.+ const attribution = 'argMinIf(`profile.properties.plan`, created_at,';+ expect(render(chart).sql).toContain(attribution);+ expect(render(profiles).sql).toContain(attribution);+ await explain(chart);+ await explain(profiles);+ });+});++describe('funnel.sql / funnel CTE — step pre-filter', () => {+ // Only rows matching at least one step can advance windowFunnel, so the+ // funnel CTE must not feed rows that share a step's event name but fail its+ // filters into the aggregation. Each step's COMPLETE condition therefore+ // appears in two places: inside windowFunnel and in the row-level+ // pre-filter. Asserting on the whole clause (not on a substring count)+ // catches a regression that drops a step from either one — and, unlike+ // counting, is not fooled by two steps whose conditions differ only in a+ // bound value.+ const conditionTexts = (series: ISeriesItem[]) =>+ funnelStepConditions(onlyReportEvents(series), PROJECT_ID, []).map(+ (condition) => normalize(condition.toStatement().query)+ );++ const windowFunnelClause = (conditions: string[]) =>+ `(toUInt64(toUnixTimestamp64Milli(created_at)), ${conditions.join(', ')}) AS level`;++ const preFilterClause = (conditions: string[]) =>+ ` AND ((${conditions.join(') OR (')})) GROUP BY `;++ it('filters the scan to rows matching at least one step condition', async () => {+ const sql = await chartSql([]);+ const conditions = conditionTexts(SERIES);+ expect(sql).toContain(windowFunnelClause(conditions));+ expect(sql).toContain(preFilterClause(conditions));+ });++ it("includes each step's own filters in the pre-filter", async () => {+ const series = [+ event({+ filters: [+ { id: 'f', name: 'path', operator: 'is', value: ['/pricing'] },+ ],+ }),+ event({ id: 'B', name: 'sign_up' }),+ ];+ const sql = render(await profilesStatement({ series })).sql;+ const conditions = conditionTexts(series);+ // The full filtered condition (path AND name) must gate the scan, not only+ // the windowFunnel arm — and so must the unfiltered second step.+ expect(conditions[0]).toContain('path');+ expect(sql).toContain(windowFunnelClause(conditions));+ expect(sql).toContain(preFilterClause(conditions));+ });++ it('pre-filtered funnel SQL still parses and resolves', async () => {+ await explain(+ await profilesStatement({+ breakdowns: [breakdown('profile.properties.plan')],+ })+ );+ });+});++describe('funnel.sql / buildFunnelBase — breakdown attribution', () => {+ // The breakdown value must be read at the user's FIRST funnel step, not used+ // as a windowFunnel GROUP BY key. Per-row grouping splits a user's steps+ // across buckets whenever the property isn't identical on every step, so the+ // sequence never connects and downstream steps show 0.+ it('attributes event-property breakdowns at the entry step', async () => {+ const sql = await chartSql([breakdown('properties.experiment')]);+ expect(sql).toContain("argMinIf(properties['experiment'], created_at,");+ // The windowFunnel aggregation groups by the primary key only; b_0 is an+ // aggregate, not a grouping key. (The outer chart GROUP BY level, b_0 is+ // unaffected.)+ expect(sql).toMatch(/GROUP BY session_id\b/);+ expect(sql).not.toMatch(/GROUP BY session_id, b_0/);+ });++ it('entry-step attribution parses and resolves', async () => {+ await explain(+ await chartStatement({ breakdowns: [breakdown('properties.experiment')] })+ );+ });++ it('keeps per-row grouping for group breakdowns (fan-out is intended)', async () => {+ // A user in three groups should appear in all three funnels, so group.*+ // breakdowns keep the ARRAY JOIN fan-out and per-row GROUP BY.+ const sql = await chartSql([breakdown('group.plan')]);+ expect(sql).not.toContain('argMinIf');+ expect(sql).toMatch(/GROUP BY session_id, b_0/);+ });++ it('group-breakdown SQL parses and resolves', async () => {+ await explain(+ await chartStatement({ breakdowns: [breakdown('group.plan')] })+ );+ });+});++describe('funnel.sql / funnel CTE — windowFunnel ordering', () => {+ afterAll(() => {+ setNonStrictOrdering(originalNonStrict);+ });++ it('defaults to strict_increase (unchanged behavior)', async () => {+ // Set to '' rather than deleting: a dev shell with the variable exported+ // would otherwise flip this test.+ setNonStrictOrdering('');+ expect(await chartSql([])).toContain("'strict_increase'");+ });++ it('drops strict_increase when FUNNEL_NON_STRICT_ORDERING is set', async () => {+ setNonStrictOrdering('1');+ const sql = await chartSql([]);+ expect(sql).not.toContain('strict_increase');+ expect(sql).toMatch(/windowFunnel\(\{p:UInt64\}\)\(/);+ });++ it('non-strict funnel SQL parses and resolves', async () => {+ setNonStrictOrdering('true');+ const statement = await chartStatement();+ expect(render(statement).sql).not.toContain('strict_increase');+ await explain(statement);+ });+});++describe('funnel.sql / buildFunnelBase — group breakdowns', () => {+ it('adds the group array join for a group breakdown', async () => {+ const statement = await profilesStatement({+ breakdowns: [breakdown('group.plan')],+ });+ const { sql } = render(statement);+ expect(sql).toContain('ARRAY JOIN groups AS _group_id');+ expect(sql).toContain('LEFT ANY JOIN _g ON _g.id = _group_id');+ await explain(statement);+ });+});++describe('funnel.sql / profile-property narrowing', () => {+ const seriesWithProfileFilter = [+ event({+ filters: [+ {+ id: 'f1',+ name: 'profile.properties.plan',+ operator: 'is' as const,+ value: ['pro'],+ },+ ],+ }),+ event({ id: 'B', name: 'sign_up' }),+ ];++ it('joins scalar columns instead of the whole properties Map', async () => {+ const statement = await profilesStatement({+ series: seriesWithProfileFilter,+ breakdowns: [breakdown('profile.properties.experiment')],+ });+ const { sql } = render(statement);++ expect(sql).toContain("properties['plan'] as `profile.properties.plan`");+ expect(sql).toContain(+ "properties['experiment'] as `profile.properties.experiment`"+ );+ expect(sql).not.toContain('properties as "profile.properties"');+ // Conditions (windowFunnel + pre-filter) and the breakdown expression are+ // rewritten to the scalar aliases.+ expect(sql).not.toContain("profile.properties['plan']");+ expect(sql).not.toContain("profile.properties['experiment']");+ });++ it('leaves funnels without profile-property refs untouched', async () => {+ expect(await chartSql([])).not.toContain('`profile.properties.');+ });++ it('narrowed funnel SQL parses and resolves', async () => {+ await explain(+ await chartStatement({+ series: seriesWithProfileFilter,+ breakdowns: [breakdown('profile.properties.experiment')],+ })+ );+ });+});diff --git a/packages/core/src/modules/chart/src/funnel.sql.ts b/packages/core/src/modules/chart/src/funnel.sql.tsnew file mode 100644index 000000000..2e6aa4415--- /dev/null+++ b/packages/core/src/modules/chart/src/funnel.sql.ts@@ -0,0 +1,417 @@+// Every ClickHouse statement the funnel service runs, as pure `sql` fragments+// (ADR-013). Converted 1:1 from packages/db/src/services/funnel.service.ts and+// the `getFunnelProfiles` body in packages/trpc/src/routers/chart.ts (M7-004):+// the SQL text is V1's clix output, with the project id, event names, dates,+// the funnel window, the target level and the clicked breakdown values bound+// as `{pN:Type}` parameters instead of `sqlstring`-escaped literals. Result-set+// proof: funnel.sql.proof.md.+//+// The chart and the profile list are built from ONE base (V1's+// `buildFunnelBase`): a breakdown expression only works if the join it+// references was added, and the joins depend on the breakdowns, so building+// the selects in one place and the joins in another is exactly the bug that+// made funnel "View Users" return "No users found" for profile-property and+// cohort breakdowns.+//+// The field resolver and filter compiler still render text (see compiled.ts);+// their output — filter clauses, breakdown expressions, the profile CTE+// columns and the cohort joins — is spliced, everything else is bound.+//+// Cluster note (docs/ENVIRONMENT.md): `events`, `profiles`, `groups` and+// `cohort_members` are Distributed on Cloud. Every join keeps V1's exact shape+// and runs under the client's `distributed_product_mode: 'allow'` as before;+// no `IN (subquery)` is introduced or removed.++import { type SqlFragment, sql } from '@openpanel/db/src/clickhouse/sql';+import type { IChartBreakdown, IChartEvent } from '@openpanel/validation';+import { compiledText, compiledTextWithProfileRefs } from './compiled';+import {+ buildInlineCohortJoin,+ CHART_TABLE,+ type CohortMetadata,+ collectBreakdownCohortIds,+ collectProfilePropertyKeys,+ extractCohortId,+ getSelectPropertyKey,+ isAllCohortsBreakdown,+ isKnownEventField,+ profilePropertiesCteSelect,+} from './field-resolution';+import { getEventFiltersWhereClause } from './filter-where';++/** The two keys windowFunnel may be computed per — a closed set. */+const FUNNEL_GROUPS = ['session_id', 'profile_id'] as const;+export type FunnelGroup = (typeof FUNNEL_GROUPS)[number];++/** Display label for null/empty breakdown values (e.g. property not set). */+export const EMPTY_BREAKDOWN_LABEL = 'Not set';++/** Profile columns the funnel's profiles CTE may expose beyond `properties`. */+const JOINABLE_PROFILE_COLUMNS = [+ 'email',+ 'first_name',+ 'last_name',+ 'created_at',+ 'last_seen_at',+];++/**+ * windowFunnel's 'strict_increase' mode requires every step's timestamp to be+ * strictly greater than the previous step's, so same-timestamp sequences+ * (server-side senders, batched SDKs, imported data with coarse timestamps)+ * never connect. Mixpanel/Amplitude count those as ordered, so deployments+ * migrating from them can opt into the default (>=) mode; strict stays the+ * default here.+ */+const STRICT_INCREASE_MODE = ", 'strict_increase'";++function windowFunnelMode(): SqlFragment {+ const nonStrictOrdering =+ process.env.FUNNEL_NON_STRICT_ORDERING === '1' ||+ process.env.FUNNEL_NON_STRICT_ORDERING === 'true';+ return nonStrictOrdering ? sql.empty : compiledText(STRICT_INCREASE_MODE);+}++export interface FunnelBaseInput {+ projectId: string;+ startDate: string;+ endDate: string;+ /** Already merged with the report's global filters and event-only. */+ eventSeries: IChartEvent[];+ /** As requested; unresolvable and all-cohorts breakdowns are dropped here. */+ breakdowns: IChartBreakdown[];+ funnelWindowMilliseconds: number;+ group: FunnelGroup;+ /** The raw `funnelGroup` option — `'group'` forces the groups ARRAY JOIN. */+ funnelGroup?: string;+ cohortMetadata: Map<string, CohortMetadata>;+ /** Travels to `session_timezone`; `toDateTime('…')` parses in it. */+ timezone: string;+}++/** Everything the funnel chart and the funnel profile list share. */+export interface FunnelBase {+ ctes: SqlFragment;+ eventSeries: IChartEvent[];+ breakdowns: IChartBreakdown[];+ group: FunnelGroup;+ timezone: string;+}++/**+ * Drop breakdowns that don't resolve to a known events column, properties+ * path, profile path, group path, or specific cohort — the funnel inlines each+ * breakdown's name via getSelectPropertyKey, so anything that doesn't resolve+ * leaks into the SQL verbatim.+ *+ * `isKnownEventField` accepts the bare `cohort` breakdown because the chart's+ * all-cohorts feature uses it, but the funnel has no equivalent: it renders as+ * `cohort as b_0 FROM events`, which fails with UNKNOWN_IDENTIFIER. Exclude it+ * explicitly rather than relying on the generic check.+ */+export function knownFunnelBreakdowns(+ breakdowns: IChartBreakdown[]+): IChartBreakdown[] {+ return breakdowns.filter(+ (breakdown) =>+ isKnownEventField(breakdown.name) &&+ !isAllCohortsBreakdown(breakdown.name)+ );+}++/**+ * V1 `getFunnelConditions`: one `<filters> AND events.name = <name>` per step.+ * Exported because each step's complete condition has to appear twice — inside+ * windowFunnel and in the row-level pre-filter — and funnel.sql.test.ts counts+ * exactly that.+ */+export function funnelStepConditions(+ eventSeries: IChartEvent[],+ projectId: string,+ profilePropertyKeys: string[]+): SqlFragment[] {+ return eventSeries.map((event) => {+ // Qualify with 'events' so event-level `properties[...]` becomes+ // `events.properties[...]` — required because the funnel CTE may join the+ // profiles table (which also exposes a `properties` column). Without the+ // qualifier ClickHouse fails with "ambiguous identifier 'properties'"+ // whenever a step filters on properties.X while another step filters on+ // profile.properties.Y.+ const filters = Object.values(+ getEventFiltersWhereClause(event.filters, projectId, 'events')+ )+ .filter(Boolean)+ .map((clause) =>+ compiledTextWithProfileRefs(clause, profilePropertyKeys)+ );+ return sql.join(+ [...filters, sql`events.name = ${sql.string(event.name)}`],+ ' AND '+ );+ });+}++/**+ * Attribute each breakdown to its value at the user's FIRST funnel step, as a+ * per-group aggregate (argMinIf) — not by adding it to the windowFunnel GROUP+ * BY. Grouping the sequence by a per-row value splits a user's steps across+ * buckets whenever the value isn't identical on every step (e.g. an experiment+ * tag set on the entry event but absent on the conversion event): the later+ * step lands in a separate bucket, the windowFunnel sequence never connects,+ * and downstream steps show 0.+ *+ * `group.*` breakdowns are the exception: their ARRAY JOIN fans each event out+ * per group, and grouping by the group value is intentional — a user in three+ * groups should appear in all three funnels.+ */+function breakdownSelects(+ input: FunnelBaseInput,+ profilePropertyKeys: string[],+ firstStepCondition: SqlFragment+): SqlFragment[] {+ return input.breakdowns.map((breakdown, index) => {+ const cohortId = extractCohortId(breakdown.name);+ const cohortName = cohortId+ ? input.cohortMetadata.get(cohortId)?.name+ : undefined;+ const expression = compiledTextWithProfileRefs(+ getSelectPropertyKey(+ breakdown.name,+ input.projectId,+ cohortId ?? undefined,+ cohortName+ ),+ profilePropertyKeys+ );+ const alias = compiledText(`b_${index}`);+ if (breakdown.name.startsWith('group.')) {+ return sql`${expression} as ${alias}`;+ }+ return sql`argMinIf(${expression}, created_at, ${firstStepCondition}) as ${alias}`;+ });+}++/**+ * The profile join has to cover breakdowns as well as filters — a `profile.*`+ * breakdown renders `profile.properties[...]` into the select, so the alias+ * must exist in scope even when no filter touches profiles.+ *+ * Only the referenced profile-property keys are joined, as scalar columns,+ * instead of every profile's whole properties Map: the join hash was carrying+ * ~1KB of Map per profile and OOMing at scale.+ */+function profileCteColumns(+ input: FunnelBaseInput,+ profileFilters: string[],+ profileKeys: { keys: string[]; needsFullMap: boolean }+): string[] {+ const profileBreakdowns = input.breakdowns.filter((breakdown) =>+ breakdown.name.startsWith('profile.')+ );+ if (profileFilters.length === 0 && profileBreakdowns.length === 0) {+ return [];+ }++ const fields = new Set<string>(['id']);+ for (const filter of profileFilters) {+ const fieldName = filter.split('.')[0] as string;+ if (fieldName !== 'properties') {+ fields.add(fieldName);+ }+ }+ for (const breakdown of profileBreakdowns) {+ const fieldName = breakdown.name.replace('profile.', '').split('.')[0];+ if (fieldName && JOINABLE_PROFILE_COLUMNS.includes(fieldName)) {+ fields.add(fieldName);+ }+ }++ const columns = Array.from(fields);+ const referencesProperties =+ profileFilters.some((filter) => filter.startsWith('properties')) ||+ profileBreakdowns.some((breakdown) =>+ breakdown.name.startsWith('profile.properties')+ );+ if (referencesProperties) {+ columns.push(+ profilePropertiesCteSelect(profileKeys.keys, profileKeys.needsFullMap)+ );+ }+ return columns;+}++/** V1 `getProfileFilters`: the `profile.`-prefixed filter names, unprefixed. */+function profileFilterNames(eventSeries: IChartEvent[]): string[] {+ return eventSeries.flatMap(+ (event) =>+ event.filters+ ?.filter((filter) => filter.name.startsWith('profile.'))+ .map((filter) => filter.name.replace('profile.', '')) ?? []+ );+}++export function funnelBase(input: FunnelBaseInput): FunnelBase {+ const { projectId, eventSeries, breakdowns, group } = input;++ if (eventSeries.length === 0) {+ throw new Error('events are required');+ }++ const profileFilters = profileFilterNames(eventSeries);+ const anyFilterOnGroup = eventSeries.some((event) =>+ event.filters?.some((filter) => filter.name.startsWith('group.'))+ );+ const anyBreakdownOnGroup = breakdowns.some((breakdown) =>+ breakdown.name.startsWith('group.')+ );+ const needsGroupArrayJoin =+ anyFilterOnGroup || anyBreakdownOnGroup || input.funnelGroup === 'group';++ const profileKeys = collectProfilePropertyKeys([+ ...eventSeries.flatMap((event) => event.filters ?? []),+ ...breakdowns,+ ]);++ const conditions = funnelStepConditions(+ eventSeries,+ projectId,+ profileKeys.keys+ );+ const selects = breakdownSelects(+ input,+ profileKeys.keys,+ conditions[0] as SqlFragment+ );++ const primaryKey = sql.id(group, FUNNEL_GROUPS);+ const cteSelects: SqlFragment[] = [+ primaryKey,+ sql`windowFunnel(${sql.uint64(input.funnelWindowMilliseconds)}${windowFunnelMode()})(toUInt64(toUnixTimestamp64Milli(created_at)), ${sql.join(conditions)}) AS level`,+ ];+ if (group === 'session_id') {+ // Resolves identity changes mid-session.+ cteSelects.push(sql`argMax(profile_id, created_at) AS profile_id`);+ }+ cteSelects.push(...selects);++ const groupByKeys: SqlFragment[] = [primaryKey];+ breakdowns.forEach((breakdown, index) => {+ if (breakdown.name.startsWith('group.')) {+ groupByKeys.push(compiledText(`b_${index}`));+ }+ });++ const columns = profileCteColumns(input, profileFilters, profileKeys);+ const profileJoin =+ columns.length > 0+ ? sql` LEFT JOIN (SELECT ${compiledText(columns.join(', '))} FROM ${sql.id(CHART_TABLE.profiles)} FINAL+ WHERE project_id = ${sql.string(projectId)}) as profile ON profile.id = events.profile_id`+ : sql.empty;+ const groupJoin = needsGroupArrayJoin+ ? sql` ARRAY JOIN groups AS _group_id LEFT ANY JOIN _g ON _g.id = _group_id`+ : sql.empty;+ // A cohort breakdown renders `cohort_<id>.profile_id`, so every cohort+ // referenced by a breakdown needs its join.+ const cohortIds = collectBreakdownCohortIds(breakdowns);+ const cohortJoins =+ cohortIds.length > 0+ ? sql` ${sql.join(+ cohortIds.map((cohortId) =>+ compiledText(buildInlineCohortJoin(cohortId, projectId, 'events'))+ ),+ ' '+ )}`+ : sql.empty;++ // Only rows matching at least one step can advance windowFunnel, so rows+ // that share a step's event name but fail its filters are dead weight — with+ // filtered steps (e.g. screen_view + a path filter) they can be the vast+ // majority of what the name IN(...) lets through. Dropping them here shrinks+ // the aggregation input; windowFunnel ignores non-matching rows either way,+ // so levels are unchanged.+ const stepPreFilter = sql`(${sql.join(+ conditions.map((condition) => sql`(${condition})`),+ ' OR '+ )})`;++ const funnelCte = sql`SELECT ${sql.join(cteSelects)} FROM ${sql.id(CHART_TABLE.events)}${profileJoin}${groupJoin}${cohortJoins} WHERE project_id = ${sql.string(projectId)} AND created_at BETWEEN toDateTime(${sql.string(input.startDate)}) AND toDateTime(${sql.string(input.endDate)}) AND events.name IN ${sql.array(+ 'String',+ eventSeries.map((event) => event.name)+ )} AND ${stepPreFilter} GROUP BY ${sql.join(groupByKeys)}`;++ const groupsCte = needsGroupArrayJoin+ ? sql`_g AS (SELECT id, name, type, properties FROM ${sql.id(CHART_TABLE.groups)} FINAL WHERE project_id = ${sql.string(projectId)}), `+ : sql.empty;++ // windowFunnel is computed per primary key, so the `funnel` CTE only has to+ // drop the level-0 rows — no re-aggregation.+ return {+ ctes: sql`WITH ${groupsCte}session_funnel AS (${funnelCte}), funnel AS (SELECT * FROM session_funnel WHERE level != 0) `,+ eventSeries,+ breakdowns,+ group,+ timezone: input.timezone,+ };+}++/** V1 `FunnelService.getFunnel`: one row per (level, breakdown combination). */+export function funnelChartQuery(base: FunnelBase): SqlFragment {+ const aliases = base.breakdowns.map((_, index) => compiledText(`b_${index}`));+ const select = sql.join([sql`level`, ...aliases, sql`count() as count`]);+ const groupBy = sql.join([sql`level`, ...aliases]);+ return sql`${base.ctes}SELECT ${select} FROM funnel GROUP BY ${groupBy} ORDER BY level DESC`;+}++export interface FunnelProfilesInput {+ /** 1-based funnel level. */+ targetLevel: number;+ /** Selects only the profiles that stopped exactly at this level. */+ showDropoffs: boolean;+ /**+ * Display labels for the clicked row, per breakdown index. `undefined`+ * leaves that breakdown unconstrained.+ */+ breakdownValues: (string | undefined)[];+ limit: number;+}++/**+ * The clicked row carries DISPLAY labels (trimmed, empty/null shown as+ * EMPTY_BREAKDOWN_LABEL), so the comparison normalises the column the same+ * way. `toString`/`ifNull` keep it valid for numeric and Nullable breakdowns.+ */+function breakdownValueWhere(index: number, value: string): SqlFragment {+ const normalized = compiledText(`trim(ifNull(toString(b_${index}), ''))`);+ if (value === EMPTY_BREAKDOWN_LABEL) {+ return sql`(${normalized} = '' OR ${normalized} = ${sql.string(EMPTY_BREAKDOWN_LABEL)})`;+ }+ return sql`${normalized} = ${sql.string(value)}`;+}++/** V1 `getFunnelProfiles`: the distinct profiles at (or dropping off at) a step. */+export function funnelProfilesQuery(+ base: FunnelBase,+ input: FunnelProfilesInput+): SqlFragment {+ const where: SqlFragment[] = [+ input.showDropoffs+ ? sql`level = ${sql.uint64(input.targetLevel)}`+ : sql`level >= ${sql.uint64(input.targetLevel)}`,+ ];+ base.breakdowns.forEach((_, index) => {+ const value = input.breakdownValues[index];+ if (value !== undefined) {+ where.push(breakdownValueWhere(index, value));+ }+ });+ return sql`${base.ctes}SELECT DISTINCT profile_id FROM funnel WHERE ${sql.join(where, ' AND ')} LIMIT ${sql.uint64(input.limit)}`;+}++/** Exported by V1 as `buildSessionsCte`; no in-repo caller, kept as published API. */+export function funnelSessionsQuery(input: {+ projectId: string;+ startDate: string;+ endDate: string;+}): SqlFragment {+ return sql`SELECT profile_id as pid, id as sid FROM ${sql.id(CHART_TABLE.sessions)} WHERE project_id = ${sql.string(input.projectId)} AND created_at BETWEEN toDateTime(${sql.string(input.startDate)}) AND toDateTime(${sql.string(input.endDate)})`;+}diff --git a/packages/core/src/modules/chart/src/retention.sql.proof.md b/packages/core/src/modules/chart/src/retention.sql.proof.mdnew file mode 100644index 000000000..056d2745d--- /dev/null+++ b/packages/core/src/modules/chart/src/retention.sql.proof.md@@ -0,0 +1,178 @@+# retention.sql.ts — V1 -> V2 result-set proof (M7-004)++Every builder in `retention.sql.ts` was executed twice against the same data — once as V1 (the `git HEAD` `packages/db/src/services/retention.service.ts`, `c4760ac2`, captured by wrapping its `chQuery`) and once as V2 (the fragment returned by the core builder, bound through `query_params`) — through one `@clickhouse/client` with `format: 'JSON'` and the same client settings. The two responses were compared on `data` (every row, every column, as JSON) and `meta` (column names and types).++- **Date**: 2026-09-04. **Data**: local prod-copy `openpanel` (319,499,000 events, static). Projects: `verdict`, `secure-privacy`, `dream-mate`.+- **Machine**: single-node ClickHouse on a 4-vCPU box — timings are directional only; production is 2 shards x 2 replicas (`docs/ENVIRONMENT.md`). None of these statements contains an `IN (subquery)`: the cohort matrix is a five-CTE self-join, unchanged from V1, so no `IN` / `GLOBAL IN` decision is made or unmade here.+- **Whitespace**: V2 joins the base WHERE conditions and the interval columns with the `sql` tag's closed separator set (`' AND '`, `', '`) instead of V1's newline-indented ones. Text-only; every result set below is byte-identical.+- **Contract test**: `packages/db/src/services/retention.service.test.ts` (11 cases, seeded fixtures on `openpanel_test`) is unchanged and runs against this code through the V1 shim.++- **Verdict**: 14 cases, 0 not identical.++### retentionSeriesQuery — verdict++**IDENTICAL** — rows V1/V2 = 9/9; rows_read V1/V2 = 88473638/88473638; wall V1/V2 = 3103 ms / 3053 ms.++```sql+-- V1+WITH weekly_active AS ( SELECT profile_id, toStartOfWeek(created_at) AS week FROM events WHERE project_id = 'verdict' AND profile_id != device_id GROUP BY profile_id, week ) SELECT cur.week AS date, countDistinct(cur.profile_id) AS active_users, countDistinct(nxt.profile_id) AS retained_users, (100 * (countDistinct(nxt.profile_id) / CAST(countDistinct(cur.profile_id), 'Float64'))) AS retention FROM weekly_active AS cur LEFT JOIN weekly_active AS nxt ON cur.profile_id = nxt.profile_id AND nxt.week = cur.week + toIntervalWeek(1) GROUP BY date ORDER BY date ASC -- Unmatched LEFT JOIN rows must be NULL (not the empty-string default), -- otherwise countDistinct(nxt.profile_id) counts '' as a retained user. SETTINGS join_use_nulls = 1+-- V2+WITH weekly_active AS ( SELECT profile_id, toStartOfWeek(created_at) AS week FROM events WHERE project_id = {p1:String} AND profile_id != device_id GROUP BY profile_id, week ) SELECT cur.week AS date, countDistinct(cur.profile_id) AS active_users, countDistinct(nxt.profile_id) AS retained_users, (100 * (countDistinct(nxt.profile_id) / CAST(countDistinct(cur.profile_id), 'Float64'))) AS retention FROM weekly_active AS cur LEFT JOIN weekly_active AS nxt ON cur.profile_id = nxt.profile_id AND nxt.week = cur.week + toIntervalWeek(1) GROUP BY date ORDER BY date ASC -- Unmatched LEFT JOIN rows must be NULL (not the empty-string default), -- otherwise countDistinct(nxt.profile_id) counts '' as a retained user. SETTINGS join_use_nulls = 1+-- V2 params: {"p1":"verdict"}+```++### retentionSeriesQuery — secure-privacy++**IDENTICAL** — rows V1/V2 = 9/9; rows_read V1/V2 = 1130322/1130322; wall V1/V2 = 85 ms / 102 ms.++```sql+-- V1+WITH weekly_active AS ( SELECT profile_id, toStartOfWeek(created_at) AS week FROM events WHERE project_id = 'secure-privacy' AND profile_id != device_id GROUP BY profile_id, week ) SELECT cur.week AS date, countDistinct(cur.profile_id) AS active_users, countDistinct(nxt.profile_id) AS retained_users, (100 * (countDistinct(nxt.profile_id) / CAST(countDistinct(cur.profile_id), 'Float64'))) AS retention FROM weekly_active AS cur LEFT JOIN weekly_active AS nxt ON cur.profile_id = nxt.profile_id AND nxt.week = cur.week + toIntervalWeek(1) GROUP BY date ORDER BY date ASC -- Unmatched LEFT JOIN rows must be NULL (not the empty-string default), -- otherwise countDistinct(nxt.profile_id) counts '' as a retained user. SETTINGS join_use_nulls = 1+-- V2+WITH weekly_active AS ( SELECT profile_id, toStartOfWeek(created_at) AS week FROM events WHERE project_id = {p1:String} AND profile_id != device_id GROUP BY profile_id, week ) SELECT cur.week AS date, countDistinct(cur.profile_id) AS active_users, countDistinct(nxt.profile_id) AS retained_users, (100 * (countDistinct(nxt.profile_id) / CAST(countDistinct(cur.profile_id), 'Float64'))) AS retention FROM weekly_active AS cur LEFT JOIN weekly_active AS nxt ON cur.profile_id = nxt.profile_id AND nxt.week = cur.week + toIntervalWeek(1) GROUP BY date ORDER BY date ASC -- Unmatched LEFT JOIN rows must be NULL (not the empty-string default), -- otherwise countDistinct(nxt.profile_id) counts '' as a retained user. SETTINGS join_use_nulls = 1+-- V2 params: {"p1":"secure-privacy"}+```++### rollingActiveUsersQuery — verdict, 7d++**IDENTICAL** — rows V1/V2 = 62/62; rows_read V1/V2 = 73247/73247; wall V1/V2 = 355 ms / 358 ms.++```sql+-- V1+SELECT date, uniqMerge(profile_id) AS users FROM ( SELECT date + n AS date, profile_id, project_id FROM ( SELECT * FROM dau_mv WHERE project_id = 'verdict' ) ARRAY JOIN range(7) AS n ) WHERE project_id = 'verdict' GROUP BY date+-- V2+SELECT date, uniqMerge(profile_id) AS users FROM ( SELECT date + n AS date, profile_id, project_id FROM ( SELECT * FROM dau_mv WHERE project_id = {p1:String} ) ARRAY JOIN range({p2:UInt64}) AS n ) WHERE project_id = {p3:String} GROUP BY date+-- V2 params: {"p1":"verdict","p2":7,"p3":"verdict"}+```++### rollingActiveUsersQuery — secure-privacy, 30d++**IDENTICAL** — rows V1/V2 = 85/85; rows_read V1/V2 = 73247/73247; wall V1/V2 = 312 ms / 313 ms.++```sql+-- V1+SELECT date, uniqMerge(profile_id) AS users FROM ( SELECT date + n AS date, profile_id, project_id FROM ( SELECT * FROM dau_mv WHERE project_id = 'secure-privacy' ) ARRAY JOIN range(30) AS n ) WHERE project_id = 'secure-privacy' GROUP BY date+-- V2+SELECT date, uniqMerge(profile_id) AS users FROM ( SELECT date + n AS date, profile_id, project_id FROM ( SELECT * FROM dau_mv WHERE project_id = {p1:String} ) ARRAY JOIN range({p2:UInt64}) AS n ) WHERE project_id = {p3:String} GROUP BY date+-- V2 params: {"p1":"secure-privacy","p2":30,"p3":"secure-privacy"}+```++### rollingActiveUsersQuery — dream-mate, 1d++**IDENTICAL** — rows V1/V2 = 56/56; rows_read V1/V2 = 73247/73247; wall V1/V2 = 250 ms / 252 ms.++```sql+-- V1+SELECT date, uniqMerge(profile_id) AS users FROM ( SELECT date + n AS date, profile_id, project_id FROM ( SELECT * FROM dau_mv WHERE project_id = 'dream-mate' ) ARRAY JOIN range(1) AS n ) WHERE project_id = 'dream-mate' GROUP BY date+-- V2+SELECT date, uniqMerge(profile_id) AS users FROM ( SELECT date + n AS date, profile_id, project_id FROM ( SELECT * FROM dau_mv WHERE project_id = {p1:String} ) ARRAY JOIN range({p2:UInt64}) AS n ) WHERE project_id = {p3:String} GROUP BY date+-- V2 params: {"p1":"dream-mate","p2":1,"p3":"dream-mate"}+```++### retentionLastSeenSeriesQuery — verdict++**IDENTICAL** — rows V1/V2 = 56/56; rows_read V1/V2 = 44236819/44236819; wall V1/V2 = 1276 ms / 1274 ms.++```sql+-- V1+WITH last_active AS ( SELECT max(created_at) AS last_active, profile_id FROM events WHERE (project_id = 'verdict') AND (device_id != profile_id) GROUP BY profile_id ) SELECT dateDiff('day', last_active, today()) AS days, countDistinct(profile_id) AS users FROM last_active GROUP BY days ORDER BY days ASC+-- V2+WITH last_active AS ( SELECT max(created_at) AS last_active, profile_id FROM events WHERE (project_id = {p1:String}) AND (device_id != profile_id) GROUP BY profile_id ) SELECT dateDiff('day', last_active, today()) AS days, countDistinct(profile_id) AS users FROM last_active GROUP BY days ORDER BY days ASC+-- V2 params: {"p1":"verdict"}+```++### retentionLastSeenSeriesQuery — secure-privacy++**IDENTICAL** — rows V1/V2 = 56/56; rows_read V1/V2 = 565161/565161; wall V1/V2 = 46 ms / 54 ms.++```sql+-- V1+WITH last_active AS ( SELECT max(created_at) AS last_active, profile_id FROM events WHERE (project_id = 'secure-privacy') AND (device_id != profile_id) GROUP BY profile_id ) SELECT dateDiff('day', last_active, today()) AS days, countDistinct(profile_id) AS users FROM last_active GROUP BY days ORDER BY days ASC+-- V2+WITH last_active AS ( SELECT max(created_at) AS last_active, profile_id FROM events WHERE (project_id = {p1:String}) AND (device_id != profile_id) GROUP BY profile_id ) SELECT dateDiff('day', last_active, today()) AS days, countDistinct(profile_id) AS users FROM last_active GROUP BY days ORDER BY days ASC+-- V2 params: {"p1":"secure-privacy"}+```++### retentionCohortQuery — any event, week, on_or_after (getRetentionCohortCore shape)++**IDENTICAL** — rows V1/V2 = 9/9; rows_read V1/V2 = 8945664/8945664; wall V1/V2 = 369 ms / 374 ms.++```sql+-- V1+WITH cohort_users AS ( SELECT profile_id AS userID, toStartOfWeek(min(created_at)) AS cohort_interval FROM cohort_events_mv WHERE project_id = 'verdict' AND created_at >= toDateTime('2026-06-14 00:00:00') AND created_at <= toDateTime('2026-09-06 00:00:00') GROUP BY profile_id ), last_event AS ( SELECT profile_id, toDate(created_at) AS event_date FROM cohort_events_mv WHERE project_id = 'verdict' AND created_at >= toDateTime('2026-06-14 00:00:00') AND created_at <= toDateTime('2026-09-06 00:00:00') + INTERVAL 12 WEEK ), retention_matrix AS ( SELECT f.cohort_interval, l.profile_id, dateDiff('WEEK', f.cohort_interval, toStartOfWeek(l.event_date)) AS x_after_cohort FROM cohort_users AS f INNER JOIN last_event AS l ON f.userID = l.profile_id WHERE l.event_date >= f.cohort_interval AND dateDiff('WEEK', f.cohort_interval, toStartOfWeek(l.event_date)) <= 12 ), interval_users AS ( SELECT cohort_interval, groupUniqArrayIf(profile_id, x_after_cohort >= 0) AS interval_0_users, groupUniqArrayIf(profile_id, x_after_cohort >= 1) AS interval_1_users, groupUniqArrayIf(profile_id, x_after_cohort >= 2) AS interval_2_users, groupUniqArrayIf(profile_id, x_after_cohort >= 3) AS interval_3_users, groupUniqArrayIf(profile_id, x_after_cohort >= 4) AS interval_4_users, groupUniqArrayIf(profile_id, x_after_cohort >= 5) AS interval_5_users, groupUniqArrayIf(profile_id, x_after_cohort >= 6) AS interval_6_users, groupUniqArrayIf(profile_id, x_after_cohort >= 7) AS interval_7_users, groupUniqArrayIf(profile_id, x_after_cohort >= 8) AS interval_8_users, groupUniqArrayIf(profile_id, x_after_cohort >= 9) AS interval_9_users, groupUniqArrayIf(profile_id, x_after_cohort >= 10) AS interval_10_users, groupUniqArrayIf(profile_id, x_after_cohort >= 11) AS interval_11_users, groupUniqArrayIf(profile_id, x_after_cohort >= 12) AS interval_12_users FROM retention_matrix GROUP BY cohort_interval ), cohort_sizes AS ( SELECT cohort_interval, COUNT(DISTINCT userID) AS total_first_event_count FROM cohort_users GROUP BY cohort_interval ) SELECT interval_users.cohort_interval AS cohort_interval, cs.total_first_event_count AS total_first_event_count, length(interval_0_users) AS interval_0_user_count, length(interval_1_users) AS interval_1_user_count, length(interval_2_users) AS interval_2_user_count, length(interval_3_users) AS interval_3_user_count, length(interval_4_users) AS interval_4_user_count, length(interval_5_users) AS interval_5_user_count, length(interval_6_users) AS interval_6_user_count, length(interval_7_users) AS interval_7_user_count, length(interval_8_users) AS interval_8_user_count, length(interval_9_users) AS interval_9_user_count, length(interval_10_users) AS interval_10_user_count, length(interval_11_users) AS interval_11_user_count, length(interval_12_users) AS interval_12_user_count FROM interval_users LEFT JOIN cohort_sizes AS cs ON interval_users.cohort_interval = cs.cohort_interval ORDER BY cohort_interval ASC+-- V2+WITH cohort_users AS ( SELECT profile_id AS userID, toStartOfWeek(min(created_at)) AS cohort_interval FROM cohort_events_mv WHERE project_id = {p1:String} AND created_at >= toDateTime({p2:String}) AND created_at <= toDateTime({p3:String}) GROUP BY profile_id ), last_event AS ( SELECT profile_id, toDate(created_at) AS event_date FROM cohort_events_mv WHERE project_id = {p4:String} AND created_at >= toDateTime({p5:String}) AND created_at <= toDateTime({p6:String}) + INTERVAL {p7:UInt64} WEEK ), retention_matrix AS ( SELECT f.cohort_interval, l.profile_id, dateDiff({p8:String}, f.cohort_interval, toStartOfWeek(l.event_date)) AS x_after_cohort FROM cohort_users AS f INNER JOIN last_event AS l ON f.userID = l.profile_id WHERE l.event_date >= f.cohort_interval AND dateDiff({p9:String}, f.cohort_interval, toStartOfWeek(l.event_date)) <= {p10:UInt64} ), interval_users AS ( SELECT cohort_interval, groupUniqArrayIf(profile_id, x_after_cohort >= {p11:UInt64}) AS interval_0_users, groupUniqArrayIf(profile_id, x_after_cohort >= {p12:UInt64}) AS interval_1_users, groupUniqArrayIf(profile_id, x_after_cohort >= {p13:UInt64}) AS interval_2_users, groupUniqArrayIf(profile_id, x_after_cohort >= {p14:UInt64}) AS interval_3_users, groupUniqArrayIf(profile_id, x_after_cohort >= {p15:UInt64}) AS interval_4_users, groupUniqArrayIf(profile_id, x_after_cohort >= {p16:UInt64}) AS interval_5_users, groupUniqArrayIf(profile_id, x_after_cohort >= {p17:UInt64}) AS interval_6_users, groupUniqArrayIf(profile_id, x_after_cohort >= {p18:UInt64}) AS interval_7_users, groupUniqArrayIf(profile_id, x_after_cohort >= {p19:UInt64}) AS interval_8_users, groupUniqArrayIf(profile_id, x_after_cohort >= {p20:UInt64}) AS interval_9_users, groupUniqArrayIf(profile_id, x_after_cohort >= {p21:UInt64}) AS interval_10_users, groupUniqArrayIf(profile_id, x_after_cohort >= {p22:UInt64}) AS interval_11_users, groupUniqArrayIf(profile_id, x_after_cohort >= {p23:UInt64}) AS interval_12_users FROM retention_matrix GROUP BY cohort_interval ), cohort_sizes AS ( SELECT cohort_interval, COUNT(DISTINCT userID) AS total_first_event_count FROM cohort_users GROUP BY cohort_interval ) SELECT interval_users.cohort_interval AS cohort_interval, cs.total_first_event_count AS total_first_event_count, length(interval_0_users) AS interval_0_user_count, length(interval_1_users) AS interval_1_user_count, length(interval_2_users) AS interval_2_user_count, length(interval_3_users) AS interval_3_user_count, length(interval_4_users) AS interval_4_user_count, length(interval_5_users) AS interval_5_user_count, length(interval_6_users) AS interval_6_user_count, length(interval_7_users) AS interval_7_user_count, length(interval_8_users) AS interval_8_user_count, length(interval_9_users) AS interval_9_user_count, length(interval_10_users) AS interval_10_user_count, length(interval_11_users) AS interval_11_user_count, length(interval_12_users) AS interval_12_user_count FROM interval_users LEFT JOIN cohort_sizes AS cs ON interval_users.cohort_interval = cs.cohort_interval ORDER BY cohort_interval ASC+-- V2 params: {"p1":"verdict","p2":"2026-06-14 00:00:00","p3":"2026-09-06 00:00:00","p4":"verdict","p5":"2026-06-14 00:00:00","p6":"2026-09-06 00:00:00","p7":12,"p8":"WEEK","p9":"WEEK","p10":12,"p11":0,"p12":1,"p13":2,"p14":3,"p15":4,"p16":5,"p17":6,"p18":7,"p19":8,"p20":9,"p21":10,"p22":11,"p23":12}+```++### retentionCohortQuery — named first+second event, day, on++**IDENTICAL** — rows V1/V2 = 7/7; rows_read V1/V2 = 163840/163840; wall V1/V2 = 42 ms / 38 ms.++```sql+-- V1+WITH cohort_users AS ( SELECT profile_id AS userID, toDate(min(created_at)) AS cohort_interval FROM cohort_events_mv WHERE project_id = 'verdict' AND name = 'session_start' AND created_at >= toDateTime('2026-07-06 00:00:00') AND created_at <= toDateTime('2026-07-12 23:59:59') GROUP BY profile_id ), last_event AS ( SELECT profile_id, toDate(created_at) AS event_date FROM cohort_events_mv WHERE project_id = 'verdict' AND name = 'view_page' AND created_at >= toDateTime('2026-07-06 00:00:00') AND created_at <= toDateTime('2026-07-12 23:59:59') + INTERVAL 6 DAY ), retention_matrix AS ( SELECT f.cohort_interval, l.profile_id, dateDiff('DAY', f.cohort_interval, toDate(l.event_date)) AS x_after_cohort FROM cohort_users AS f INNER JOIN last_event AS l ON f.userID = l.profile_id WHERE l.event_date >= f.cohort_interval AND dateDiff('DAY', f.cohort_interval, toDate(l.event_date)) <= 6 ), interval_users AS ( SELECT cohort_interval, groupUniqArrayIf(profile_id, x_after_cohort = 0) AS interval_0_users, groupUniqArrayIf(profile_id, x_after_cohort = 1) AS interval_1_users, groupUniqArrayIf(profile_id, x_after_cohort = 2) AS interval_2_users, groupUniqArrayIf(profile_id, x_after_cohort = 3) AS interval_3_users, groupUniqArrayIf(profile_id, x_after_cohort = 4) AS interval_4_users, groupUniqArrayIf(profile_id, x_after_cohort = 5) AS interval_5_users, groupUniqArrayIf(profile_id, x_after_cohort = 6) AS interval_6_users FROM retention_matrix GROUP BY cohort_interval ), cohort_sizes AS ( SELECT cohort_interval, COUNT(DISTINCT userID) AS total_first_event_count FROM cohort_users GROUP BY cohort_interval ) SELECT interval_users.cohort_interval AS cohort_interval, cs.total_first_event_count AS total_first_event_count, length(interval_0_users) AS interval_0_user_count, length(interval_1_users) AS interval_1_user_count, length(interval_2_users) AS interval_2_user_count, length(interval_3_users) AS interval_3_user_count, length(interval_4_users) AS interval_4_user_count, length(interval_5_users) AS interval_5_user_count, length(interval_6_users) AS interval_6_user_count FROM interval_users LEFT JOIN cohort_sizes AS cs ON interval_users.cohort_interval = cs.cohort_interval ORDER BY cohort_interval ASC+-- V2+WITH cohort_users AS ( SELECT profile_id AS userID, toDate(min(created_at)) AS cohort_interval FROM cohort_events_mv WHERE project_id = {p1:String} AND name = {p2:String} AND created_at >= toDateTime({p3:String}) AND created_at <= toDateTime({p4:String}) GROUP BY profile_id ), last_event AS ( SELECT profile_id, toDate(created_at) AS event_date FROM cohort_events_mv WHERE project_id = {p5:String} AND name = {p6:String} AND created_at >= toDateTime({p7:String}) AND created_at <= toDateTime({p8:String}) + INTERVAL {p9:UInt64} DAY ), retention_matrix AS ( SELECT f.cohort_interval, l.profile_id, dateDiff({p10:String}, f.cohort_interval, toDate(l.event_date)) AS x_after_cohort FROM cohort_users AS f INNER JOIN last_event AS l ON f.userID = l.profile_id WHERE l.event_date >= f.cohort_interval AND dateDiff({p11:String}, f.cohort_interval, toDate(l.event_date)) <= {p12:UInt64} ), interval_users AS ( SELECT cohort_interval, groupUniqArrayIf(profile_id, x_after_cohort = {p13:UInt64}) AS interval_0_users, groupUniqArrayIf(profile_id, x_after_cohort = {p14:UInt64}) AS interval_1_users, groupUniqArrayIf(profile_id, x_after_cohort = {p15:UInt64}) AS interval_2_users, groupUniqArrayIf(profile_id, x_after_cohort = {p16:UInt64}) AS interval_3_users, groupUniqArrayIf(profile_id, x_after_cohort = {p17:UInt64}) AS interval_4_users, groupUniqArrayIf(profile_id, x_after_cohort = {p18:UInt64}) AS interval_5_users, groupUniqArrayIf(profile_id, x_after_cohort = {p19:UInt64}) AS interval_6_users FROM retention_matrix GROUP BY cohort_interval ), cohort_sizes AS ( SELECT cohort_interval, COUNT(DISTINCT userID) AS total_first_event_count FROM cohort_users GROUP BY cohort_interval ) SELECT interval_users.cohort_interval AS cohort_interval, cs.total_first_event_count AS total_first_event_count, length(interval_0_users) AS interval_0_user_count, length(interval_1_users) AS interval_1_user_count, length(interval_2_users) AS interval_2_user_count, length(interval_3_users) AS interval_3_user_count, length(interval_4_users) AS interval_4_user_count, length(interval_5_users) AS interval_5_user_count, length(interval_6_users) AS interval_6_user_count FROM interval_users LEFT JOIN cohort_sizes AS cs ON interval_users.cohort_interval = cs.cohort_interval ORDER BY cohort_interval ASC+-- V2 params: {"p1":"verdict","p2":"session_start","p3":"2026-07-06 00:00:00","p4":"2026-07-12 23:59:59","p5":"verdict","p6":"view_page","p7":"2026-07-06 00:00:00","p8":"2026-07-12 23:59:59","p9":6,"p10":"DAY","p11":"DAY","p12":6,"p13":0,"p14":1,"p15":2,"p16":3,"p17":4,"p18":5,"p19":6}+```++### retentionCohortQuery — multi first event (IN), week, on_or_after++**IDENTICAL** — rows V1/V2 = 3/3; rows_read V1/V2 = 65536/65536; wall V1/V2 = 25 ms / 29 ms.++```sql+-- V1+WITH cohort_users AS ( SELECT profile_id AS userID, toStartOfWeek(min(created_at)) AS cohort_interval FROM cohort_events_mv WHERE project_id = 'secure-privacy' AND name IN ('session_start', 'screen_view') AND created_at >= toDateTime('2026-06-01 00:00:00') AND created_at <= toDateTime('2026-07-12 23:59:59') GROUP BY profile_id ), last_event AS ( SELECT profile_id, toDate(created_at) AS event_date FROM cohort_events_mv WHERE project_id = 'secure-privacy' AND name IN ('screen_view', 'session_end') AND created_at >= toDateTime('2026-06-01 00:00:00') AND created_at <= toDateTime('2026-07-12 23:59:59') + INTERVAL 5 WEEK ), retention_matrix AS ( SELECT f.cohort_interval, l.profile_id, dateDiff('WEEK', f.cohort_interval, toStartOfWeek(l.event_date)) AS x_after_cohort FROM cohort_users AS f INNER JOIN last_event AS l ON f.userID = l.profile_id WHERE l.event_date >= f.cohort_interval AND dateDiff('WEEK', f.cohort_interval, toStartOfWeek(l.event_date)) <= 5 ), interval_users AS ( SELECT cohort_interval, groupUniqArrayIf(profile_id, x_after_cohort >= 0) AS interval_0_users, groupUniqArrayIf(profile_id, x_after_cohort >= 1) AS interval_1_users, groupUniqArrayIf(profile_id, x_after_cohort >= 2) AS interval_2_users, groupUniqArrayIf(profile_id, x_after_cohort >= 3) AS interval_3_users, groupUniqArrayIf(profile_id, x_after_cohort >= 4) AS interval_4_users, groupUniqArrayIf(profile_id, x_after_cohort >= 5) AS interval_5_users FROM retention_matrix GROUP BY cohort_interval ), cohort_sizes AS ( SELECT cohort_interval, COUNT(DISTINCT userID) AS total_first_event_count FROM cohort_users GROUP BY cohort_interval ) SELECT interval_users.cohort_interval AS cohort_interval, cs.total_first_event_count AS total_first_event_count, length(interval_0_users) AS interval_0_user_count, length(interval_1_users) AS interval_1_user_count, length(interval_2_users) AS interval_2_user_count, length(interval_3_users) AS interval_3_user_count, length(interval_4_users) AS interval_4_user_count, length(interval_5_users) AS interval_5_user_count FROM interval_users LEFT JOIN cohort_sizes AS cs ON interval_users.cohort_interval = cs.cohort_interval ORDER BY cohort_interval ASC+-- V2+WITH cohort_users AS ( SELECT profile_id AS userID, toStartOfWeek(min(created_at)) AS cohort_interval FROM cohort_events_mv WHERE project_id = {p1:String} AND name IN {p2:Array(String)} AND created_at >= toDateTime({p3:String}) AND created_at <= toDateTime({p4:String}) GROUP BY profile_id ), last_event AS ( SELECT profile_id, toDate(created_at) AS event_date FROM cohort_events_mv WHERE project_id = {p5:String} AND name IN {p6:Array(String)} AND created_at >= toDateTime({p7:String}) AND created_at <= toDateTime({p8:String}) + INTERVAL {p9:UInt64} WEEK ), retention_matrix AS ( SELECT f.cohort_interval, l.profile_id, dateDiff({p10:String}, f.cohort_interval, toStartOfWeek(l.event_date)) AS x_after_cohort FROM cohort_users AS f INNER JOIN last_event AS l ON f.userID = l.profile_id WHERE l.event_date >= f.cohort_interval AND dateDiff({p11:String}, f.cohort_interval, toStartOfWeek(l.event_date)) <= {p12:UInt64} ), interval_users AS ( SELECT cohort_interval, groupUniqArrayIf(profile_id, x_after_cohort >= {p13:UInt64}) AS interval_0_users, groupUniqArrayIf(profile_id, x_after_cohort >= {p14:UInt64}) AS interval_1_users, groupUniqArrayIf(profile_id, x_after_cohort >= {p15:UInt64}) AS interval_2_users, groupUniqArrayIf(profile_id, x_after_cohort >= {p16:UInt64}) AS interval_3_users, groupUniqArrayIf(profile_id, x_after_cohort >= {p17:UInt64}) AS interval_4_users, groupUniqArrayIf(profile_id, x_after_cohort >= {p18:UInt64}) AS interval_5_users FROM retention_matrix GROUP BY cohort_interval ), cohort_sizes AS ( SELECT cohort_interval, COUNT(DISTINCT userID) AS total_first_event_count FROM cohort_users GROUP BY cohort_interval ) SELECT interval_users.cohort_interval AS cohort_interval, cs.total_first_event_count AS total_first_event_count, length(interval_0_users) AS interval_0_user_count, length(interval_1_users) AS interval_1_user_count, length(interval_2_users) AS interval_2_user_count, length(interval_3_users) AS interval_3_user_count, length(interval_4_users) AS interval_4_user_count, length(interval_5_users) AS interval_5_user_count FROM interval_users LEFT JOIN cohort_sizes AS cs ON interval_users.cohort_interval = cs.cohort_interval ORDER BY cohort_interval ASC+-- V2 params: {"p1":"secure-privacy","p2":["session_start","screen_view"],"p3":"2026-06-01 00:00:00","p4":"2026-07-12 23:59:59","p5":"secure-privacy","p6":["screen_view","session_end"],"p7":"2026-06-01 00:00:00","p8":"2026-07-12 23:59:59","p9":5,"p10":"WEEK","p11":"WEEK","p12":5,"p13":0,"p14":1,"p15":2,"p16":3,"p17":4,"p18":5}+```++### retentionCohortQuery — month interval, any event++**IDENTICAL** — rows V1/V2 = 1/1; rows_read V1/V2 = 114688/114688; wall V1/V2 = 23 ms / 24 ms.++```sql+-- V1+WITH cohort_users AS ( SELECT profile_id AS userID, toStartOfMonth(min(created_at)) AS cohort_interval FROM cohort_events_mv WHERE project_id = 'secure-privacy' AND created_at >= toDateTime('2026-04-01 00:00:00') AND created_at <= toDateTime('2026-07-01 00:00:00') GROUP BY profile_id ), last_event AS ( SELECT profile_id, toDate(created_at) AS event_date FROM cohort_events_mv WHERE project_id = 'secure-privacy' AND created_at >= toDateTime('2026-04-01 00:00:00') AND created_at <= toDateTime('2026-07-01 00:00:00') + INTERVAL 3 MONTH ), retention_matrix AS ( SELECT f.cohort_interval, l.profile_id, dateDiff('MONTH', f.cohort_interval, toStartOfMonth(l.event_date)) AS x_after_cohort FROM cohort_users AS f INNER JOIN last_event AS l ON f.userID = l.profile_id WHERE l.event_date >= f.cohort_interval AND dateDiff('MONTH', f.cohort_interval, toStartOfMonth(l.event_date)) <= 3 ), interval_users AS ( SELECT cohort_interval, groupUniqArrayIf(profile_id, x_after_cohort >= 0) AS interval_0_users, groupUniqArrayIf(profile_id, x_after_cohort >= 1) AS interval_1_users, groupUniqArrayIf(profile_id, x_after_cohort >= 2) AS interval_2_users, groupUniqArrayIf(profile_id, x_after_cohort >= 3) AS interval_3_users FROM retention_matrix GROUP BY cohort_interval ), cohort_sizes AS ( SELECT cohort_interval, COUNT(DISTINCT userID) AS total_first_event_count FROM cohort_users GROUP BY cohort_interval ) SELECT interval_users.cohort_interval AS cohort_interval, cs.total_first_event_count AS total_first_event_count, length(interval_0_users) AS interval_0_user_count, length(interval_1_users) AS interval_1_user_count, length(interval_2_users) AS interval_2_user_count, length(interval_3_users) AS interval_3_user_count FROM interval_users LEFT JOIN cohort_sizes AS cs ON interval_users.cohort_interval = cs.cohort_interval ORDER BY cohort_interval ASC+-- V2+WITH cohort_users AS ( SELECT profile_id AS userID, toStartOfMonth(min(created_at)) AS cohort_interval FROM cohort_events_mv WHERE project_id = {p1:String} AND created_at >= toDateTime({p2:String}) AND created_at <= toDateTime({p3:String}) GROUP BY profile_id ), last_event AS ( SELECT profile_id, toDate(created_at) AS event_date FROM cohort_events_mv WHERE project_id = {p4:String} AND created_at >= toDateTime({p5:String}) AND created_at <= toDateTime({p6:String}) + INTERVAL {p7:UInt64} MONTH ), retention_matrix AS ( SELECT f.cohort_interval, l.profile_id, dateDiff({p8:String}, f.cohort_interval, toStartOfMonth(l.event_date)) AS x_after_cohort FROM cohort_users AS f INNER JOIN last_event AS l ON f.userID = l.profile_id WHERE l.event_date >= f.cohort_interval AND dateDiff({p9:String}, f.cohort_interval, toStartOfMonth(l.event_date)) <= {p10:UInt64} ), interval_users AS ( SELECT cohort_interval, groupUniqArrayIf(profile_id, x_after_cohort >= {p11:UInt64}) AS interval_0_users, groupUniqArrayIf(profile_id, x_after_cohort >= {p12:UInt64}) AS interval_1_users, groupUniqArrayIf(profile_id, x_after_cohort >= {p13:UInt64}) AS interval_2_users, groupUniqArrayIf(profile_id, x_after_cohort >= {p14:UInt64}) AS interval_3_users FROM retention_matrix GROUP BY cohort_interval ), cohort_sizes AS ( SELECT cohort_interval, COUNT(DISTINCT userID) AS total_first_event_count FROM cohort_users GROUP BY cohort_interval ) SELECT interval_users.cohort_interval AS cohort_interval, cs.total_first_event_count AS total_first_event_count, length(interval_0_users) AS interval_0_user_count, length(interval_1_users) AS interval_1_user_count, length(interval_2_users) AS interval_2_user_count, length(interval_3_users) AS interval_3_user_count FROM interval_users LEFT JOIN cohort_sizes AS cs ON interval_users.cohort_interval = cs.cohort_interval ORDER BY cohort_interval ASC+-- V2 params: {"p1":"secure-privacy","p2":"2026-04-01 00:00:00","p3":"2026-07-01 00:00:00","p4":"secure-privacy","p5":"2026-04-01 00:00:00","p6":"2026-07-01 00:00:00","p7":3,"p8":"MONTH","p9":"MONTH","p10":3,"p11":0,"p12":1,"p13":2,"p14":3}+```++### retentionCohortQuery — property filter -> raw events fallback++**IDENTICAL** — rows V1/V2 = 4/4; rows_read V1/V2 = 196602/196602; wall V1/V2 = 50 ms / 57 ms.++```sql+-- V1+WITH cohort_users AS ( SELECT profile_id AS userID, toDate(min(created_at)) AS cohort_interval FROM events WHERE project_id = 'secure-privacy' AND profile_id != device_id AND country = 'US' AND name = 'screen_view' AND created_at >= toDateTime('2026-07-06 00:00:00') AND created_at <= toDateTime('2026-07-09 23:59:59') GROUP BY profile_id ), last_event AS ( SELECT profile_id, toDate(created_at) AS event_date FROM events WHERE project_id = 'secure-privacy' AND profile_id != device_id AND country = 'US' AND created_at >= toDateTime('2026-07-06 00:00:00') AND created_at <= toDateTime('2026-07-09 23:59:59') + INTERVAL 3 DAY ), retention_matrix AS ( SELECT f.cohort_interval, l.profile_id, dateDiff('DAY', f.cohort_interval, toDate(l.event_date)) AS x_after_cohort FROM cohort_users AS f INNER JOIN last_event AS l ON f.userID = l.profile_id WHERE l.event_date >= f.cohort_interval AND dateDiff('DAY', f.cohort_interval, toDate(l.event_date)) <= 3 ), interval_users AS ( SELECT cohort_interval, groupUniqArrayIf(profile_id, x_after_cohort >= 0) AS interval_0_users, groupUniqArrayIf(profile_id, x_after_cohort >= 1) AS interval_1_users, groupUniqArrayIf(profile_id, x_after_cohort >= 2) AS interval_2_users, groupUniqArrayIf(profile_id, x_after_cohort >= 3) AS interval_3_users FROM retention_matrix GROUP BY cohort_interval ), cohort_sizes AS ( SELECT cohort_interval, COUNT(DISTINCT userID) AS total_first_event_count FROM cohort_users GROUP BY cohort_interval ) SELECT interval_users.cohort_interval AS cohort_interval, cs.total_first_event_count AS total_first_event_count, length(interval_0_users) AS interval_0_user_count, length(interval_1_users) AS interval_1_user_count, length(interval_2_users) AS interval_2_user_count, length(interval_3_users) AS interval_3_user_count FROM interval_users LEFT JOIN cohort_sizes AS cs ON interval_users.cohort_interval = cs.cohort_interval ORDER BY cohort_interval ASC+-- V2+WITH cohort_users AS ( SELECT profile_id AS userID, toDate(min(created_at)) AS cohort_interval FROM events WHERE project_id = {p1:String} AND profile_id != device_id AND country = 'US' AND name = {p2:String} AND created_at >= toDateTime({p3:String}) AND created_at <= toDateTime({p4:String}) GROUP BY profile_id ), last_event AS ( SELECT profile_id, toDate(created_at) AS event_date FROM events WHERE project_id = {p5:String} AND profile_id != device_id AND country = 'US' AND created_at >= toDateTime({p6:String}) AND created_at <= toDateTime({p7:String}) + INTERVAL {p8:UInt64} DAY ), retention_matrix AS ( SELECT f.cohort_interval, l.profile_id, dateDiff({p9:String}, f.cohort_interval, toDate(l.event_date)) AS x_after_cohort FROM cohort_users AS f INNER JOIN last_event AS l ON f.userID = l.profile_id WHERE l.event_date >= f.cohort_interval AND dateDiff({p10:String}, f.cohort_interval, toDate(l.event_date)) <= {p11:UInt64} ), interval_users AS ( SELECT cohort_interval, groupUniqArrayIf(profile_id, x_after_cohort >= {p12:UInt64}) AS interval_0_users, groupUniqArrayIf(profile_id, x_after_cohort >= {p13:UInt64}) AS interval_1_users, groupUniqArrayIf(profile_id, x_after_cohort >= {p14:UInt64}) AS interval_2_users, groupUniqArrayIf(profile_id, x_after_cohort >= {p15:UInt64}) AS interval_3_users FROM retention_matrix GROUP BY cohort_interval ), cohort_sizes AS ( SELECT cohort_interval, COUNT(DISTINCT userID) AS total_first_event_count FROM cohort_users GROUP BY cohort_interval ) SELECT interval_users.cohort_interval AS cohort_interval, cs.total_first_event_count AS total_first_event_count, length(interval_0_users) AS interval_0_user_count, length(interval_1_users) AS interval_1_user_count, length(interval_2_users) AS interval_2_user_count, length(interval_3_users) AS interval_3_user_count FROM interval_users LEFT JOIN cohort_sizes AS cs ON interval_users.cohort_interval = cs.cohort_interval ORDER BY cohort_interval ASC+-- V2 params: {"p1":"secure-privacy","p2":"screen_view","p3":"2026-07-06 00:00:00","p4":"2026-07-09 23:59:59","p5":"secure-privacy","p6":"2026-07-06 00:00:00","p7":"2026-07-09 23:59:59","p8":3,"p9":"DAY","p10":"DAY","p11":3,"p12":0,"p13":1,"p14":2,"p15":3}+```++### retentionCohortQuery — hour interval (toDate/DAY mapping), any event++**IDENTICAL** — rows V1/V2 = 3/3; rows_read V1/V2 = 3579904/3579904; wall V1/V2 = 90 ms / 95 ms.++```sql+-- V1+WITH cohort_users AS ( SELECT profile_id AS userID, toDate(min(created_at)) AS cohort_interval FROM cohort_events_mv WHERE project_id = 'dream-mate' AND created_at >= toDateTime('2026-07-06 00:00:00') AND created_at <= toDateTime('2026-07-08 00:00:00') GROUP BY profile_id ), last_event AS ( SELECT profile_id, toDate(created_at) AS event_date FROM cohort_events_mv WHERE project_id = 'dream-mate' AND created_at >= toDateTime('2026-07-06 00:00:00') AND created_at <= toDateTime('2026-07-08 00:00:00') + INTERVAL 2 DAY ), retention_matrix AS ( SELECT f.cohort_interval, l.profile_id, dateDiff('DAY', f.cohort_interval, toDate(l.event_date)) AS x_after_cohort FROM cohort_users AS f INNER JOIN last_event AS l ON f.userID = l.profile_id WHERE l.event_date >= f.cohort_interval AND dateDiff('DAY', f.cohort_interval, toDate(l.event_date)) <= 2 ), interval_users AS ( SELECT cohort_interval, groupUniqArrayIf(profile_id, x_after_cohort >= 0) AS interval_0_users, groupUniqArrayIf(profile_id, x_after_cohort >= 1) AS interval_1_users, groupUniqArrayIf(profile_id, x_after_cohort >= 2) AS interval_2_users FROM retention_matrix GROUP BY cohort_interval ), cohort_sizes AS ( SELECT cohort_interval, COUNT(DISTINCT userID) AS total_first_event_count FROM cohort_users GROUP BY cohort_interval ) SELECT interval_users.cohort_interval AS cohort_interval, cs.total_first_event_count AS total_first_event_count, length(interval_0_users) AS interval_0_user_count, length(interval_1_users) AS interval_1_user_count, length(interval_2_users) AS interval_2_user_count FROM interval_users LEFT JOIN cohort_sizes AS cs ON interval_users.cohort_interval = cs.cohort_interval ORDER BY cohort_interval ASC+-- V2+WITH cohort_users AS ( SELECT profile_id AS userID, toDate(min(created_at)) AS cohort_interval FROM cohort_events_mv WHERE project_id = {p1:String} AND created_at >= toDateTime({p2:String}) AND created_at <= toDateTime({p3:String}) GROUP BY profile_id ), last_event AS ( SELECT profile_id, toDate(created_at) AS event_date FROM cohort_events_mv WHERE project_id = {p4:String} AND created_at >= toDateTime({p5:String}) AND created_at <= toDateTime({p6:String}) + INTERVAL {p7:UInt64} DAY ), retention_matrix AS ( SELECT f.cohort_interval, l.profile_id, dateDiff({p8:String}, f.cohort_interval, toDate(l.event_date)) AS x_after_cohort FROM cohort_users AS f INNER JOIN last_event AS l ON f.userID = l.profile_id WHERE l.event_date >= f.cohort_interval AND dateDiff({p9:String}, f.cohort_interval, toDate(l.event_date)) <= {p10:UInt64} ), interval_users AS ( SELECT cohort_interval, groupUniqArrayIf(profile_id, x_after_cohort >= {p11:UInt64}) AS interval_0_users, groupUniqArrayIf(profile_id, x_after_cohort >= {p12:UInt64}) AS interval_1_users, groupUniqArrayIf(profile_id, x_after_cohort >= {p13:UInt64}) AS interval_2_users FROM retention_matrix GROUP BY cohort_interval ), cohort_sizes AS ( SELECT cohort_interval, COUNT(DISTINCT userID) AS total_first_event_count FROM cohort_users GROUP BY cohort_interval ) SELECT interval_users.cohort_interval AS cohort_interval, cs.total_first_event_count AS total_first_event_count, length(interval_0_users) AS interval_0_user_count, length(interval_1_users) AS interval_1_user_count, length(interval_2_users) AS interval_2_user_count FROM interval_users LEFT JOIN cohort_sizes AS cs ON interval_users.cohort_interval = cs.cohort_interval ORDER BY cohort_interval ASC+-- V2 params: {"p1":"dream-mate","p2":"2026-07-06 00:00:00","p3":"2026-07-08 00:00:00","p4":"dream-mate","p5":"2026-07-06 00:00:00","p6":"2026-07-08 00:00:00","p7":2,"p8":"DAY","p9":"DAY","p10":2,"p11":0,"p12":1,"p13":2}+```++### retentionCohortQuery — ISO dates with T separator++**IDENTICAL** — rows V1/V2 = 4/4; rows_read V1/V2 = 1474560/1474560; wall V1/V2 = 55 ms / 59 ms.++```sql+-- V1+WITH cohort_users AS ( SELECT profile_id AS userID, toDate(min(created_at)) AS cohort_interval FROM cohort_events_mv WHERE project_id = 'verdict' AND created_at >= toDateTime('2026-07-06 00:00:00') AND created_at <= toDateTime('2026-07-09 23:59:59') GROUP BY profile_id ), last_event AS ( SELECT profile_id, toDate(created_at) AS event_date FROM cohort_events_mv WHERE project_id = 'verdict' AND created_at >= toDateTime('2026-07-06 00:00:00') AND created_at <= toDateTime('2026-07-09 23:59:59') + INTERVAL 3 DAY ), retention_matrix AS ( SELECT f.cohort_interval, l.profile_id, dateDiff('DAY', f.cohort_interval, toDate(l.event_date)) AS x_after_cohort FROM cohort_users AS f INNER JOIN last_event AS l ON f.userID = l.profile_id WHERE l.event_date >= f.cohort_interval AND dateDiff('DAY', f.cohort_interval, toDate(l.event_date)) <= 3 ), interval_users AS ( SELECT cohort_interval, groupUniqArrayIf(profile_id, x_after_cohort >= 0) AS interval_0_users, groupUniqArrayIf(profile_id, x_after_cohort >= 1) AS interval_1_users, groupUniqArrayIf(profile_id, x_after_cohort >= 2) AS interval_2_users, groupUniqArrayIf(profile_id, x_after_cohort >= 3) AS interval_3_users FROM retention_matrix GROUP BY cohort_interval ), cohort_sizes AS ( SELECT cohort_interval, COUNT(DISTINCT userID) AS total_first_event_count FROM cohort_users GROUP BY cohort_interval ) SELECT interval_users.cohort_interval AS cohort_interval, cs.total_first_event_count AS total_first_event_count, length(interval_0_users) AS interval_0_user_count, length(interval_1_users) AS interval_1_user_count, length(interval_2_users) AS interval_2_user_count, length(interval_3_users) AS interval_3_user_count FROM interval_users LEFT JOIN cohort_sizes AS cs ON interval_users.cohort_interval = cs.cohort_interval ORDER BY cohort_interval ASC+-- V2+WITH cohort_users AS ( SELECT profile_id AS userID, toDate(min(created_at)) AS cohort_interval FROM cohort_events_mv WHERE project_id = {p1:String} AND created_at >= toDateTime({p2:String}) AND created_at <= toDateTime({p3:String}) GROUP BY profile_id ), last_event AS ( SELECT profile_id, toDate(created_at) AS event_date FROM cohort_events_mv WHERE project_id = {p4:String} AND created_at >= toDateTime({p5:String}) AND created_at <= toDateTime({p6:String}) + INTERVAL {p7:UInt64} DAY ), retention_matrix AS ( SELECT f.cohort_interval, l.profile_id, dateDiff({p8:String}, f.cohort_interval, toDate(l.event_date)) AS x_after_cohort FROM cohort_users AS f INNER JOIN last_event AS l ON f.userID = l.profile_id WHERE l.event_date >= f.cohort_interval AND dateDiff({p9:String}, f.cohort_interval, toDate(l.event_date)) <= {p10:UInt64} ), interval_users AS ( SELECT cohort_interval, groupUniqArrayIf(profile_id, x_after_cohort >= {p11:UInt64}) AS interval_0_users, groupUniqArrayIf(profile_id, x_after_cohort >= {p12:UInt64}) AS interval_1_users, groupUniqArrayIf(profile_id, x_after_cohort >= {p13:UInt64}) AS interval_2_users, groupUniqArrayIf(profile_id, x_after_cohort >= {p14:UInt64}) AS interval_3_users FROM retention_matrix GROUP BY cohort_interval ), cohort_sizes AS ( SELECT cohort_interval, COUNT(DISTINCT userID) AS total_first_event_count FROM cohort_users GROUP BY cohort_interval ) SELECT interval_users.cohort_interval AS cohort_interval, cs.total_first_event_count AS total_first_event_count, length(interval_0_users) AS interval_0_user_count, length(interval_1_users) AS interval_1_user_count, length(interval_2_users) AS interval_2_user_count, length(interval_3_users) AS interval_3_user_count FROM interval_users LEFT JOIN cohort_sizes AS cs ON interval_users.cohort_interval = cs.cohort_interval ORDER BY cohort_interval ASC+-- V2 params: {"p1":"verdict","p2":"2026-07-06 00:00:00","p3":"2026-07-09 23:59:59","p4":"verdict","p5":"2026-07-06 00:00:00","p6":"2026-07-09 23:59:59","p7":3,"p8":"DAY","p9":"DAY","p10":3,"p11":0,"p12":1,"p13":2,"p14":3}+```diff --git a/packages/core/src/modules/chart/src/retention.sql.ts b/packages/core/src/modules/chart/src/retention.sql.tsnew file mode 100644index 000000000..202a81592--- /dev/null+++ b/packages/core/src/modules/chart/src/retention.sql.ts@@ -0,0 +1,267 @@+// Every ClickHouse query the retention service runs, as pure `sql` fragments+// (ADR-013). Converted 1:1 from packages/db/src/services/retention.service.ts+// (M7-004): the SQL text is V1's, with project id, event names, dates, the+// interval unit and the column count bound as `{pN:Type}` parameters instead+// of `sqlstring`-escaped literals. Result-set proofs against the local+// prod-copy are in retention.sql.proof.md.+//+// The filter compiler still renders text (see compiled.ts); its output is the+// only thing spliced into the cohort statement.+//+// Cluster note (docs/ENVIRONMENT.md): `events`, `dau_mv` and `cohort_events_mv`+// are Distributed on Cloud. None of these statements contains an+// `IN (subquery)` — the cohort matrix is a CTE self-join, exactly V1's shape —+// so no `IN` / `GLOBAL IN` decision is made or unmade here.++import { type SqlFragment, sql } from '@openpanel/db/src/clickhouse/sql';+import type { IChartEventFilter } from '@openpanel/validation';+import { compiledText } from './compiled';+import { CHART_TABLE } from './field-resolution';+import { getEventFiltersWhereClause } from './filter-where';++export type IRetentionInterval = 'minute' | 'hour' | 'day' | 'week' | 'month';+export type IRetentionCriteria = 'on' | 'on_or_after';++/** ClickHouse `toStartOf*` function per interval — a closed set, never input. */+export const SQL_START_OF: Record<IRetentionInterval, string> = {+ minute: 'toDate',+ hour: 'toDate',+ day: 'toDate',+ week: 'toStartOfWeek',+ month: 'toStartOfMonth',+};++/** `dateDiff` unit / `INTERVAL n <unit>` per interval — also a closed set. */+export const SQL_INTERVAL: Record<IRetentionInterval, string> = {+ minute: 'DAY',+ hour: 'DAY',+ day: 'DAY',+ week: 'WEEK',+ month: 'MONTH',+};++const COUNT_CRITERIA: Record<IRetentionCriteria, string> = {+ on_or_after: '>=',+ on: '=',+};++/** Week-over-week active-user retention, one row per week. */+export function retentionSeriesQuery(projectId: string): SqlFragment {+ return sql`+ WITH weekly_active AS (+ SELECT+ profile_id,+ toStartOfWeek(created_at) AS week+ FROM ${sql.id(CHART_TABLE.events)}+ WHERE project_id = ${sql.string(projectId)}+ AND profile_id != device_id+ GROUP BY profile_id, week+ )+ SELECT+ cur.week AS date,+ countDistinct(cur.profile_id) AS active_users,+ countDistinct(nxt.profile_id) AS retained_users,+ (100 * (countDistinct(nxt.profile_id) / CAST(countDistinct(cur.profile_id), 'Float64'))) AS retention+ FROM weekly_active AS cur+ LEFT JOIN weekly_active AS nxt+ ON cur.profile_id = nxt.profile_id+ AND nxt.week = cur.week + toIntervalWeek(1)+ GROUP BY date+ ORDER BY date ASC+ -- Unmatched LEFT JOIN rows must be NULL (not the empty-string default),+ -- otherwise countDistinct(nxt.profile_id) counts '' as a retained user.+ SETTINGS join_use_nulls = 1`;+}++/** Rolling N-day unique actives off the daily-active materialized view. */+export function rollingActiveUsersQuery(+ projectId: string,+ days: number+): SqlFragment {+ return sql`+ SELECT+ date,+ uniqMerge(profile_id) AS users+ FROM+ (+ SELECT+ date + n AS date,+ profile_id,+ project_id+ FROM+ (+ SELECT *+ FROM ${sql.id(CHART_TABLE.dauMv)}+ WHERE project_id = ${sql.string(projectId)}+ )+ ARRAY JOIN range(${sql.uint64(days)}) AS n+ )+ WHERE project_id = ${sql.string(projectId)}+ GROUP BY date`;+}++/** Days-since-last-seen distribution over identified profiles. */+export function retentionLastSeenSeriesQuery(projectId: string): SqlFragment {+ return sql`+ WITH last_active AS (+ SELECT+ max(created_at) AS last_active,+ profile_id+ FROM ${sql.id(CHART_TABLE.events)}+ WHERE (project_id = ${sql.string(projectId)}) AND (device_id != profile_id)+ GROUP BY profile_id+ )+ SELECT+ dateDiff('day', last_active, today()) AS days,+ countDistinct(profile_id) AS users+ FROM last_active+ GROUP BY days+ ORDER BY days ASC`;+}++export interface RetentionCohortQueryInput {+ projectId: string;+ firstEvent?: string[];+ secondEvent?: string[];+ criteria: IRetentionCriteria;+ interval: IRetentionInterval;+ /** Already normalised to `yyyy-MM-dd HH:mm:ss`. */+ start: string;+ end: string;+ /** Number of retention columns beyond the cohort's own interval. */+ diffInterval: number;+ filters: IChartEventFilter[];+}++/** V1's `name = …` / `name IN (…)`; `sql.empty` means "any event". */+function eventNameWhere(events: string[] | undefined): SqlFragment {+ if (!events || events.length === 0) {+ return sql.empty;+ }+ if (events.length === 1) {+ return sql`AND name = ${sql.string(events[0] as string)}`;+ }+ return sql`AND name IN ${sql.array('String', events)}`;+}++/**+ * The cohort retention matrix (V1 `getRetentionCohort`'s five-CTE query).+ *+ * Source table is a closed two-way choice, not a caller input: cohort+ * membership filters stay on the skinny `cohort_events_mv`, anything reading a+ * property or column falls back to raw `events`.+ */+export function retentionCohortQuery(+ input: RetentionCohortQueryInput+): SqlFragment {+ const {+ projectId,+ firstEvent,+ secondEvent,+ criteria,+ interval,+ start,+ end,+ diffInterval,+ filters,+ } = input;++ const sqlInterval = SQL_INTERVAL[interval];+ const toStartOf = compiledText(SQL_START_OF[interval]);+ const intervalUnit = compiledText(sqlInterval);+ const countCriteria = compiledText(COUNT_CRITERIA[criteria]);++ const needRawEvents = filters.some(+ (filter) =>+ filter.operator !== 'inCohort' && filter.operator !== 'notInCohort'+ );+ const source = sql.id(+ needRawEvents ? CHART_TABLE.events : CHART_TABLE.cohortEventsMv+ );++ const baseConditions: SqlFragment[] = [+ sql`project_id = ${sql.string(projectId)}`,+ ];+ if (needRawEvents) {+ // cohort_events_mv only stores identified-user rows; replicate that here.+ baseConditions.push(sql`profile_id != device_id`);+ }+ if (filters.length > 0) {+ baseConditions.push(+ ...Object.values(getEventFiltersWhereClause(filters, projectId)).map(+ compiledText+ )+ );+ }+ const baseWhere = sql.join(baseConditions, ' AND ');++ const columns = Array.from({ length: diffInterval + 1 }, (_, i) => i);+ const usersSelect = sql.join(+ columns.map(+ (index) =>+ sql`groupUniqArrayIf(profile_id, x_after_cohort ${countCriteria} ${sql.uint64(index)}) AS interval_${compiledText(String(index))}_users`+ )+ );+ const countsSelect = sql.join(+ columns.map(+ (index) =>+ sql`length(interval_${compiledText(String(index))}_users) AS interval_${compiledText(String(index))}_user_count`+ )+ );++ return sql`+ WITH+ cohort_users AS (+ SELECT+ profile_id AS userID,+ ${toStartOf}(min(created_at)) AS cohort_interval+ FROM ${source}+ WHERE ${baseWhere}+ ${eventNameWhere(firstEvent)}+ AND created_at >= toDateTime(${sql.string(start)})+ AND created_at <= toDateTime(${sql.string(end)})+ GROUP BY profile_id+ ),+ last_event AS (+ SELECT+ profile_id,+ toDate(created_at) AS event_date+ FROM ${source}+ WHERE ${baseWhere}+ ${eventNameWhere(secondEvent)}+ AND created_at >= toDateTime(${sql.string(start)})+ AND created_at <= toDateTime(${sql.string(end)}) + INTERVAL ${sql.uint64(diffInterval)} ${intervalUnit}+ ),+ retention_matrix AS (+ SELECT+ f.cohort_interval,+ l.profile_id,+ dateDiff(${sql.string(sqlInterval)}, f.cohort_interval, ${toStartOf}(l.event_date)) AS x_after_cohort+ FROM cohort_users AS f+ INNER JOIN last_event AS l ON f.userID = l.profile_id+ WHERE l.event_date >= f.cohort_interval+ AND dateDiff(${sql.string(sqlInterval)}, f.cohort_interval, ${toStartOf}(l.event_date)) <= ${sql.uint64(diffInterval)}+ ),+ interval_users AS (+ SELECT+ cohort_interval,+ ${usersSelect}+ FROM retention_matrix+ GROUP BY cohort_interval+ ),+ cohort_sizes AS (+ SELECT+ cohort_interval,+ COUNT(DISTINCT userID) AS total_first_event_count+ FROM cohort_users+ GROUP BY cohort_interval+ )+ SELECT+ interval_users.cohort_interval AS cohort_interval,+ cs.total_first_event_count AS total_first_event_count,+ ${countsSelect}+ FROM interval_users+ LEFT JOIN cohort_sizes AS cs ON interval_users.cohort_interval = cs.cohort_interval+ ORDER BY cohort_interval ASC+ `;+}diff --git a/packages/core/src/modules/chart/src/run-query.ts b/packages/core/src/modules/chart/src/run-query.tsnew file mode 100644index 000000000..a623f4474--- /dev/null+++ b/packages/core/src/modules/chart/src/run-query.ts@@ -0,0 +1,23 @@+// The one place the chart module reaches @openpanel/db's ClickHouse client.+//+// The import is lazy on purpose: the core barrel pulls this module into nearly+// every test file, and constructing @openpanel/db's clients at import time+// costs a pino-pretty worker per file.++import type { SqlFragment } from '@openpanel/db/src/clickhouse/sql';++export function loadChClient() {+ return import('@openpanel/db/src/clickhouse/client');+}++/** `timezone` travels as `session_timezone`, exactly as clix sent it. */+export async function runQuery<T extends object>(+ statement: SqlFragment,+ timezone?: string+): Promise<T[]> {+ const { chQuery } = await loadChClient();+ return chQuery<T>(+ statement,+ timezone ? { session_timezone: timezone } : undefined+ );+}diff --git a/packages/core/src/modules/chart/src/sankey.sql.proof.md b/packages/core/src/modules/chart/src/sankey.sql.proof.mdnew file mode 100644index 000000000..b1bb06c71--- /dev/null+++ b/packages/core/src/modules/chart/src/sankey.sql.proof.md@@ -0,0 +1,286 @@+# sankey.sql.ts — V1 -> V2 result-set proof (M7-004)++Both statements of every sankey mode were executed twice against the same data — once as V1 (the `git HEAD` `packages/db/src/services/sankey.service.ts`, `c4760ac2`, captured by wrapping the shared ClickHouse client) and once as V2 (the fragment the core service sends) — through one `@clickhouse/client` with `format: 'JSON'` and the same `session_timezone`. The two responses were compared on `data` and `meta`. In addition, the two services' full JSON results (nodes + links, after all of the post-processing) were compared for all 12 cases: **12/12 identical**.++- **Date**: 2026-09-04. **Data**: local prod-copy `openpanel` (319,499,000 events, static). Projects: `secure-privacy`, `bayse` (America/New_York).+- **Machine**: single-node ClickHouse on a 4-vCPU box — timings are directional only; production is 2 shards x 2 replicas (`docs/ENVIRONMENT.md`).+- **Cluster note.** The `HAVING session_id IN (SELECT session_id FROM start_event_sessions)` clauses are V1's and are ported **verbatim**: no `IN` was converted to or from `GLOBAL IN`. Whether they should be `GLOBAL IN` on the 2x2 cluster is a pre-existing question this conversion neither answers nor changes (`docs/ENVIRONMENT.md`), and it is recorded as debt in the task report.+- **One case reads IDENTICAL AS SET.** The transitions statement orders by `step ASC, value DESC`, which leaves rows sharing a `(step, value)` pair in arbitrary order. Measured on 2026-09-04: **V1's own statement, executed 4 times unchanged, returned 4 different row orders and one identical row set.** Row order is therefore not part of the contract — the service groups transitions by step and source before using them — so that case is compared as a set, exactly as M7-003 compared the chart statements.++- **Verdict**: 23 cases, 0 not identical.++### sankey topEntries — after, screen_view, 3 steps (golden insights-user-flow-after)++**IDENTICAL** — rows V1/V2 = 1/1; rows_read V1/V2 = 352206/352206; wall V1/V2 = 117 ms / 81 ms; clickhouse_settings: {"session_timezone":"UTC"} (both).++```sql+-- V1+WITH session_paths AS (WITH start_event_sessions AS (SELECT session_id FROM events WHERE project_id = 'secure-privacy' AND name = 'screen_view' AND created_at BETWEEN toDateTime('2026-07-06 00:00:00') AND toDateTime('2026-07-12 23:59:59') GROUP BY session_id), events_deduped_cte AS (WITH ordered_events AS (SELECT session_id, name as event_name, created_at FROM events WHERE project_id = 'secure-privacy' AND created_at BETWEEN toDateTime('2026-07-06 00:00:00') AND toDateTime('2026-07-12 23:59:59') ORDER BY session_id ASC, created_at ASC) SELECT session_id, arrayFilter( (x, i) -> i = 1 OR x != events_raw[i - 1], groupArray(event_name) as events_raw, arrayEnumerate(events_raw) ) as events_deduped FROM ordered_events GROUP BY session_id), events_sliced_cte AS (SELECT session_id, arraySlice(events_deduped, arrayFirstIndex(x -> x = 'screen_view', events_deduped), 3) as events_sliced FROM events_deduped_cte HAVING session_id IN (SELECT session_id FROM start_event_sessions)) SELECT session_id, if( arrayFirstIndex(x -> x > 1, arrayEnumerateUniq(events_sliced)) = 0, events_sliced, arraySlice( events_sliced, 1, arrayFirstIndex(x -> x > 1, arrayEnumerateUniq(events_sliced)) - 1 ) ) as events, events[1] as entry_event FROM events_sliced_cte HAVING length(events) >= 2) SELECT entry_event, count() as count FROM session_paths GROUP BY entry_event ORDER BY count DESC LIMIT 3+-- V2+WITH session_paths AS (WITH start_event_sessions AS (SELECT session_id FROM events WHERE project_id = {p1:String} AND name = {p2:String} AND created_at BETWEEN toDateTime({p3:String}) AND toDateTime({p4:String}) GROUP BY session_id), events_deduped_cte AS (WITH ordered_events AS (SELECT session_id, name as event_name, created_at FROM events WHERE project_id = {p5:String} AND created_at BETWEEN toDateTime({p6:String}) AND toDateTime({p7:String}) ORDER BY session_id ASC, created_at ASC) SELECT session_id, arrayFilter( (x, i) -> i = 1 OR x != events_raw[i - 1], groupArray(event_name) as events_raw, arrayEnumerate(events_raw) ) as events_deduped FROM ordered_events GROUP BY session_id), events_sliced_cte AS (SELECT session_id, arraySlice(events_deduped, arrayFirstIndex(x -> x = {p8:String}, events_deduped), {p9:UInt64}) as events_sliced FROM events_deduped_cte HAVING session_id IN (SELECT session_id FROM start_event_sessions)) SELECT session_id, if( arrayFirstIndex(x -> x > 1, arrayEnumerateUniq(events_sliced)) = 0, events_sliced, arraySlice( events_sliced, 1, arrayFirstIndex(x -> x > 1, arrayEnumerateUniq(events_sliced)) - 1 ) ) as events, events[1] as entry_event FROM events_sliced_cte HAVING length(events) >= {p10:UInt64}) SELECT entry_event, count() as count FROM session_paths GROUP BY entry_event ORDER BY count DESC LIMIT {p11:UInt64}+-- V2 params: {"p1":"secure-privacy","p2":"screen_view","p3":"2026-07-06 00:00:00","p4":"2026-07-12 23:59:59","p5":"secure-privacy","p6":"2026-07-06 00:00:00","p7":"2026-07-12 23:59:59","p8":"screen_view","p9":3,"p10":2,"p11":3}+```++### sankey transitions — after, screen_view, 3 steps (golden insights-user-flow-after)++**IDENTICAL** — rows V1/V2 = 19/19; rows_read V1/V2 = 352206/352206; wall V1/V2 = 120 ms / 127 ms; clickhouse_settings: {"session_timezone":"UTC"} (both).++```sql+-- V1+WITH session_paths_base AS (WITH start_event_sessions AS (SELECT session_id FROM events WHERE project_id = 'secure-privacy' AND name = 'screen_view' AND created_at BETWEEN toDateTime('2026-07-06 00:00:00') AND toDateTime('2026-07-12 23:59:59') GROUP BY session_id), events_deduped_cte AS (WITH ordered_events AS (SELECT session_id, name as event_name, created_at FROM events WHERE project_id = 'secure-privacy' AND created_at BETWEEN toDateTime('2026-07-06 00:00:00') AND toDateTime('2026-07-12 23:59:59') ORDER BY session_id ASC, created_at ASC) SELECT session_id, arrayFilter( (x, i) -> i = 1 OR x != events_raw[i - 1], groupArray(event_name) as events_raw, arrayEnumerate(events_raw) ) as events_deduped FROM ordered_events GROUP BY session_id), events_sliced_cte AS (SELECT session_id, arraySlice(events_deduped, arrayFirstIndex(x -> x = 'screen_view', events_deduped), 3) as events_sliced FROM events_deduped_cte HAVING session_id IN (SELECT session_id FROM start_event_sessions)) SELECT session_id, if( arrayFirstIndex(x -> x > 1, arrayEnumerateUniq(events_sliced)) = 0, events_sliced, arraySlice( events_sliced, 1, arrayFirstIndex(x -> x > 1, arrayEnumerateUniq(events_sliced)) - 1 ) ) as events, events[1] as entry_event FROM events_sliced_cte HAVING length(events) >= 2), session_paths AS (SELECT session_id, events FROM session_paths_base HAVING events[1] IN ('screen_view')) SELECT pair.1 as source, pair.2 as target, pair.3 as step, count() as value FROM ((SELECT arrayJoin(arrayMap(i -> (events[i], events[i + 1], i), range(1, length(events)))) as pair FROM session_paths WHERE length(events) >= 2)) GROUP BY source, target, step ORDER BY step ASC, value DESC+-- V2+WITH session_paths_base AS (WITH start_event_sessions AS (SELECT session_id FROM events WHERE project_id = {p1:String} AND name = {p2:String} AND created_at BETWEEN toDateTime({p3:String}) AND toDateTime({p4:String}) GROUP BY session_id), events_deduped_cte AS (WITH ordered_events AS (SELECT session_id, name as event_name, created_at FROM events WHERE project_id = {p5:String} AND created_at BETWEEN toDateTime({p6:String}) AND toDateTime({p7:String}) ORDER BY session_id ASC, created_at ASC) SELECT session_id, arrayFilter( (x, i) -> i = 1 OR x != events_raw[i - 1], groupArray(event_name) as events_raw, arrayEnumerate(events_raw) ) as events_deduped FROM ordered_events GROUP BY session_id), events_sliced_cte AS (SELECT session_id, arraySlice(events_deduped, arrayFirstIndex(x -> x = {p8:String}, events_deduped), {p9:UInt64}) as events_sliced FROM events_deduped_cte HAVING session_id IN (SELECT session_id FROM start_event_sessions)) SELECT session_id, if( arrayFirstIndex(x -> x > 1, arrayEnumerateUniq(events_sliced)) = 0, events_sliced, arraySlice( events_sliced, 1, arrayFirstIndex(x -> x > 1, arrayEnumerateUniq(events_sliced)) - 1 ) ) as events, events[1] as entry_event FROM events_sliced_cte HAVING length(events) >= {p10:UInt64}), session_paths AS (SELECT session_id, events FROM session_paths_base HAVING events[1] IN {p11:Array(String)}) SELECT pair.1 as source, pair.2 as target, pair.3 as step, count() as value FROM ((SELECT arrayJoin(arrayMap(i -> (events[i], events[i + 1], i), range(1, length(events)))) as pair FROM session_paths WHERE length(events) >= 2)) GROUP BY source, target, step ORDER BY step ASC, value DESC+-- V2 params: {"p1":"secure-privacy","p2":"screen_view","p3":"2026-07-06 00:00:00","p4":"2026-07-12 23:59:59","p5":"secure-privacy","p6":"2026-07-06 00:00:00","p7":"2026-07-12 23:59:59","p8":"screen_view","p9":3,"p10":2,"p11":["screen_view"]}+```++### sankey topEntries — before, session_end, 3 steps (golden insights-user-flow-before)++**IDENTICAL** — rows V1/V2 = 3/3; rows_read V1/V2 = 352206/352206; wall V1/V2 = 121 ms / 102 ms; clickhouse_settings: {"session_timezone":"UTC"} (both).++```sql+-- V1+WITH session_paths AS (WITH start_event_sessions AS (SELECT session_id FROM events WHERE project_id = 'secure-privacy' AND name = 'session_end' AND created_at BETWEEN toDateTime('2026-07-06 00:00:00') AND toDateTime('2026-07-12 23:59:59') GROUP BY session_id), events_deduped_cte AS (WITH ordered_events AS (SELECT session_id, name as event_name, created_at FROM events WHERE project_id = 'secure-privacy' AND created_at BETWEEN toDateTime('2026-07-06 00:00:00') AND toDateTime('2026-07-12 23:59:59') ORDER BY session_id ASC, created_at ASC) SELECT session_id, arrayFilter( (x, i) -> i = 1 OR x != events_raw[i - 1], groupArray(event_name) as events_raw, arrayEnumerate(events_raw) ) as events_deduped FROM ordered_events GROUP BY session_id), events_sliced_cte AS (SELECT session_id, arraySlice( events_deduped, greatest(1, arrayFirstIndex(x -> x = 'session_end', events_deduped) - 3 + 1), arrayFirstIndex(x -> x = 'session_end', events_deduped) - greatest(1, arrayFirstIndex(x -> x = 'session_end', events_deduped) - 3 + 1) + 1 ) as events_sliced FROM events_deduped_cte HAVING session_id IN (SELECT session_id FROM start_event_sessions)) SELECT session_id, events_sliced as events, events[1] as entry_event FROM events_sliced_cte HAVING length(events) >= 2) SELECT entry_event, count() as count FROM session_paths GROUP BY entry_event ORDER BY count DESC LIMIT 3+-- V2+WITH session_paths AS (WITH start_event_sessions AS (SELECT session_id FROM events WHERE project_id = {p1:String} AND name = {p2:String} AND created_at BETWEEN toDateTime({p3:String}) AND toDateTime({p4:String}) GROUP BY session_id), events_deduped_cte AS (WITH ordered_events AS (SELECT session_id, name as event_name, created_at FROM events WHERE project_id = {p5:String} AND created_at BETWEEN toDateTime({p6:String}) AND toDateTime({p7:String}) ORDER BY session_id ASC, created_at ASC) SELECT session_id, arrayFilter( (x, i) -> i = 1 OR x != events_raw[i - 1], groupArray(event_name) as events_raw, arrayEnumerate(events_raw) ) as events_deduped FROM ordered_events GROUP BY session_id), events_sliced_cte AS (SELECT session_id, arraySlice( events_deduped, greatest(1, arrayFirstIndex(x -> x = {p8:String}, events_deduped) - {p9:UInt64} + 1), arrayFirstIndex(x -> x = {p10:String}, events_deduped) - greatest(1, arrayFirstIndex(x -> x = {p11:String}, events_deduped) - {p12:UInt64} + 1) + 1 ) as events_sliced FROM events_deduped_cte HAVING session_id IN (SELECT session_id FROM start_event_sessions)) SELECT session_id, events_sliced as events, events[1] as entry_event FROM events_sliced_cte HAVING length(events) >= {p13:UInt64}) SELECT entry_event, count() as count FROM session_paths GROUP BY entry_event ORDER BY count DESC LIMIT {p14:UInt64}+-- V2 params: {"p1":"secure-privacy","p2":"session_end","p3":"2026-07-06 00:00:00","p4":"2026-07-12 23:59:59","p5":"secure-privacy","p6":"2026-07-06 00:00:00","p7":"2026-07-12 23:59:59","p8":"session_end","p9":3,"p10":"session_end","p11":"session_end","p12":3,"p13":2,"p14":3}+```++### sankey transitions — before, session_end, 3 steps (golden insights-user-flow-before)++**IDENTICAL** — rows V1/V2 = 7/7; rows_read V1/V2 = 352206/352206; wall V1/V2 = 98 ms / 96 ms; clickhouse_settings: {"session_timezone":"UTC"} (both).++```sql+-- V1+WITH session_paths_base AS (WITH start_event_sessions AS (SELECT session_id FROM events WHERE project_id = 'secure-privacy' AND name = 'session_end' AND created_at BETWEEN toDateTime('2026-07-06 00:00:00') AND toDateTime('2026-07-12 23:59:59') GROUP BY session_id), events_deduped_cte AS (WITH ordered_events AS (SELECT session_id, name as event_name, created_at FROM events WHERE project_id = 'secure-privacy' AND created_at BETWEEN toDateTime('2026-07-06 00:00:00') AND toDateTime('2026-07-12 23:59:59') ORDER BY session_id ASC, created_at ASC) SELECT session_id, arrayFilter( (x, i) -> i = 1 OR x != events_raw[i - 1], groupArray(event_name) as events_raw, arrayEnumerate(events_raw) ) as events_deduped FROM ordered_events GROUP BY session_id), events_sliced_cte AS (SELECT session_id, arraySlice( events_deduped, greatest(1, arrayFirstIndex(x -> x = 'session_end', events_deduped) - 3 + 1), arrayFirstIndex(x -> x = 'session_end', events_deduped) - greatest(1, arrayFirstIndex(x -> x = 'session_end', events_deduped) - 3 + 1) + 1 ) as events_sliced FROM events_deduped_cte HAVING session_id IN (SELECT session_id FROM start_event_sessions)) SELECT session_id, events_sliced as events, events[1] as entry_event FROM events_sliced_cte HAVING length(events) >= 2), session_paths AS (SELECT session_id, events FROM session_paths_base HAVING events[1] IN ('session_start', 'link_out', 'screen_view')) SELECT pair.1 as source, pair.2 as target, pair.3 as step, count() as value FROM ((SELECT arrayJoin(arrayMap(i -> (events[i], events[i + 1], i), range(1, length(events)))) as pair FROM session_paths WHERE length(events) >= 2)) GROUP BY source, target, step ORDER BY step ASC, value DESC+-- V2+WITH session_paths_base AS (WITH start_event_sessions AS (SELECT session_id FROM events WHERE project_id = {p1:String} AND name = {p2:String} AND created_at BETWEEN toDateTime({p3:String}) AND toDateTime({p4:String}) GROUP BY session_id), events_deduped_cte AS (WITH ordered_events AS (SELECT session_id, name as event_name, created_at FROM events WHERE project_id = {p5:String} AND created_at BETWEEN toDateTime({p6:String}) AND toDateTime({p7:String}) ORDER BY session_id ASC, created_at ASC) SELECT session_id, arrayFilter( (x, i) -> i = 1 OR x != events_raw[i - 1], groupArray(event_name) as events_raw, arrayEnumerate(events_raw) ) as events_deduped FROM ordered_events GROUP BY session_id), events_sliced_cte AS (SELECT session_id, arraySlice( events_deduped, greatest(1, arrayFirstIndex(x -> x = {p8:String}, events_deduped) - {p9:UInt64} + 1), arrayFirstIndex(x -> x = {p10:String}, events_deduped) - greatest(1, arrayFirstIndex(x -> x = {p11:String}, events_deduped) - {p12:UInt64} + 1) + 1 ) as events_sliced FROM events_deduped_cte HAVING session_id IN (SELECT session_id FROM start_event_sessions)) SELECT session_id, events_sliced as events, events[1] as entry_event FROM events_sliced_cte HAVING length(events) >= {p13:UInt64}), session_paths AS (SELECT session_id, events FROM session_paths_base HAVING events[1] IN {p14:Array(String)}) SELECT pair.1 as source, pair.2 as target, pair.3 as step, count() as value FROM ((SELECT arrayJoin(arrayMap(i -> (events[i], events[i + 1], i), range(1, length(events)))) as pair FROM session_paths WHERE length(events) >= 2)) GROUP BY source, target, step ORDER BY step ASC, value DESC+-- V2 params: {"p1":"secure-privacy","p2":"session_end","p3":"2026-07-06 00:00:00","p4":"2026-07-12 23:59:59","p5":"secure-privacy","p6":"2026-07-06 00:00:00","p7":"2026-07-12 23:59:59","p8":"session_end","p9":3,"p10":"session_end","p11":"session_end","p12":3,"p13":2,"p14":["session_start","link_out","screen_view"]}+```++### sankey topEntries — between, session_start -> session_end, 4 steps (golden insights-user-flow-between)++**IDENTICAL** — rows V1/V2 = 1/1; rows_read V1/V2 = 458695/458695; wall V1/V2 = 138 ms / 170 ms; clickhouse_settings: {"session_timezone":"UTC"} (both).++```sql+-- V1+WITH session_paths AS (WITH between_sessions AS (WITH session_paths AS (WITH start_event_sessions AS (SELECT session_id FROM events WHERE project_id = 'secure-privacy' AND name = 'session_start' AND created_at BETWEEN toDateTime('2026-07-06 00:00:00') AND toDateTime('2026-07-12 23:59:59') GROUP BY session_id), end_event_sessions AS (SELECT session_id FROM events WHERE project_id = 'secure-privacy' AND name = 'session_end' AND created_at BETWEEN toDateTime('2026-07-06 00:00:00') AND toDateTime('2026-07-12 23:59:59') GROUP BY session_id), events_deduped_cte AS (WITH ordered_events AS (SELECT session_id, name as event_name, created_at FROM events WHERE project_id = 'secure-privacy' AND created_at BETWEEN toDateTime('2026-07-06 00:00:00') AND toDateTime('2026-07-12 23:59:59') ORDER BY session_id ASC, created_at ASC) SELECT session_id, arrayFilter( (x, i) -> i = 1 OR x != events_raw[i - 1], groupArray(event_name) as events_raw, arrayEnumerate(events_raw) ) as events_deduped FROM ordered_events GROUP BY session_id), events_sliced_cte AS (SELECT session_id, arraySlice(events_deduped, 1, 4) as events_sliced FROM events_deduped_cte HAVING session_id IN (SELECT session_id FROM start_event_sessions) AND session_id IN (SELECT session_id FROM end_event_sessions)) SELECT session_id, if( arrayFirstIndex(x -> x > 1, arrayEnumerateUniq(events_sliced)) = 0, events_sliced, arraySlice( events_sliced, 1, arrayFirstIndex(x -> x > 1, arrayEnumerateUniq(events_sliced)) - 1 ) ) as events, events[1] as entry_event FROM events_sliced_cte HAVING length(events) >= 2) SELECT session_id, events, arrayFirstIndex(x -> x = 'session_start', events) as start_index, arrayFirstIndex(x -> x = 'session_end', events) as end_index FROM session_paths HAVING start_index > 0 AND end_index > 0 AND start_index < end_index) SELECT session_id, arraySlice(events, start_index, end_index - start_index + 1) as events, events[start_index] as entry_event FROM between_sessions) SELECT entry_event, count() as count FROM session_paths GROUP BY entry_event ORDER BY count DESC LIMIT 3+-- V2+WITH session_paths AS (WITH between_sessions AS (WITH session_paths AS (WITH start_event_sessions AS (SELECT session_id FROM events WHERE project_id = {p1:String} AND name = {p2:String} AND created_at BETWEEN toDateTime({p3:String}) AND toDateTime({p4:String}) GROUP BY session_id), end_event_sessions AS (SELECT session_id FROM events WHERE project_id = {p5:String} AND name = {p6:String} AND created_at BETWEEN toDateTime({p7:String}) AND toDateTime({p8:String}) GROUP BY session_id), events_deduped_cte AS (WITH ordered_events AS (SELECT session_id, name as event_name, created_at FROM events WHERE project_id = {p9:String} AND created_at BETWEEN toDateTime({p10:String}) AND toDateTime({p11:String}) ORDER BY session_id ASC, created_at ASC) SELECT session_id, arrayFilter( (x, i) -> i = 1 OR x != events_raw[i - 1], groupArray(event_name) as events_raw, arrayEnumerate(events_raw) ) as events_deduped FROM ordered_events GROUP BY session_id), events_sliced_cte AS (SELECT session_id, arraySlice(events_deduped, 1, {p12:UInt64}) as events_sliced FROM events_deduped_cte HAVING session_id IN (SELECT session_id FROM start_event_sessions) AND session_id IN (SELECT session_id FROM end_event_sessions)) SELECT session_id, if( arrayFirstIndex(x -> x > 1, arrayEnumerateUniq(events_sliced)) = 0, events_sliced, arraySlice( events_sliced, 1, arrayFirstIndex(x -> x > 1, arrayEnumerateUniq(events_sliced)) - 1 ) ) as events, events[1] as entry_event FROM events_sliced_cte HAVING length(events) >= {p13:UInt64}) SELECT session_id, events, arrayFirstIndex(x -> x = {p14:String}, events) as start_index, arrayFirstIndex(x -> x = {p15:String}, events) as end_index FROM session_paths HAVING start_index > 0 AND end_index > 0 AND start_index < end_index) SELECT session_id, arraySlice(events, start_index, end_index - start_index + 1) as events, events[start_index] as entry_event FROM between_sessions) SELECT entry_event, count() as count FROM session_paths GROUP BY entry_event ORDER BY count DESC LIMIT {p16:UInt64}+-- V2 params: {"p1":"secure-privacy","p2":"session_start","p3":"2026-07-06 00:00:00","p4":"2026-07-12 23:59:59","p5":"secure-privacy","p6":"session_end","p7":"2026-07-06 00:00:00","p8":"2026-07-12 23:59:59","p9":"secure-privacy","p10":"2026-07-06 00:00:00","p11":"2026-07-12 23:59:59","p12":4,"p13":2,"p14":"session_start","p15":"session_end","p16":3}+```++### sankey transitions — between, session_start -> session_end, 4 steps (golden insights-user-flow-between)++**IDENTICAL** — rows V1/V2 = 9/9; rows_read V1/V2 = 458695/458695; wall V1/V2 = 205 ms / 167 ms; clickhouse_settings: {"session_timezone":"UTC"} (both).++```sql+-- V1+WITH between_sessions AS (WITH session_paths AS (WITH start_event_sessions AS (SELECT session_id FROM events WHERE project_id = 'secure-privacy' AND name = 'session_start' AND created_at BETWEEN toDateTime('2026-07-06 00:00:00') AND toDateTime('2026-07-12 23:59:59') GROUP BY session_id), end_event_sessions AS (SELECT session_id FROM events WHERE project_id = 'secure-privacy' AND name = 'session_end' AND created_at BETWEEN toDateTime('2026-07-06 00:00:00') AND toDateTime('2026-07-12 23:59:59') GROUP BY session_id), events_deduped_cte AS (WITH ordered_events AS (SELECT session_id, name as event_name, created_at FROM events WHERE project_id = 'secure-privacy' AND created_at BETWEEN toDateTime('2026-07-06 00:00:00') AND toDateTime('2026-07-12 23:59:59') ORDER BY session_id ASC, created_at ASC) SELECT session_id, arrayFilter( (x, i) -> i = 1 OR x != events_raw[i - 1], groupArray(event_name) as events_raw, arrayEnumerate(events_raw) ) as events_deduped FROM ordered_events GROUP BY session_id), events_sliced_cte AS (SELECT session_id, arraySlice(events_deduped, 1, 4) as events_sliced FROM events_deduped_cte HAVING session_id IN (SELECT session_id FROM start_event_sessions) AND session_id IN (SELECT session_id FROM end_event_sessions)) SELECT session_id, if( arrayFirstIndex(x -> x > 1, arrayEnumerateUniq(events_sliced)) = 0, events_sliced, arraySlice( events_sliced, 1, arrayFirstIndex(x -> x > 1, arrayEnumerateUniq(events_sliced)) - 1 ) ) as events, events[1] as entry_event FROM events_sliced_cte HAVING length(events) >= 2) SELECT session_id, events, arrayFirstIndex(x -> x = 'session_start', events) as start_index, arrayFirstIndex(x -> x = 'session_end', events) as end_index FROM session_paths HAVING start_index > 0 AND end_index > 0 AND start_index < end_index), session_paths AS (SELECT session_id, arraySlice(events, start_index, end_index - start_index + 1) as events FROM between_sessions HAVING events[1] IN ('session_start')) SELECT pair.1 as source, pair.2 as target, pair.3 as step, count() as value FROM ((SELECT arrayJoin(arrayMap(i -> (events[i], events[i + 1], i), range(1, length(events)))) as pair FROM session_paths WHERE length(events) >= 2)) GROUP BY source, target, step ORDER BY step ASC, value DESC+-- V2+WITH between_sessions AS (WITH session_paths AS (WITH start_event_sessions AS (SELECT session_id FROM events WHERE project_id = {p1:String} AND name = {p2:String} AND created_at BETWEEN toDateTime({p3:String}) AND toDateTime({p4:String}) GROUP BY session_id), end_event_sessions AS (SELECT session_id FROM events WHERE project_id = {p5:String} AND name = {p6:String} AND created_at BETWEEN toDateTime({p7:String}) AND toDateTime({p8:String}) GROUP BY session_id), events_deduped_cte AS (WITH ordered_events AS (SELECT session_id, name as event_name, created_at FROM events WHERE project_id = {p9:String} AND created_at BETWEEN toDateTime({p10:String}) AND toDateTime({p11:String}) ORDER BY session_id ASC, created_at ASC) SELECT session_id, arrayFilter( (x, i) -> i = 1 OR x != events_raw[i - 1], groupArray(event_name) as events_raw, arrayEnumerate(events_raw) ) as events_deduped FROM ordered_events GROUP BY session_id), events_sliced_cte AS (SELECT session_id, arraySlice(events_deduped, 1, {p12:UInt64}) as events_sliced FROM events_deduped_cte HAVING session_id IN (SELECT session_id FROM start_event_sessions) AND session_id IN (SELECT session_id FROM end_event_sessions)) SELECT session_id, if( arrayFirstIndex(x -> x > 1, arrayEnumerateUniq(events_sliced)) = 0, events_sliced, arraySlice( events_sliced, 1, arrayFirstIndex(x -> x > 1, arrayEnumerateUniq(events_sliced)) - 1 ) ) as events, events[1] as entry_event FROM events_sliced_cte HAVING length(events) >= {p13:UInt64}) SELECT session_id, events, arrayFirstIndex(x -> x = {p14:String}, events) as start_index, arrayFirstIndex(x -> x = {p15:String}, events) as end_index FROM session_paths HAVING start_index > 0 AND end_index > 0 AND start_index < end_index), session_paths AS (SELECT session_id, arraySlice(events, start_index, end_index - start_index + 1) as events FROM between_sessions HAVING events[1] IN {p16:Array(String)}) SELECT pair.1 as source, pair.2 as target, pair.3 as step, count() as value FROM ((SELECT arrayJoin(arrayMap(i -> (events[i], events[i + 1], i), range(1, length(events)))) as pair FROM session_paths WHERE length(events) >= 2)) GROUP BY source, target, step ORDER BY step ASC, value DESC+-- V2 params: {"p1":"secure-privacy","p2":"session_start","p3":"2026-07-06 00:00:00","p4":"2026-07-12 23:59:59","p5":"secure-privacy","p6":"session_end","p7":"2026-07-06 00:00:00","p8":"2026-07-12 23:59:59","p9":"secure-privacy","p10":"2026-07-06 00:00:00","p11":"2026-07-12 23:59:59","p12":4,"p13":2,"p14":"session_start","p15":"session_end","p16":["session_start"]}+```++### sankey topEntries — after with exclude (golden insights-user-flow-exclude)++**IDENTICAL** — rows V1/V2 = 1/1; rows_read V1/V2 = 352206/352206; wall V1/V2 = 108 ms / 81 ms; clickhouse_settings: {"session_timezone":"UTC"} (both).++```sql+-- V1+WITH session_paths AS (WITH start_event_sessions AS (SELECT session_id FROM events WHERE project_id = 'secure-privacy' AND name = 'screen_view' AND created_at BETWEEN toDateTime('2026-07-06 00:00:00') AND toDateTime('2026-07-12 23:59:59') GROUP BY session_id), events_deduped_cte AS (WITH ordered_events AS (SELECT session_id, name as event_name, created_at FROM events WHERE project_id = 'secure-privacy' AND created_at BETWEEN toDateTime('2026-07-06 00:00:00') AND toDateTime('2026-07-12 23:59:59') AND name NOT IN ('session_end') ORDER BY session_id ASC, created_at ASC) SELECT session_id, arrayFilter( (x, i) -> i = 1 OR x != events_raw[i - 1], groupArray(event_name) as events_raw, arrayEnumerate(events_raw) ) as events_deduped FROM ordered_events GROUP BY session_id), events_sliced_cte AS (SELECT session_id, arraySlice(events_deduped, arrayFirstIndex(x -> x = 'screen_view', events_deduped), 3) as events_sliced FROM events_deduped_cte HAVING session_id IN (SELECT session_id FROM start_event_sessions)) SELECT session_id, if( arrayFirstIndex(x -> x > 1, arrayEnumerateUniq(events_sliced)) = 0, events_sliced, arraySlice( events_sliced, 1, arrayFirstIndex(x -> x > 1, arrayEnumerateUniq(events_sliced)) - 1 ) ) as events, events[1] as entry_event FROM events_sliced_cte HAVING length(events) >= 2) SELECT entry_event, count() as count FROM session_paths GROUP BY entry_event ORDER BY count DESC LIMIT 3+-- V2+WITH session_paths AS (WITH start_event_sessions AS (SELECT session_id FROM events WHERE project_id = {p1:String} AND name = {p2:String} AND created_at BETWEEN toDateTime({p3:String}) AND toDateTime({p4:String}) GROUP BY session_id), events_deduped_cte AS (WITH ordered_events AS (SELECT session_id, name as event_name, created_at FROM events WHERE project_id = {p5:String} AND created_at BETWEEN toDateTime({p6:String}) AND toDateTime({p7:String}) AND name NOT IN {p8:Array(String)} ORDER BY session_id ASC, created_at ASC) SELECT session_id, arrayFilter( (x, i) -> i = 1 OR x != events_raw[i - 1], groupArray(event_name) as events_raw, arrayEnumerate(events_raw) ) as events_deduped FROM ordered_events GROUP BY session_id), events_sliced_cte AS (SELECT session_id, arraySlice(events_deduped, arrayFirstIndex(x -> x = {p9:String}, events_deduped), {p10:UInt64}) as events_sliced FROM events_deduped_cte HAVING session_id IN (SELECT session_id FROM start_event_sessions)) SELECT session_id, if( arrayFirstIndex(x -> x > 1, arrayEnumerateUniq(events_sliced)) = 0, events_sliced, arraySlice( events_sliced, 1, arrayFirstIndex(x -> x > 1, arrayEnumerateUniq(events_sliced)) - 1 ) ) as events, events[1] as entry_event FROM events_sliced_cte HAVING length(events) >= {p11:UInt64}) SELECT entry_event, count() as count FROM session_paths GROUP BY entry_event ORDER BY count DESC LIMIT {p12:UInt64}+-- V2 params: {"p1":"secure-privacy","p2":"screen_view","p3":"2026-07-06 00:00:00","p4":"2026-07-12 23:59:59","p5":"secure-privacy","p6":"2026-07-06 00:00:00","p7":"2026-07-12 23:59:59","p8":["session_end"],"p9":"screen_view","p10":3,"p11":2,"p12":3}+```++### sankey transitions — after with exclude (golden insights-user-flow-exclude)++**IDENTICAL** — rows V1/V2 = 15/15; rows_read V1/V2 = 352206/352206; wall V1/V2 = 123 ms / 113 ms; clickhouse_settings: {"session_timezone":"UTC"} (both).++```sql+-- V1+WITH session_paths_base AS (WITH start_event_sessions AS (SELECT session_id FROM events WHERE project_id = 'secure-privacy' AND name = 'screen_view' AND created_at BETWEEN toDateTime('2026-07-06 00:00:00') AND toDateTime('2026-07-12 23:59:59') GROUP BY session_id), events_deduped_cte AS (WITH ordered_events AS (SELECT session_id, name as event_name, created_at FROM events WHERE project_id = 'secure-privacy' AND created_at BETWEEN toDateTime('2026-07-06 00:00:00') AND toDateTime('2026-07-12 23:59:59') AND name NOT IN ('session_end') ORDER BY session_id ASC, created_at ASC) SELECT session_id, arrayFilter( (x, i) -> i = 1 OR x != events_raw[i - 1], groupArray(event_name) as events_raw, arrayEnumerate(events_raw) ) as events_deduped FROM ordered_events GROUP BY session_id), events_sliced_cte AS (SELECT session_id, arraySlice(events_deduped, arrayFirstIndex(x -> x = 'screen_view', events_deduped), 3) as events_sliced FROM events_deduped_cte HAVING session_id IN (SELECT session_id FROM start_event_sessions)) SELECT session_id, if( arrayFirstIndex(x -> x > 1, arrayEnumerateUniq(events_sliced)) = 0, events_sliced, arraySlice( events_sliced, 1, arrayFirstIndex(x -> x > 1, arrayEnumerateUniq(events_sliced)) - 1 ) ) as events, events[1] as entry_event FROM events_sliced_cte HAVING length(events) >= 2), session_paths AS (SELECT session_id, events FROM session_paths_base HAVING events[1] IN ('screen_view')) SELECT pair.1 as source, pair.2 as target, pair.3 as step, count() as value FROM ((SELECT arrayJoin(arrayMap(i -> (events[i], events[i + 1], i), range(1, length(events)))) as pair FROM session_paths WHERE length(events) >= 2)) GROUP BY source, target, step ORDER BY step ASC, value DESC+-- V2+WITH session_paths_base AS (WITH start_event_sessions AS (SELECT session_id FROM events WHERE project_id = {p1:String} AND name = {p2:String} AND created_at BETWEEN toDateTime({p3:String}) AND toDateTime({p4:String}) GROUP BY session_id), events_deduped_cte AS (WITH ordered_events AS (SELECT session_id, name as event_name, created_at FROM events WHERE project_id = {p5:String} AND created_at BETWEEN toDateTime({p6:String}) AND toDateTime({p7:String}) AND name NOT IN {p8:Array(String)} ORDER BY session_id ASC, created_at ASC) SELECT session_id, arrayFilter( (x, i) -> i = 1 OR x != events_raw[i - 1], groupArray(event_name) as events_raw, arrayEnumerate(events_raw) ) as events_deduped FROM ordered_events GROUP BY session_id), events_sliced_cte AS (SELECT session_id, arraySlice(events_deduped, arrayFirstIndex(x -> x = {p9:String}, events_deduped), {p10:UInt64}) as events_sliced FROM events_deduped_cte HAVING session_id IN (SELECT session_id FROM start_event_sessions)) SELECT session_id, if( arrayFirstIndex(x -> x > 1, arrayEnumerateUniq(events_sliced)) = 0, events_sliced, arraySlice( events_sliced, 1, arrayFirstIndex(x -> x > 1, arrayEnumerateUniq(events_sliced)) - 1 ) ) as events, events[1] as entry_event FROM events_sliced_cte HAVING length(events) >= {p11:UInt64}), session_paths AS (SELECT session_id, events FROM session_paths_base HAVING events[1] IN {p12:Array(String)}) SELECT pair.1 as source, pair.2 as target, pair.3 as step, count() as value FROM ((SELECT arrayJoin(arrayMap(i -> (events[i], events[i + 1], i), range(1, length(events)))) as pair FROM session_paths WHERE length(events) >= 2)) GROUP BY source, target, step ORDER BY step ASC, value DESC+-- V2 params: {"p1":"secure-privacy","p2":"screen_view","p3":"2026-07-06 00:00:00","p4":"2026-07-12 23:59:59","p5":"secure-privacy","p6":"2026-07-06 00:00:00","p7":"2026-07-12 23:59:59","p8":["session_end"],"p9":"screen_view","p10":3,"p11":2,"p12":["screen_view"]}+```++### sankey topEntries — after with include list++**IDENTICAL** — rows V1/V2 = 1/1; rows_read V1/V2 = 352206/352206; wall V1/V2 = 136 ms / 96 ms; clickhouse_settings: {"session_timezone":"UTC"} (both).++```sql+-- V1+WITH session_paths AS (WITH start_event_sessions AS (SELECT session_id FROM events WHERE project_id = 'secure-privacy' AND name = 'screen_view' AND created_at BETWEEN toDateTime('2026-07-06 00:00:00') AND toDateTime('2026-07-12 23:59:59') GROUP BY session_id), events_deduped_cte AS (WITH ordered_events AS (SELECT session_id, name as event_name, created_at FROM events WHERE project_id = 'secure-privacy' AND created_at BETWEEN toDateTime('2026-07-06 00:00:00') AND toDateTime('2026-07-12 23:59:59') AND name IN ('session_start', 'session_end', 'screen_view') ORDER BY session_id ASC, created_at ASC) SELECT session_id, arrayFilter( (x, i) -> i = 1 OR x != events_raw[i - 1], groupArray(event_name) as events_raw, arrayEnumerate(events_raw) ) as events_deduped FROM ordered_events GROUP BY session_id), events_sliced_cte AS (SELECT session_id, arraySlice(events_deduped, arrayFirstIndex(x -> x = 'screen_view', events_deduped), 3) as events_sliced FROM events_deduped_cte HAVING session_id IN (SELECT session_id FROM start_event_sessions)) SELECT session_id, if( arrayFirstIndex(x -> x > 1, arrayEnumerateUniq(events_sliced)) = 0, events_sliced, arraySlice( events_sliced, 1, arrayFirstIndex(x -> x > 1, arrayEnumerateUniq(events_sliced)) - 1 ) ) as events, events[1] as entry_event FROM events_sliced_cte HAVING length(events) >= 2) SELECT entry_event, count() as count FROM session_paths GROUP BY entry_event ORDER BY count DESC LIMIT 3+-- V2+WITH session_paths AS (WITH start_event_sessions AS (SELECT session_id FROM events WHERE project_id = {p1:String} AND name = {p2:String} AND created_at BETWEEN toDateTime({p3:String}) AND toDateTime({p4:String}) GROUP BY session_id), events_deduped_cte AS (WITH ordered_events AS (SELECT session_id, name as event_name, created_at FROM events WHERE project_id = {p5:String} AND created_at BETWEEN toDateTime({p6:String}) AND toDateTime({p7:String}) AND name IN {p8:Array(String)} ORDER BY session_id ASC, created_at ASC) SELECT session_id, arrayFilter( (x, i) -> i = 1 OR x != events_raw[i - 1], groupArray(event_name) as events_raw, arrayEnumerate(events_raw) ) as events_deduped FROM ordered_events GROUP BY session_id), events_sliced_cte AS (SELECT session_id, arraySlice(events_deduped, arrayFirstIndex(x -> x = {p9:String}, events_deduped), {p10:UInt64}) as events_sliced FROM events_deduped_cte HAVING session_id IN (SELECT session_id FROM start_event_sessions)) SELECT session_id, if( arrayFirstIndex(x -> x > 1, arrayEnumerateUniq(events_sliced)) = 0, events_sliced, arraySlice( events_sliced, 1, arrayFirstIndex(x -> x > 1, arrayEnumerateUniq(events_sliced)) - 1 ) ) as events, events[1] as entry_event FROM events_sliced_cte HAVING length(events) >= {p11:UInt64}) SELECT entry_event, count() as count FROM session_paths GROUP BY entry_event ORDER BY count DESC LIMIT {p12:UInt64}+-- V2 params: {"p1":"secure-privacy","p2":"screen_view","p3":"2026-07-06 00:00:00","p4":"2026-07-12 23:59:59","p5":"secure-privacy","p6":"2026-07-06 00:00:00","p7":"2026-07-12 23:59:59","p8":["session_start","session_end","screen_view"],"p9":"screen_view","p10":3,"p11":2,"p12":3}+```++### sankey transitions — after with include list++**IDENTICAL** — rows V1/V2 = 2/2; rows_read V1/V2 = 352206/352206; wall V1/V2 = 118 ms / 102 ms; clickhouse_settings: {"session_timezone":"UTC"} (both).++```sql+-- V1+WITH session_paths_base AS (WITH start_event_sessions AS (SELECT session_id FROM events WHERE project_id = 'secure-privacy' AND name = 'screen_view' AND created_at BETWEEN toDateTime('2026-07-06 00:00:00') AND toDateTime('2026-07-12 23:59:59') GROUP BY session_id), events_deduped_cte AS (WITH ordered_events AS (SELECT session_id, name as event_name, created_at FROM events WHERE project_id = 'secure-privacy' AND created_at BETWEEN toDateTime('2026-07-06 00:00:00') AND toDateTime('2026-07-12 23:59:59') AND name IN ('session_start', 'session_end', 'screen_view') ORDER BY session_id ASC, created_at ASC) SELECT session_id, arrayFilter( (x, i) -> i = 1 OR x != events_raw[i - 1], groupArray(event_name) as events_raw, arrayEnumerate(events_raw) ) as events_deduped FROM ordered_events GROUP BY session_id), events_sliced_cte AS (SELECT session_id, arraySlice(events_deduped, arrayFirstIndex(x -> x = 'screen_view', events_deduped), 3) as events_sliced FROM events_deduped_cte HAVING session_id IN (SELECT session_id FROM start_event_sessions)) SELECT session_id, if( arrayFirstIndex(x -> x > 1, arrayEnumerateUniq(events_sliced)) = 0, events_sliced, arraySlice( events_sliced, 1, arrayFirstIndex(x -> x > 1, arrayEnumerateUniq(events_sliced)) - 1 ) ) as events, events[1] as entry_event FROM events_sliced_cte HAVING length(events) >= 2), session_paths AS (SELECT session_id, events FROM session_paths_base HAVING events[1] IN ('screen_view')) SELECT pair.1 as source, pair.2 as target, pair.3 as step, count() as value FROM ((SELECT arrayJoin(arrayMap(i -> (events[i], events[i + 1], i), range(1, length(events)))) as pair FROM session_paths WHERE length(events) >= 2)) GROUP BY source, target, step ORDER BY step ASC, value DESC+-- V2+WITH session_paths_base AS (WITH start_event_sessions AS (SELECT session_id FROM events WHERE project_id = {p1:String} AND name = {p2:String} AND created_at BETWEEN toDateTime({p3:String}) AND toDateTime({p4:String}) GROUP BY session_id), events_deduped_cte AS (WITH ordered_events AS (SELECT session_id, name as event_name, created_at FROM events WHERE project_id = {p5:String} AND created_at BETWEEN toDateTime({p6:String}) AND toDateTime({p7:String}) AND name IN {p8:Array(String)} ORDER BY session_id ASC, created_at ASC) SELECT session_id, arrayFilter( (x, i) -> i = 1 OR x != events_raw[i - 1], groupArray(event_name) as events_raw, arrayEnumerate(events_raw) ) as events_deduped FROM ordered_events GROUP BY session_id), events_sliced_cte AS (SELECT session_id, arraySlice(events_deduped, arrayFirstIndex(x -> x = {p9:String}, events_deduped), {p10:UInt64}) as events_sliced FROM events_deduped_cte HAVING session_id IN (SELECT session_id FROM start_event_sessions)) SELECT session_id, if( arrayFirstIndex(x -> x > 1, arrayEnumerateUniq(events_sliced)) = 0, events_sliced, arraySlice( events_sliced, 1, arrayFirstIndex(x -> x > 1, arrayEnumerateUniq(events_sliced)) - 1 ) ) as events, events[1] as entry_event FROM events_sliced_cte HAVING length(events) >= {p11:UInt64}), session_paths AS (SELECT session_id, events FROM session_paths_base HAVING events[1] IN {p12:Array(String)}) SELECT pair.1 as source, pair.2 as target, pair.3 as step, count() as value FROM ((SELECT arrayJoin(arrayMap(i -> (events[i], events[i + 1], i), range(1, length(events)))) as pair FROM session_paths WHERE length(events) >= 2)) GROUP BY source, target, step ORDER BY step ASC, value DESC+-- V2 params: {"p1":"secure-privacy","p2":"screen_view","p3":"2026-07-06 00:00:00","p4":"2026-07-12 23:59:59","p5":"secure-privacy","p6":"2026-07-06 00:00:00","p7":"2026-07-12 23:59:59","p8":["session_start","session_end","screen_view"],"p9":"screen_view","p10":3,"p11":2,"p12":["screen_view"]}+```++### sankey topEntries — after with a start-event filter (country is US)++**IDENTICAL** — rows V1/V2 = 1/1; rows_read V1/V2 = 344016/344016; wall V1/V2 = 89 ms / 73 ms; clickhouse_settings: {"session_timezone":"UTC"} (both).++```sql+-- V1+WITH session_paths AS (WITH start_event_sessions AS (SELECT session_id FROM events WHERE project_id = 'secure-privacy' AND name = 'screen_view' AND created_at BETWEEN toDateTime('2026-07-06 00:00:00') AND toDateTime('2026-07-12 23:59:59') AND country = 'US' GROUP BY session_id), events_deduped_cte AS (WITH ordered_events AS (SELECT session_id, name as event_name, created_at FROM events WHERE project_id = 'secure-privacy' AND created_at BETWEEN toDateTime('2026-07-06 00:00:00') AND toDateTime('2026-07-12 23:59:59') ORDER BY session_id ASC, created_at ASC) SELECT session_id, arrayFilter( (x, i) -> i = 1 OR x != events_raw[i - 1], groupArray(event_name) as events_raw, arrayEnumerate(events_raw) ) as events_deduped FROM ordered_events GROUP BY session_id), events_sliced_cte AS (SELECT session_id, arraySlice(events_deduped, arrayFirstIndex(x -> x = 'screen_view', events_deduped), 3) as events_sliced FROM events_deduped_cte HAVING session_id IN (SELECT session_id FROM start_event_sessions)) SELECT session_id, if( arrayFirstIndex(x -> x > 1, arrayEnumerateUniq(events_sliced)) = 0, events_sliced, arraySlice( events_sliced, 1, arrayFirstIndex(x -> x > 1, arrayEnumerateUniq(events_sliced)) - 1 ) ) as events, events[1] as entry_event FROM events_sliced_cte HAVING length(events) >= 2) SELECT entry_event, count() as count FROM session_paths GROUP BY entry_event ORDER BY count DESC LIMIT 3+-- V2+WITH session_paths AS (WITH start_event_sessions AS (SELECT session_id FROM events WHERE project_id = {p1:String} AND name = {p2:String} AND created_at BETWEEN toDateTime({p3:String}) AND toDateTime({p4:String}) AND country = 'US' GROUP BY session_id), events_deduped_cte AS (WITH ordered_events AS (SELECT session_id, name as event_name, created_at FROM events WHERE project_id = {p5:String} AND created_at BETWEEN toDateTime({p6:String}) AND toDateTime({p7:String}) ORDER BY session_id ASC, created_at ASC) SELECT session_id, arrayFilter( (x, i) -> i = 1 OR x != events_raw[i - 1], groupArray(event_name) as events_raw, arrayEnumerate(events_raw) ) as events_deduped FROM ordered_events GROUP BY session_id), events_sliced_cte AS (SELECT session_id, arraySlice(events_deduped, arrayFirstIndex(x -> x = {p8:String}, events_deduped), {p9:UInt64}) as events_sliced FROM events_deduped_cte HAVING session_id IN (SELECT session_id FROM start_event_sessions)) SELECT session_id, if( arrayFirstIndex(x -> x > 1, arrayEnumerateUniq(events_sliced)) = 0, events_sliced, arraySlice( events_sliced, 1, arrayFirstIndex(x -> x > 1, arrayEnumerateUniq(events_sliced)) - 1 ) ) as events, events[1] as entry_event FROM events_sliced_cte HAVING length(events) >= {p10:UInt64}) SELECT entry_event, count() as count FROM session_paths GROUP BY entry_event ORDER BY count DESC LIMIT {p11:UInt64}+-- V2 params: {"p1":"secure-privacy","p2":"screen_view","p3":"2026-07-06 00:00:00","p4":"2026-07-12 23:59:59","p5":"secure-privacy","p6":"2026-07-06 00:00:00","p7":"2026-07-12 23:59:59","p8":"screen_view","p9":3,"p10":2,"p11":3}+```++### sankey transitions — after with a start-event filter (country is US)++**IDENTICAL** — rows V1/V2 = 9/9; rows_read V1/V2 = 344016/344016; wall V1/V2 = 101 ms / 79 ms; clickhouse_settings: {"session_timezone":"UTC"} (both).++```sql+-- V1+WITH session_paths_base AS (WITH start_event_sessions AS (SELECT session_id FROM events WHERE project_id = 'secure-privacy' AND name = 'screen_view' AND created_at BETWEEN toDateTime('2026-07-06 00:00:00') AND toDateTime('2026-07-12 23:59:59') AND country = 'US' GROUP BY session_id), events_deduped_cte AS (WITH ordered_events AS (SELECT session_id, name as event_name, created_at FROM events WHERE project_id = 'secure-privacy' AND created_at BETWEEN toDateTime('2026-07-06 00:00:00') AND toDateTime('2026-07-12 23:59:59') ORDER BY session_id ASC, created_at ASC) SELECT session_id, arrayFilter( (x, i) -> i = 1 OR x != events_raw[i - 1], groupArray(event_name) as events_raw, arrayEnumerate(events_raw) ) as events_deduped FROM ordered_events GROUP BY session_id), events_sliced_cte AS (SELECT session_id, arraySlice(events_deduped, arrayFirstIndex(x -> x = 'screen_view', events_deduped), 3) as events_sliced FROM events_deduped_cte HAVING session_id IN (SELECT session_id FROM start_event_sessions)) SELECT session_id, if( arrayFirstIndex(x -> x > 1, arrayEnumerateUniq(events_sliced)) = 0, events_sliced, arraySlice( events_sliced, 1, arrayFirstIndex(x -> x > 1, arrayEnumerateUniq(events_sliced)) - 1 ) ) as events, events[1] as entry_event FROM events_sliced_cte HAVING length(events) >= 2), session_paths AS (SELECT session_id, events FROM session_paths_base HAVING events[1] IN ('screen_view')) SELECT pair.1 as source, pair.2 as target, pair.3 as step, count() as value FROM ((SELECT arrayJoin(arrayMap(i -> (events[i], events[i + 1], i), range(1, length(events)))) as pair FROM session_paths WHERE length(events) >= 2)) GROUP BY source, target, step ORDER BY step ASC, value DESC+-- V2+WITH session_paths_base AS (WITH start_event_sessions AS (SELECT session_id FROM events WHERE project_id = {p1:String} AND name = {p2:String} AND created_at BETWEEN toDateTime({p3:String}) AND toDateTime({p4:String}) AND country = 'US' GROUP BY session_id), events_deduped_cte AS (WITH ordered_events AS (SELECT session_id, name as event_name, created_at FROM events WHERE project_id = {p5:String} AND created_at BETWEEN toDateTime({p6:String}) AND toDateTime({p7:String}) ORDER BY session_id ASC, created_at ASC) SELECT session_id, arrayFilter( (x, i) -> i = 1 OR x != events_raw[i - 1], groupArray(event_name) as events_raw, arrayEnumerate(events_raw) ) as events_deduped FROM ordered_events GROUP BY session_id), events_sliced_cte AS (SELECT session_id, arraySlice(events_deduped, arrayFirstIndex(x -> x = {p8:String}, events_deduped), {p9:UInt64}) as events_sliced FROM events_deduped_cte HAVING session_id IN (SELECT session_id FROM start_event_sessions)) SELECT session_id, if( arrayFirstIndex(x -> x > 1, arrayEnumerateUniq(events_sliced)) = 0, events_sliced, arraySlice( events_sliced, 1, arrayFirstIndex(x -> x > 1, arrayEnumerateUniq(events_sliced)) - 1 ) ) as events, events[1] as entry_event FROM events_sliced_cte HAVING length(events) >= {p10:UInt64}), session_paths AS (SELECT session_id, events FROM session_paths_base HAVING events[1] IN {p11:Array(String)}) SELECT pair.1 as source, pair.2 as target, pair.3 as step, count() as value FROM ((SELECT arrayJoin(arrayMap(i -> (events[i], events[i + 1], i), range(1, length(events)))) as pair FROM session_paths WHERE length(events) >= 2)) GROUP BY source, target, step ORDER BY step ASC, value DESC+-- V2 params: {"p1":"secure-privacy","p2":"screen_view","p3":"2026-07-06 00:00:00","p4":"2026-07-12 23:59:59","p5":"secure-privacy","p6":"2026-07-06 00:00:00","p7":"2026-07-12 23:59:59","p8":"screen_view","p9":3,"p10":2,"p11":["screen_view"]}+```++### sankey topEntries — after, 10 steps (max)++**IDENTICAL** — rows V1/V2 = 1/1; rows_read V1/V2 = 352206/352206; wall V1/V2 = 122 ms / 107 ms; clickhouse_settings: {"session_timezone":"UTC"} (both).++```sql+-- V1+WITH session_paths AS (WITH start_event_sessions AS (SELECT session_id FROM events WHERE project_id = 'secure-privacy' AND name = 'session_start' AND created_at BETWEEN toDateTime('2026-07-06 00:00:00') AND toDateTime('2026-07-12 23:59:59') GROUP BY session_id), events_deduped_cte AS (WITH ordered_events AS (SELECT session_id, name as event_name, created_at FROM events WHERE project_id = 'secure-privacy' AND created_at BETWEEN toDateTime('2026-07-06 00:00:00') AND toDateTime('2026-07-12 23:59:59') ORDER BY session_id ASC, created_at ASC) SELECT session_id, arrayFilter( (x, i) -> i = 1 OR x != events_raw[i - 1], groupArray(event_name) as events_raw, arrayEnumerate(events_raw) ) as events_deduped FROM ordered_events GROUP BY session_id), events_sliced_cte AS (SELECT session_id, arraySlice(events_deduped, arrayFirstIndex(x -> x = 'session_start', events_deduped), 10) as events_sliced FROM events_deduped_cte HAVING session_id IN (SELECT session_id FROM start_event_sessions)) SELECT session_id, if( arrayFirstIndex(x -> x > 1, arrayEnumerateUniq(events_sliced)) = 0, events_sliced, arraySlice( events_sliced, 1, arrayFirstIndex(x -> x > 1, arrayEnumerateUniq(events_sliced)) - 1 ) ) as events, events[1] as entry_event FROM events_sliced_cte HAVING length(events) >= 2) SELECT entry_event, count() as count FROM session_paths GROUP BY entry_event ORDER BY count DESC LIMIT 3+-- V2+WITH session_paths AS (WITH start_event_sessions AS (SELECT session_id FROM events WHERE project_id = {p1:String} AND name = {p2:String} AND created_at BETWEEN toDateTime({p3:String}) AND toDateTime({p4:String}) GROUP BY session_id), events_deduped_cte AS (WITH ordered_events AS (SELECT session_id, name as event_name, created_at FROM events WHERE project_id = {p5:String} AND created_at BETWEEN toDateTime({p6:String}) AND toDateTime({p7:String}) ORDER BY session_id ASC, created_at ASC) SELECT session_id, arrayFilter( (x, i) -> i = 1 OR x != events_raw[i - 1], groupArray(event_name) as events_raw, arrayEnumerate(events_raw) ) as events_deduped FROM ordered_events GROUP BY session_id), events_sliced_cte AS (SELECT session_id, arraySlice(events_deduped, arrayFirstIndex(x -> x = {p8:String}, events_deduped), {p9:UInt64}) as events_sliced FROM events_deduped_cte HAVING session_id IN (SELECT session_id FROM start_event_sessions)) SELECT session_id, if( arrayFirstIndex(x -> x > 1, arrayEnumerateUniq(events_sliced)) = 0, events_sliced, arraySlice( events_sliced, 1, arrayFirstIndex(x -> x > 1, arrayEnumerateUniq(events_sliced)) - 1 ) ) as events, events[1] as entry_event FROM events_sliced_cte HAVING length(events) >= {p10:UInt64}) SELECT entry_event, count() as count FROM session_paths GROUP BY entry_event ORDER BY count DESC LIMIT {p11:UInt64}+-- V2 params: {"p1":"secure-privacy","p2":"session_start","p3":"2026-07-06 00:00:00","p4":"2026-07-12 23:59:59","p5":"secure-privacy","p6":"2026-07-06 00:00:00","p7":"2026-07-12 23:59:59","p8":"session_start","p9":10,"p10":2,"p11":3}+```++### sankey transitions — after, 10 steps (max)++**IDENTICAL** — rows V1/V2 = 21/21; rows_read V1/V2 = 352206/352206; wall V1/V2 = 106 ms / 116 ms; clickhouse_settings: {"session_timezone":"UTC"} (both).++```sql+-- V1+WITH session_paths_base AS (WITH start_event_sessions AS (SELECT session_id FROM events WHERE project_id = 'secure-privacy' AND name = 'session_start' AND created_at BETWEEN toDateTime('2026-07-06 00:00:00') AND toDateTime('2026-07-12 23:59:59') GROUP BY session_id), events_deduped_cte AS (WITH ordered_events AS (SELECT session_id, name as event_name, created_at FROM events WHERE project_id = 'secure-privacy' AND created_at BETWEEN toDateTime('2026-07-06 00:00:00') AND toDateTime('2026-07-12 23:59:59') ORDER BY session_id ASC, created_at ASC) SELECT session_id, arrayFilter( (x, i) -> i = 1 OR x != events_raw[i - 1], groupArray(event_name) as events_raw, arrayEnumerate(events_raw) ) as events_deduped FROM ordered_events GROUP BY session_id), events_sliced_cte AS (SELECT session_id, arraySlice(events_deduped, arrayFirstIndex(x -> x = 'session_start', events_deduped), 10) as events_sliced FROM events_deduped_cte HAVING session_id IN (SELECT session_id FROM start_event_sessions)) SELECT session_id, if( arrayFirstIndex(x -> x > 1, arrayEnumerateUniq(events_sliced)) = 0, events_sliced, arraySlice( events_sliced, 1, arrayFirstIndex(x -> x > 1, arrayEnumerateUniq(events_sliced)) - 1 ) ) as events, events[1] as entry_event FROM events_sliced_cte HAVING length(events) >= 2), session_paths AS (SELECT session_id, events FROM session_paths_base HAVING events[1] IN ('session_start')) SELECT pair.1 as source, pair.2 as target, pair.3 as step, count() as value FROM ((SELECT arrayJoin(arrayMap(i -> (events[i], events[i + 1], i), range(1, length(events)))) as pair FROM session_paths WHERE length(events) >= 2)) GROUP BY source, target, step ORDER BY step ASC, value DESC+-- V2+WITH session_paths_base AS (WITH start_event_sessions AS (SELECT session_id FROM events WHERE project_id = {p1:String} AND name = {p2:String} AND created_at BETWEEN toDateTime({p3:String}) AND toDateTime({p4:String}) GROUP BY session_id), events_deduped_cte AS (WITH ordered_events AS (SELECT session_id, name as event_name, created_at FROM events WHERE project_id = {p5:String} AND created_at BETWEEN toDateTime({p6:String}) AND toDateTime({p7:String}) ORDER BY session_id ASC, created_at ASC) SELECT session_id, arrayFilter( (x, i) -> i = 1 OR x != events_raw[i - 1], groupArray(event_name) as events_raw, arrayEnumerate(events_raw) ) as events_deduped FROM ordered_events GROUP BY session_id), events_sliced_cte AS (SELECT session_id, arraySlice(events_deduped, arrayFirstIndex(x -> x = {p8:String}, events_deduped), {p9:UInt64}) as events_sliced FROM events_deduped_cte HAVING session_id IN (SELECT session_id FROM start_event_sessions)) SELECT session_id, if( arrayFirstIndex(x -> x > 1, arrayEnumerateUniq(events_sliced)) = 0, events_sliced, arraySlice( events_sliced, 1, arrayFirstIndex(x -> x > 1, arrayEnumerateUniq(events_sliced)) - 1 ) ) as events, events[1] as entry_event FROM events_sliced_cte HAVING length(events) >= {p10:UInt64}), session_paths AS (SELECT session_id, events FROM session_paths_base HAVING events[1] IN {p11:Array(String)}) SELECT pair.1 as source, pair.2 as target, pair.3 as step, count() as value FROM ((SELECT arrayJoin(arrayMap(i -> (events[i], events[i + 1], i), range(1, length(events)))) as pair FROM session_paths WHERE length(events) >= 2)) GROUP BY source, target, step ORDER BY step ASC, value DESC+-- V2 params: {"p1":"secure-privacy","p2":"session_start","p3":"2026-07-06 00:00:00","p4":"2026-07-12 23:59:59","p5":"secure-privacy","p6":"2026-07-06 00:00:00","p7":"2026-07-12 23:59:59","p8":"session_start","p9":10,"p10":2,"p11":["session_start"]}+```++### sankey topEntries — before, 2 steps (min)++**IDENTICAL** — rows V1/V2 = 3/3; rows_read V1/V2 = 352206/352206; wall V1/V2 = 101 ms / 78 ms; clickhouse_settings: {"session_timezone":"UTC"} (both).++```sql+-- V1+WITH session_paths AS (WITH start_event_sessions AS (SELECT session_id FROM events WHERE project_id = 'secure-privacy' AND name = 'session_end' AND created_at BETWEEN toDateTime('2026-07-06 00:00:00') AND toDateTime('2026-07-12 23:59:59') GROUP BY session_id), events_deduped_cte AS (WITH ordered_events AS (SELECT session_id, name as event_name, created_at FROM events WHERE project_id = 'secure-privacy' AND created_at BETWEEN toDateTime('2026-07-06 00:00:00') AND toDateTime('2026-07-12 23:59:59') ORDER BY session_id ASC, created_at ASC) SELECT session_id, arrayFilter( (x, i) -> i = 1 OR x != events_raw[i - 1], groupArray(event_name) as events_raw, arrayEnumerate(events_raw) ) as events_deduped FROM ordered_events GROUP BY session_id), events_sliced_cte AS (SELECT session_id, arraySlice( events_deduped, greatest(1, arrayFirstIndex(x -> x = 'session_end', events_deduped) - 2 + 1), arrayFirstIndex(x -> x = 'session_end', events_deduped) - greatest(1, arrayFirstIndex(x -> x = 'session_end', events_deduped) - 2 + 1) + 1 ) as events_sliced FROM events_deduped_cte HAVING session_id IN (SELECT session_id FROM start_event_sessions)) SELECT session_id, events_sliced as events, events[1] as entry_event FROM events_sliced_cte HAVING length(events) >= 2) SELECT entry_event, count() as count FROM session_paths GROUP BY entry_event ORDER BY count DESC LIMIT 3+-- V2+WITH session_paths AS (WITH start_event_sessions AS (SELECT session_id FROM events WHERE project_id = {p1:String} AND name = {p2:String} AND created_at BETWEEN toDateTime({p3:String}) AND toDateTime({p4:String}) GROUP BY session_id), events_deduped_cte AS (WITH ordered_events AS (SELECT session_id, name as event_name, created_at FROM events WHERE project_id = {p5:String} AND created_at BETWEEN toDateTime({p6:String}) AND toDateTime({p7:String}) ORDER BY session_id ASC, created_at ASC) SELECT session_id, arrayFilter( (x, i) -> i = 1 OR x != events_raw[i - 1], groupArray(event_name) as events_raw, arrayEnumerate(events_raw) ) as events_deduped FROM ordered_events GROUP BY session_id), events_sliced_cte AS (SELECT session_id, arraySlice( events_deduped, greatest(1, arrayFirstIndex(x -> x = {p8:String}, events_deduped) - {p9:UInt64} + 1), arrayFirstIndex(x -> x = {p10:String}, events_deduped) - greatest(1, arrayFirstIndex(x -> x = {p11:String}, events_deduped) - {p12:UInt64} + 1) + 1 ) as events_sliced FROM events_deduped_cte HAVING session_id IN (SELECT session_id FROM start_event_sessions)) SELECT session_id, events_sliced as events, events[1] as entry_event FROM events_sliced_cte HAVING length(events) >= {p13:UInt64}) SELECT entry_event, count() as count FROM session_paths GROUP BY entry_event ORDER BY count DESC LIMIT {p14:UInt64}+-- V2 params: {"p1":"secure-privacy","p2":"session_end","p3":"2026-07-06 00:00:00","p4":"2026-07-12 23:59:59","p5":"secure-privacy","p6":"2026-07-06 00:00:00","p7":"2026-07-12 23:59:59","p8":"session_end","p9":2,"p10":"session_end","p11":"session_end","p12":2,"p13":2,"p14":3}+```++### sankey transitions — before, 2 steps (min)++**IDENTICAL** — rows V1/V2 = 3/3; rows_read V1/V2 = 352206/352206; wall V1/V2 = 117 ms / 132 ms; clickhouse_settings: {"session_timezone":"UTC"} (both).++```sql+-- V1+WITH session_paths_base AS (WITH start_event_sessions AS (SELECT session_id FROM events WHERE project_id = 'secure-privacy' AND name = 'session_end' AND created_at BETWEEN toDateTime('2026-07-06 00:00:00') AND toDateTime('2026-07-12 23:59:59') GROUP BY session_id), events_deduped_cte AS (WITH ordered_events AS (SELECT session_id, name as event_name, created_at FROM events WHERE project_id = 'secure-privacy' AND created_at BETWEEN toDateTime('2026-07-06 00:00:00') AND toDateTime('2026-07-12 23:59:59') ORDER BY session_id ASC, created_at ASC) SELECT session_id, arrayFilter( (x, i) -> i = 1 OR x != events_raw[i - 1], groupArray(event_name) as events_raw, arrayEnumerate(events_raw) ) as events_deduped FROM ordered_events GROUP BY session_id), events_sliced_cte AS (SELECT session_id, arraySlice( events_deduped, greatest(1, arrayFirstIndex(x -> x = 'session_end', events_deduped) - 2 + 1), arrayFirstIndex(x -> x = 'session_end', events_deduped) - greatest(1, arrayFirstIndex(x -> x = 'session_end', events_deduped) - 2 + 1) + 1 ) as events_sliced FROM events_deduped_cte HAVING session_id IN (SELECT session_id FROM start_event_sessions)) SELECT session_id, events_sliced as events, events[1] as entry_event FROM events_sliced_cte HAVING length(events) >= 2), session_paths AS (SELECT session_id, events FROM session_paths_base HAVING events[1] IN ('screen_view', 'link_out', 'web_vitals_lcp')) SELECT pair.1 as source, pair.2 as target, pair.3 as step, count() as value FROM ((SELECT arrayJoin(arrayMap(i -> (events[i], events[i + 1], i), range(1, length(events)))) as pair FROM session_paths WHERE length(events) >= 2)) GROUP BY source, target, step ORDER BY step ASC, value DESC+-- V2+WITH session_paths_base AS (WITH start_event_sessions AS (SELECT session_id FROM events WHERE project_id = {p1:String} AND name = {p2:String} AND created_at BETWEEN toDateTime({p3:String}) AND toDateTime({p4:String}) GROUP BY session_id), events_deduped_cte AS (WITH ordered_events AS (SELECT session_id, name as event_name, created_at FROM events WHERE project_id = {p5:String} AND created_at BETWEEN toDateTime({p6:String}) AND toDateTime({p7:String}) ORDER BY session_id ASC, created_at ASC) SELECT session_id, arrayFilter( (x, i) -> i = 1 OR x != events_raw[i - 1], groupArray(event_name) as events_raw, arrayEnumerate(events_raw) ) as events_deduped FROM ordered_events GROUP BY session_id), events_sliced_cte AS (SELECT session_id, arraySlice( events_deduped, greatest(1, arrayFirstIndex(x -> x = {p8:String}, events_deduped) - {p9:UInt64} + 1), arrayFirstIndex(x -> x = {p10:String}, events_deduped) - greatest(1, arrayFirstIndex(x -> x = {p11:String}, events_deduped) - {p12:UInt64} + 1) + 1 ) as events_sliced FROM events_deduped_cte HAVING session_id IN (SELECT session_id FROM start_event_sessions)) SELECT session_id, events_sliced as events, events[1] as entry_event FROM events_sliced_cte HAVING length(events) >= {p13:UInt64}), session_paths AS (SELECT session_id, events FROM session_paths_base HAVING events[1] IN {p14:Array(String)}) SELECT pair.1 as source, pair.2 as target, pair.3 as step, count() as value FROM ((SELECT arrayJoin(arrayMap(i -> (events[i], events[i + 1], i), range(1, length(events)))) as pair FROM session_paths WHERE length(events) >= 2)) GROUP BY source, target, step ORDER BY step ASC, value DESC+-- V2 params: {"p1":"secure-privacy","p2":"session_end","p3":"2026-07-06 00:00:00","p4":"2026-07-12 23:59:59","p5":"secure-privacy","p6":"2026-07-06 00:00:00","p7":"2026-07-12 23:59:59","p8":"session_end","p9":2,"p10":"session_end","p11":"session_end","p12":2,"p13":2,"p14":["screen_view","link_out","web_vitals_lcp"]}+```++### sankey topEntries — between with filters on both anchors++**IDENTICAL** — rows V1/V2 = 1/1; rows_read V1/V2 = 475070/475070; wall V1/V2 = 80 ms / 75 ms; clickhouse_settings: {"session_timezone":"UTC"} (both).++```sql+-- V1+WITH session_paths AS (WITH between_sessions AS (WITH session_paths AS (WITH start_event_sessions AS (SELECT session_id FROM events WHERE project_id = 'secure-privacy' AND name = 'session_start' AND created_at BETWEEN toDateTime('2026-07-06 00:00:00') AND toDateTime('2026-07-12 23:59:59') AND country = 'US' GROUP BY session_id), end_event_sessions AS (SELECT session_id FROM events WHERE project_id = 'secure-privacy' AND name = 'session_end' AND created_at BETWEEN toDateTime('2026-07-06 00:00:00') AND toDateTime('2026-07-12 23:59:59') AND os = 'iOS' GROUP BY session_id), events_deduped_cte AS (WITH ordered_events AS (SELECT session_id, name as event_name, created_at FROM events WHERE project_id = 'secure-privacy' AND created_at BETWEEN toDateTime('2026-07-06 00:00:00') AND toDateTime('2026-07-12 23:59:59') ORDER BY session_id ASC, created_at ASC) SELECT session_id, arrayFilter( (x, i) -> i = 1 OR x != events_raw[i - 1], groupArray(event_name) as events_raw, arrayEnumerate(events_raw) ) as events_deduped FROM ordered_events GROUP BY session_id), events_sliced_cte AS (SELECT session_id, arraySlice(events_deduped, 1, 4) as events_sliced FROM events_deduped_cte HAVING session_id IN (SELECT session_id FROM start_event_sessions) AND session_id IN (SELECT session_id FROM end_event_sessions)) SELECT session_id, if( arrayFirstIndex(x -> x > 1, arrayEnumerateUniq(events_sliced)) = 0, events_sliced, arraySlice( events_sliced, 1, arrayFirstIndex(x -> x > 1, arrayEnumerateUniq(events_sliced)) - 1 ) ) as events, events[1] as entry_event FROM events_sliced_cte HAVING length(events) >= 2) SELECT session_id, events, arrayFirstIndex(x -> x = 'session_start', events) as start_index, arrayFirstIndex(x -> x = 'session_end', events) as end_index FROM session_paths HAVING start_index > 0 AND end_index > 0 AND start_index < end_index) SELECT session_id, arraySlice(events, start_index, end_index - start_index + 1) as events, events[start_index] as entry_event FROM between_sessions) SELECT entry_event, count() as count FROM session_paths GROUP BY entry_event ORDER BY count DESC LIMIT 3+-- V2+WITH session_paths AS (WITH between_sessions AS (WITH session_paths AS (WITH start_event_sessions AS (SELECT session_id FROM events WHERE project_id = {p1:String} AND name = {p2:String} AND created_at BETWEEN toDateTime({p3:String}) AND toDateTime({p4:String}) AND country = 'US' GROUP BY session_id), end_event_sessions AS (SELECT session_id FROM events WHERE project_id = {p5:String} AND name = {p6:String} AND created_at BETWEEN toDateTime({p7:String}) AND toDateTime({p8:String}) AND os = 'iOS' GROUP BY session_id), events_deduped_cte AS (WITH ordered_events AS (SELECT session_id, name as event_name, created_at FROM events WHERE project_id = {p9:String} AND created_at BETWEEN toDateTime({p10:String}) AND toDateTime({p11:String}) ORDER BY session_id ASC, created_at ASC) SELECT session_id, arrayFilter( (x, i) -> i = 1 OR x != events_raw[i - 1], groupArray(event_name) as events_raw, arrayEnumerate(events_raw) ) as events_deduped FROM ordered_events GROUP BY session_id), events_sliced_cte AS (SELECT session_id, arraySlice(events_deduped, 1, {p12:UInt64}) as events_sliced FROM events_deduped_cte HAVING session_id IN (SELECT session_id FROM start_event_sessions) AND session_id IN (SELECT session_id FROM end_event_sessions)) SELECT session_id, if( arrayFirstIndex(x -> x > 1, arrayEnumerateUniq(events_sliced)) = 0, events_sliced, arraySlice( events_sliced, 1, arrayFirstIndex(x -> x > 1, arrayEnumerateUniq(events_sliced)) - 1 ) ) as events, events[1] as entry_event FROM events_sliced_cte HAVING length(events) >= {p13:UInt64}) SELECT session_id, events, arrayFirstIndex(x -> x = {p14:String}, events) as start_index, arrayFirstIndex(x -> x = {p15:String}, events) as end_index FROM session_paths HAVING start_index > 0 AND end_index > 0 AND start_index < end_index) SELECT session_id, arraySlice(events, start_index, end_index - start_index + 1) as events, events[start_index] as entry_event FROM between_sessions) SELECT entry_event, count() as count FROM session_paths GROUP BY entry_event ORDER BY count DESC LIMIT {p16:UInt64}+-- V2 params: {"p1":"secure-privacy","p2":"session_start","p3":"2026-07-06 00:00:00","p4":"2026-07-12 23:59:59","p5":"secure-privacy","p6":"session_end","p7":"2026-07-06 00:00:00","p8":"2026-07-12 23:59:59","p9":"secure-privacy","p10":"2026-07-06 00:00:00","p11":"2026-07-12 23:59:59","p12":4,"p13":2,"p14":"session_start","p15":"session_end","p16":3}+```++### sankey transitions — between with filters on both anchors++**IDENTICAL** — rows V1/V2 = 4/4; rows_read V1/V2 = 475070/475070; wall V1/V2 = 79 ms / 80 ms; clickhouse_settings: {"session_timezone":"UTC"} (both).++```sql+-- V1+WITH between_sessions AS (WITH session_paths AS (WITH start_event_sessions AS (SELECT session_id FROM events WHERE project_id = 'secure-privacy' AND name = 'session_start' AND created_at BETWEEN toDateTime('2026-07-06 00:00:00') AND toDateTime('2026-07-12 23:59:59') AND country = 'US' GROUP BY session_id), end_event_sessions AS (SELECT session_id FROM events WHERE project_id = 'secure-privacy' AND name = 'session_end' AND created_at BETWEEN toDateTime('2026-07-06 00:00:00') AND toDateTime('2026-07-12 23:59:59') AND os = 'iOS' GROUP BY session_id), events_deduped_cte AS (WITH ordered_events AS (SELECT session_id, name as event_name, created_at FROM events WHERE project_id = 'secure-privacy' AND created_at BETWEEN toDateTime('2026-07-06 00:00:00') AND toDateTime('2026-07-12 23:59:59') ORDER BY session_id ASC, created_at ASC) SELECT session_id, arrayFilter( (x, i) -> i = 1 OR x != events_raw[i - 1], groupArray(event_name) as events_raw, arrayEnumerate(events_raw) ) as events_deduped FROM ordered_events GROUP BY session_id), events_sliced_cte AS (SELECT session_id, arraySlice(events_deduped, 1, 4) as events_sliced FROM events_deduped_cte HAVING session_id IN (SELECT session_id FROM start_event_sessions) AND session_id IN (SELECT session_id FROM end_event_sessions)) SELECT session_id, if( arrayFirstIndex(x -> x > 1, arrayEnumerateUniq(events_sliced)) = 0, events_sliced, arraySlice( events_sliced, 1, arrayFirstIndex(x -> x > 1, arrayEnumerateUniq(events_sliced)) - 1 ) ) as events, events[1] as entry_event FROM events_sliced_cte HAVING length(events) >= 2) SELECT session_id, events, arrayFirstIndex(x -> x = 'session_start', events) as start_index, arrayFirstIndex(x -> x = 'session_end', events) as end_index FROM session_paths HAVING start_index > 0 AND end_index > 0 AND start_index < end_index), session_paths AS (SELECT session_id, arraySlice(events, start_index, end_index - start_index + 1) as events FROM between_sessions HAVING events[1] IN ('session_start')) SELECT pair.1 as source, pair.2 as target, pair.3 as step, count() as value FROM ((SELECT arrayJoin(arrayMap(i -> (events[i], events[i + 1], i), range(1, length(events)))) as pair FROM session_paths WHERE length(events) >= 2)) GROUP BY source, target, step ORDER BY step ASC, value DESC+-- V2+WITH between_sessions AS (WITH session_paths AS (WITH start_event_sessions AS (SELECT session_id FROM events WHERE project_id = {p1:String} AND name = {p2:String} AND created_at BETWEEN toDateTime({p3:String}) AND toDateTime({p4:String}) AND country = 'US' GROUP BY session_id), end_event_sessions AS (SELECT session_id FROM events WHERE project_id = {p5:String} AND name = {p6:String} AND created_at BETWEEN toDateTime({p7:String}) AND toDateTime({p8:String}) AND os = 'iOS' GROUP BY session_id), events_deduped_cte AS (WITH ordered_events AS (SELECT session_id, name as event_name, created_at FROM events WHERE project_id = {p9:String} AND created_at BETWEEN toDateTime({p10:String}) AND toDateTime({p11:String}) ORDER BY session_id ASC, created_at ASC) SELECT session_id, arrayFilter( (x, i) -> i = 1 OR x != events_raw[i - 1], groupArray(event_name) as events_raw, arrayEnumerate(events_raw) ) as events_deduped FROM ordered_events GROUP BY session_id), events_sliced_cte AS (SELECT session_id, arraySlice(events_deduped, 1, {p12:UInt64}) as events_sliced FROM events_deduped_cte HAVING session_id IN (SELECT session_id FROM start_event_sessions) AND session_id IN (SELECT session_id FROM end_event_sessions)) SELECT session_id, if( arrayFirstIndex(x -> x > 1, arrayEnumerateUniq(events_sliced)) = 0, events_sliced, arraySlice( events_sliced, 1, arrayFirstIndex(x -> x > 1, arrayEnumerateUniq(events_sliced)) - 1 ) ) as events, events[1] as entry_event FROM events_sliced_cte HAVING length(events) >= {p13:UInt64}) SELECT session_id, events, arrayFirstIndex(x -> x = {p14:String}, events) as start_index, arrayFirstIndex(x -> x = {p15:String}, events) as end_index FROM session_paths HAVING start_index > 0 AND end_index > 0 AND start_index < end_index), session_paths AS (SELECT session_id, arraySlice(events, start_index, end_index - start_index + 1) as events FROM between_sessions HAVING events[1] IN {p16:Array(String)}) SELECT pair.1 as source, pair.2 as target, pair.3 as step, count() as value FROM ((SELECT arrayJoin(arrayMap(i -> (events[i], events[i + 1], i), range(1, length(events)))) as pair FROM session_paths WHERE length(events) >= 2)) GROUP BY source, target, step ORDER BY step ASC, value DESC+-- V2 params: {"p1":"secure-privacy","p2":"session_start","p3":"2026-07-06 00:00:00","p4":"2026-07-12 23:59:59","p5":"secure-privacy","p6":"session_end","p7":"2026-07-06 00:00:00","p8":"2026-07-12 23:59:59","p9":"secure-privacy","p10":"2026-07-06 00:00:00","p11":"2026-07-12 23:59:59","p12":4,"p13":2,"p14":"session_start","p15":"session_end","p16":["session_start"]}+```++### sankey topEntries — between without endEvent falls back to simple mode++**IDENTICAL** — rows V1/V2 = 2/2; rows_read V1/V2 = 106489/106489; wall V1/V2 = 63 ms / 72 ms; clickhouse_settings: {"session_timezone":"UTC"} (both).++```sql+-- V1+WITH session_paths AS (WITH start_event_sessions AS (SELECT session_id FROM events WHERE project_id = 'secure-privacy' AND name = 'screen_view' AND created_at BETWEEN toDateTime('2026-07-06 00:00:00') AND toDateTime('2026-07-12 23:59:59') GROUP BY session_id), events_deduped_cte AS (WITH ordered_events AS (SELECT session_id, name as event_name, created_at FROM events WHERE project_id = 'secure-privacy' AND created_at BETWEEN toDateTime('2026-07-06 00:00:00') AND toDateTime('2026-07-12 23:59:59') ORDER BY session_id ASC, created_at ASC) SELECT session_id, arrayFilter( (x, i) -> i = 1 OR x != events_raw[i - 1], groupArray(event_name) as events_raw, arrayEnumerate(events_raw) ) as events_deduped FROM ordered_events GROUP BY session_id), events_sliced_cte AS (SELECT session_id, arraySlice(events_deduped, 1, 3) as events_sliced FROM events_deduped_cte HAVING 1 = 1) SELECT session_id, if( arrayFirstIndex(x -> x > 1, arrayEnumerateUniq(events_sliced)) = 0, events_sliced, arraySlice( events_sliced, 1, arrayFirstIndex(x -> x > 1, arrayEnumerateUniq(events_sliced)) - 1 ) ) as events, events[1] as entry_event FROM events_sliced_cte HAVING length(events) >= 2) SELECT entry_event, count() as count FROM session_paths GROUP BY entry_event ORDER BY count DESC LIMIT 3+-- V2+WITH session_paths AS (WITH start_event_sessions AS (SELECT session_id FROM events WHERE project_id = {p1:String} AND name = {p2:String} AND created_at BETWEEN toDateTime({p3:String}) AND toDateTime({p4:String}) GROUP BY session_id), events_deduped_cte AS (WITH ordered_events AS (SELECT session_id, name as event_name, created_at FROM events WHERE project_id = {p5:String} AND created_at BETWEEN toDateTime({p6:String}) AND toDateTime({p7:String}) ORDER BY session_id ASC, created_at ASC) SELECT session_id, arrayFilter( (x, i) -> i = 1 OR x != events_raw[i - 1], groupArray(event_name) as events_raw, arrayEnumerate(events_raw) ) as events_deduped FROM ordered_events GROUP BY session_id), events_sliced_cte AS (SELECT session_id, arraySlice(events_deduped, 1, {p8:UInt64}) as events_sliced FROM events_deduped_cte HAVING 1 = 1) SELECT session_id, if( arrayFirstIndex(x -> x > 1, arrayEnumerateUniq(events_sliced)) = 0, events_sliced, arraySlice( events_sliced, 1, arrayFirstIndex(x -> x > 1, arrayEnumerateUniq(events_sliced)) - 1 ) ) as events, events[1] as entry_event FROM events_sliced_cte HAVING length(events) >= {p9:UInt64}) SELECT entry_event, count() as count FROM session_paths GROUP BY entry_event ORDER BY count DESC LIMIT {p10:UInt64}+-- V2 params: {"p1":"secure-privacy","p2":"screen_view","p3":"2026-07-06 00:00:00","p4":"2026-07-12 23:59:59","p5":"secure-privacy","p6":"2026-07-06 00:00:00","p7":"2026-07-12 23:59:59","p8":3,"p9":2,"p10":3}+```++### sankey transitions — between without endEvent falls back to simple mode++**IDENTICAL** — rows V1/V2 = 17/17; rows_read V1/V2 = 106489/106489; wall V1/V2 = 102 ms / 82 ms; clickhouse_settings: {"session_timezone":"UTC"} (both).++```sql+-- V1+WITH session_paths_base AS (WITH start_event_sessions AS (SELECT session_id FROM events WHERE project_id = 'secure-privacy' AND name = 'screen_view' AND created_at BETWEEN toDateTime('2026-07-06 00:00:00') AND toDateTime('2026-07-12 23:59:59') GROUP BY session_id), events_deduped_cte AS (WITH ordered_events AS (SELECT session_id, name as event_name, created_at FROM events WHERE project_id = 'secure-privacy' AND created_at BETWEEN toDateTime('2026-07-06 00:00:00') AND toDateTime('2026-07-12 23:59:59') ORDER BY session_id ASC, created_at ASC) SELECT session_id, arrayFilter( (x, i) -> i = 1 OR x != events_raw[i - 1], groupArray(event_name) as events_raw, arrayEnumerate(events_raw) ) as events_deduped FROM ordered_events GROUP BY session_id), events_sliced_cte AS (SELECT session_id, arraySlice(events_deduped, 1, 3) as events_sliced FROM events_deduped_cte HAVING 1 = 1) SELECT session_id, if( arrayFirstIndex(x -> x > 1, arrayEnumerateUniq(events_sliced)) = 0, events_sliced, arraySlice( events_sliced, 1, arrayFirstIndex(x -> x > 1, arrayEnumerateUniq(events_sliced)) - 1 ) ) as events, events[1] as entry_event FROM events_sliced_cte HAVING length(events) >= 2), session_paths AS (SELECT session_id, events FROM session_paths_base HAVING events[1] IN ('session_start', 'screen_view')) SELECT pair.1 as source, pair.2 as target, pair.3 as step, count() as value FROM ((SELECT arrayJoin(arrayMap(i -> (events[i], events[i + 1], i), range(1, length(events)))) as pair FROM session_paths WHERE length(events) >= 2)) GROUP BY source, target, step ORDER BY step ASC, value DESC+-- V2+WITH session_paths_base AS (WITH start_event_sessions AS (SELECT session_id FROM events WHERE project_id = {p1:String} AND name = {p2:String} AND created_at BETWEEN toDateTime({p3:String}) AND toDateTime({p4:String}) GROUP BY session_id), events_deduped_cte AS (WITH ordered_events AS (SELECT session_id, name as event_name, created_at FROM events WHERE project_id = {p5:String} AND created_at BETWEEN toDateTime({p6:String}) AND toDateTime({p7:String}) ORDER BY session_id ASC, created_at ASC) SELECT session_id, arrayFilter( (x, i) -> i = 1 OR x != events_raw[i - 1], groupArray(event_name) as events_raw, arrayEnumerate(events_raw) ) as events_deduped FROM ordered_events GROUP BY session_id), events_sliced_cte AS (SELECT session_id, arraySlice(events_deduped, 1, {p8:UInt64}) as events_sliced FROM events_deduped_cte HAVING 1 = 1) SELECT session_id, if( arrayFirstIndex(x -> x > 1, arrayEnumerateUniq(events_sliced)) = 0, events_sliced, arraySlice( events_sliced, 1, arrayFirstIndex(x -> x > 1, arrayEnumerateUniq(events_sliced)) - 1 ) ) as events, events[1] as entry_event FROM events_sliced_cte HAVING length(events) >= {p9:UInt64}), session_paths AS (SELECT session_id, events FROM session_paths_base HAVING events[1] IN {p10:Array(String)}) SELECT pair.1 as source, pair.2 as target, pair.3 as step, count() as value FROM ((SELECT arrayJoin(arrayMap(i -> (events[i], events[i + 1], i), range(1, length(events)))) as pair FROM session_paths WHERE length(events) >= 2)) GROUP BY source, target, step ORDER BY step ASC, value DESC+-- V2 params: {"p1":"secure-privacy","p2":"screen_view","p3":"2026-07-06 00:00:00","p4":"2026-07-12 23:59:59","p5":"secure-privacy","p6":"2026-07-06 00:00:00","p7":"2026-07-12 23:59:59","p8":3,"p9":2,"p10":["session_start","screen_view"]}+```++### sankey topEntries — non-UTC session timezone (America/New_York), after++**IDENTICAL** — rows V1/V2 = 1/1; rows_read V1/V2 = 7863873/7863873; wall V1/V2 = 3555 ms / 3868 ms; clickhouse_settings: {"session_timezone":"America/New_York"} (both).++```sql+-- V1+WITH session_paths AS (WITH start_event_sessions AS (SELECT session_id FROM events WHERE project_id = 'bayse' AND name = 'session_start' AND created_at BETWEEN toDateTime('2026-07-06 00:00:00') AND toDateTime('2026-07-08 23:59:59') GROUP BY session_id), events_deduped_cte AS (WITH ordered_events AS (SELECT session_id, name as event_name, created_at FROM events WHERE project_id = 'bayse' AND created_at BETWEEN toDateTime('2026-07-06 00:00:00') AND toDateTime('2026-07-08 23:59:59') ORDER BY session_id ASC, created_at ASC) SELECT session_id, arrayFilter( (x, i) -> i = 1 OR x != events_raw[i - 1], groupArray(event_name) as events_raw, arrayEnumerate(events_raw) ) as events_deduped FROM ordered_events GROUP BY session_id), events_sliced_cte AS (SELECT session_id, arraySlice(events_deduped, arrayFirstIndex(x -> x = 'session_start', events_deduped), 3) as events_sliced FROM events_deduped_cte HAVING session_id IN (SELECT session_id FROM start_event_sessions)) SELECT session_id, if( arrayFirstIndex(x -> x > 1, arrayEnumerateUniq(events_sliced)) = 0, events_sliced, arraySlice( events_sliced, 1, arrayFirstIndex(x -> x > 1, arrayEnumerateUniq(events_sliced)) - 1 ) ) as events, events[1] as entry_event FROM events_sliced_cte HAVING length(events) >= 2) SELECT entry_event, count() as count FROM session_paths GROUP BY entry_event ORDER BY count DESC LIMIT 3+-- V2+WITH session_paths AS (WITH start_event_sessions AS (SELECT session_id FROM events WHERE project_id = {p1:String} AND name = {p2:String} AND created_at BETWEEN toDateTime({p3:String}) AND toDateTime({p4:String}) GROUP BY session_id), events_deduped_cte AS (WITH ordered_events AS (SELECT session_id, name as event_name, created_at FROM events WHERE project_id = {p5:String} AND created_at BETWEEN toDateTime({p6:String}) AND toDateTime({p7:String}) ORDER BY session_id ASC, created_at ASC) SELECT session_id, arrayFilter( (x, i) -> i = 1 OR x != events_raw[i - 1], groupArray(event_name) as events_raw, arrayEnumerate(events_raw) ) as events_deduped FROM ordered_events GROUP BY session_id), events_sliced_cte AS (SELECT session_id, arraySlice(events_deduped, arrayFirstIndex(x -> x = {p8:String}, events_deduped), {p9:UInt64}) as events_sliced FROM events_deduped_cte HAVING session_id IN (SELECT session_id FROM start_event_sessions)) SELECT session_id, if( arrayFirstIndex(x -> x > 1, arrayEnumerateUniq(events_sliced)) = 0, events_sliced, arraySlice( events_sliced, 1, arrayFirstIndex(x -> x > 1, arrayEnumerateUniq(events_sliced)) - 1 ) ) as events, events[1] as entry_event FROM events_sliced_cte HAVING length(events) >= {p10:UInt64}) SELECT entry_event, count() as count FROM session_paths GROUP BY entry_event ORDER BY count DESC LIMIT {p11:UInt64}+-- V2 params: {"p1":"bayse","p2":"session_start","p3":"2026-07-06 00:00:00","p4":"2026-07-08 23:59:59","p5":"bayse","p6":"2026-07-06 00:00:00","p7":"2026-07-08 23:59:59","p8":"session_start","p9":3,"p10":2,"p11":3}+```++### sankey transitions — non-UTC session timezone (America/New_York), after++**IDENTICAL AS SET** — rows V1/V2 = 515/515; rows_read V1/V2 = 7863873/7863873; wall V1/V2 = 3533 ms / 3659 ms; clickhouse_settings: {"session_timezone":"America/New_York"} (both).++```sql+-- V1+WITH session_paths_base AS (WITH start_event_sessions AS (SELECT session_id FROM events WHERE project_id = 'bayse' AND name = 'session_start' AND created_at BETWEEN toDateTime('2026-07-06 00:00:00') AND toDateTime('2026-07-08 23:59:59') GROUP BY session_id), events_deduped_cte AS (WITH ordered_events AS (SELECT session_id, name as event_name, created_at FROM events WHERE project_id = 'bayse' AND created_at BETWEEN toDateTime('2026-07-06 00:00:00') AND toDateTime('2026-07-08 23:59:59') ORDER BY session_id ASC, created_at ASC) SELECT session_id, arrayFilter( (x, i) -> i = 1 OR x != events_raw[i - 1], groupArray(event_name) as events_raw, arrayEnumerate(events_raw) ) as events_deduped FROM ordered_events GROUP BY session_id), events_sliced_cte AS (SELECT session_id, arraySlice(events_deduped, arrayFirstIndex(x -> x = 'session_start', events_deduped), 3) as events_sliced FROM events_deduped_cte HAVING session_id IN (SELECT session_id FROM start_event_sessions)) SELECT session_id, if( arrayFirstIndex(x -> x > 1, arrayEnumerateUniq(events_sliced)) = 0, events_sliced, arraySlice( events_sliced, 1, arrayFirstIndex(x -> x > 1, arrayEnumerateUniq(events_sliced)) - 1 ) ) as events, events[1] as entry_event FROM events_sliced_cte HAVING length(events) >= 2), session_paths AS (SELECT session_id, events FROM session_paths_base HAVING events[1] IN ('session_start')) SELECT pair.1 as source, pair.2 as target, pair.3 as step, count() as value FROM ((SELECT arrayJoin(arrayMap(i -> (events[i], events[i + 1], i), range(1, length(events)))) as pair FROM session_paths WHERE length(events) >= 2)) GROUP BY source, target, step ORDER BY step ASC, value DESC+-- V2+WITH session_paths_base AS (WITH start_event_sessions AS (SELECT session_id FROM events WHERE project_id = {p1:String} AND name = {p2:String} AND created_at BETWEEN toDateTime({p3:String}) AND toDateTime({p4:String}) GROUP BY session_id), events_deduped_cte AS (WITH ordered_events AS (SELECT session_id, name as event_name, created_at FROM events WHERE project_id = {p5:String} AND created_at BETWEEN toDateTime({p6:String}) AND toDateTime({p7:String}) ORDER BY session_id ASC, created_at ASC) SELECT session_id, arrayFilter( (x, i) -> i = 1 OR x != events_raw[i - 1], groupArray(event_name) as events_raw, arrayEnumerate(events_raw) ) as events_deduped FROM ordered_events GROUP BY session_id), events_sliced_cte AS (SELECT session_id, arraySlice(events_deduped, arrayFirstIndex(x -> x = {p8:String}, events_deduped), {p9:UInt64}) as events_sliced FROM events_deduped_cte HAVING session_id IN (SELECT session_id FROM start_event_sessions)) SELECT session_id, if( arrayFirstIndex(x -> x > 1, arrayEnumerateUniq(events_sliced)) = 0, events_sliced, arraySlice( events_sliced, 1, arrayFirstIndex(x -> x > 1, arrayEnumerateUniq(events_sliced)) - 1 ) ) as events, events[1] as entry_event FROM events_sliced_cte HAVING length(events) >= {p10:UInt64}), session_paths AS (SELECT session_id, events FROM session_paths_base HAVING events[1] IN {p11:Array(String)}) SELECT pair.1 as source, pair.2 as target, pair.3 as step, count() as value FROM ((SELECT arrayJoin(arrayMap(i -> (events[i], events[i + 1], i), range(1, length(events)))) as pair FROM session_paths WHERE length(events) >= 2)) GROUP BY source, target, step ORDER BY step ASC, value DESC+-- V2 params: {"p1":"bayse","p2":"session_start","p3":"2026-07-06 00:00:00","p4":"2026-07-08 23:59:59","p5":"bayse","p6":"2026-07-06 00:00:00","p7":"2026-07-08 23:59:59","p8":"session_start","p9":3,"p10":2,"p11":["session_start"]}+```++### sankey stmt0 — no matching entry events -> empty++**IDENTICAL** — rows V1/V2 = 0/0; rows_read V1/V2 = 245717/245717; wall V1/V2 = 32 ms / 34 ms; clickhouse_settings: {"session_timezone":"UTC"} (both).++```sql+-- V1+WITH session_paths AS (WITH start_event_sessions AS (SELECT session_id FROM events WHERE project_id = 'secure-privacy' AND name = 'no_such_event_xyz' AND created_at BETWEEN toDateTime('2026-07-06 00:00:00') AND toDateTime('2026-07-12 23:59:59') GROUP BY session_id), events_deduped_cte AS (WITH ordered_events AS (SELECT session_id, name as event_name, created_at FROM events WHERE project_id = 'secure-privacy' AND created_at BETWEEN toDateTime('2026-07-06 00:00:00') AND toDateTime('2026-07-12 23:59:59') ORDER BY session_id ASC, created_at ASC) SELECT session_id, arrayFilter( (x, i) -> i = 1 OR x != events_raw[i - 1], groupArray(event_name) as events_raw, arrayEnumerate(events_raw) ) as events_deduped FROM ordered_events GROUP BY session_id), events_sliced_cte AS (SELECT session_id, arraySlice(events_deduped, arrayFirstIndex(x -> x = 'no_such_event_xyz', events_deduped), 3) as events_sliced FROM events_deduped_cte HAVING session_id IN (SELECT session_id FROM start_event_sessions)) SELECT session_id, if( arrayFirstIndex(x -> x > 1, arrayEnumerateUniq(events_sliced)) = 0, events_sliced, arraySlice( events_sliced, 1, arrayFirstIndex(x -> x > 1, arrayEnumerateUniq(events_sliced)) - 1 ) ) as events, events[1] as entry_event FROM events_sliced_cte HAVING length(events) >= 2) SELECT entry_event, count() as count FROM session_paths GROUP BY entry_event ORDER BY count DESC LIMIT 3+-- V2+WITH session_paths AS (WITH start_event_sessions AS (SELECT session_id FROM events WHERE project_id = {p1:String} AND name = {p2:String} AND created_at BETWEEN toDateTime({p3:String}) AND toDateTime({p4:String}) GROUP BY session_id), events_deduped_cte AS (WITH ordered_events AS (SELECT session_id, name as event_name, created_at FROM events WHERE project_id = {p5:String} AND created_at BETWEEN toDateTime({p6:String}) AND toDateTime({p7:String}) ORDER BY session_id ASC, created_at ASC) SELECT session_id, arrayFilter( (x, i) -> i = 1 OR x != events_raw[i - 1], groupArray(event_name) as events_raw, arrayEnumerate(events_raw) ) as events_deduped FROM ordered_events GROUP BY session_id), events_sliced_cte AS (SELECT session_id, arraySlice(events_deduped, arrayFirstIndex(x -> x = {p8:String}, events_deduped), {p9:UInt64}) as events_sliced FROM events_deduped_cte HAVING session_id IN (SELECT session_id FROM start_event_sessions)) SELECT session_id, if( arrayFirstIndex(x -> x > 1, arrayEnumerateUniq(events_sliced)) = 0, events_sliced, arraySlice( events_sliced, 1, arrayFirstIndex(x -> x > 1, arrayEnumerateUniq(events_sliced)) - 1 ) ) as events, events[1] as entry_event FROM events_sliced_cte HAVING length(events) >= {p10:UInt64}) SELECT entry_event, count() as count FROM session_paths GROUP BY entry_event ORDER BY count DESC LIMIT {p11:UInt64}+-- V2 params: {"p1":"secure-privacy","p2":"no_such_event_xyz","p3":"2026-07-06 00:00:00","p4":"2026-07-12 23:59:59","p5":"secure-privacy","p6":"2026-07-06 00:00:00","p7":"2026-07-12 23:59:59","p8":"no_such_event_xyz","p9":3,"p10":2,"p11":3}+```diff --git a/packages/core/src/modules/chart/src/sankey.sql.ts b/packages/core/src/modules/chart/src/sankey.sql.tsnew file mode 100644index 000000000..26176feb5--- /dev/null+++ b/packages/core/src/modules/chart/src/sankey.sql.ts@@ -0,0 +1,243 @@+// Every ClickHouse statement the sankey (user-flow) service runs, as pure+// `sql` fragments (ADR-013). Converted 1:1 from+// packages/db/src/services/sankey.service.ts (M7-004): the SQL text is V1's+// clix output, with the project id, event names, dates, the step count and the+// top-entry event list bound as `{pN:Type}` parameters instead of hand-escaped+// literals. Result-set proof: sankey.sql.proof.md.+//+// The filter compiler still renders text (see compiled.ts); only its output is+// spliced.+//+// Cluster note (docs/ENVIRONMENT.md): `events` is Distributed on Cloud, and the+// `session_id IN (SELECT session_id FROM start_event_sessions)` HAVING clauses+// are plain `IN (subquery)` over a CTE — V1's exact shape, ported verbatim. No+// `IN` is converted to or from `GLOBAL IN` here; whether these should be+// `GLOBAL IN` on a cluster is a pre-existing question this conversion neither+// answers nor changes.++import { type SqlFragment, sql } from '@openpanel/db/src/clickhouse/sql';+import { compiledText } from './compiled';+import { CHART_TABLE } from './field-resolution';++export type SankeyMode = 'after' | 'before' | 'between';++/** How many entry events the flow starts from. */+export const TOP_ENTRY_EVENTS = 3;++/** The dedup + transition pieces every mode shares, verbatim from V1. */+const DEDUPE_CONSECUTIVE = compiledText(`arrayFilter(+ (x, i) -> i = 1 OR x != events_raw[i - 1],+ groupArray(event_name) as events_raw,+ arrayEnumerate(events_raw)+ ) as events_deduped`);++/**+ * 'after' mode stops the path at the first repeated event, so a loop back to+ * the entry event ends the flow instead of folding onto itself.+ */+const TRUNCATE_AT_REPEAT = compiledText(`if(+ arrayFirstIndex(x -> x > 1, arrayEnumerateUniq(events_sliced)) = 0,+ events_sliced,+ arraySlice(+ events_sliced,+ 1,+ arrayFirstIndex(x -> x > 1, arrayEnumerateUniq(events_sliced)) - 1+ )+ )`);++const TRANSITION_PAIRS = compiledText(+ '(SELECT arrayJoin(arrayMap(i -> (events[i], events[i + 1], i), range(1, length(events)))) as pair FROM session_paths WHERE length(events) >= 2)'+);++const MIN_PATH_LENGTH = 2;++export interface SankeyEvent {+ name: string;+ /** Compiled filter clauses for this event's session CTE. */+ whereClause: string;+}++export interface SankeyPathsInput {+ projectId: string;+ startDate: string;+ endDate: string;+ steps: number;+ mode: SankeyMode;+ startEvent?: SankeyEvent;+ endEvent?: SankeyEvent;+ /** Compiled `name IN (...)` / `name NOT IN (...)` clause, if any. */+ include?: string[];+ exclude: string[];+}++function dateRange(input: SankeyPathsInput): SqlFragment {+ return sql`project_id = ${sql.string(input.projectId)} AND created_at BETWEEN toDateTime(${sql.string(input.startDate)}) AND toDateTime(${sql.string(input.endDate)})`;+}++/**+ * V1 `buildEventNameFilter`: an include list wins over an exclude list, and it+ * always carries the start/end events so the flow's anchors survive it.+ */+function eventNameFilter(input: SankeyPathsInput): SqlFragment {+ if (input.include && input.include.length > 0) {+ const names = [+ ...input.include,+ input.startEvent?.name,+ input.endEvent?.name,+ ].filter((name): name is string => name !== undefined);+ return sql` AND name IN ${sql.array('String', names)}`;+ }+ if (input.exclude.length > 0) {+ return sql` AND name NOT IN ${sql.array('String', input.exclude)}`;+ }+ return sql.empty;+}++/** Sessions in which one anchor event (with its own filters) occurred. */+function sessionEventCte(+ event: SankeyEvent,+ input: SankeyPathsInput+): SqlFragment {+ const filters = event.whereClause+ ? sql` AND ${compiledText(event.whereClause)}`+ : sql.empty;+ return sql`SELECT session_id FROM ${sql.id(CHART_TABLE.events)} WHERE project_id = ${sql.string(input.projectId)} AND name = ${sql.string(event.name)} AND created_at BETWEEN toDateTime(${sql.string(input.startDate)}) AND toDateTime(${sql.string(input.endDate)})${filters} GROUP BY session_id`;+}++interface ModeConfig {+ /** `null` when the mode has no session restriction (V1's `1 = 1`). */+ sessionFilter: SqlFragment | null;+ eventsSlice: SqlFragment;+}++function modeConfig(input: SankeyPathsInput): ModeConfig {+ const { mode, startEvent, endEvent, steps } = input;+ const stepCount = sql.uint64(steps);+ const defaultSlice = sql`arraySlice(events_deduped, 1, ${stepCount})`;+ const inStartSessions = sql`session_id IN (SELECT session_id FROM start_event_sessions)`;++ if (mode === 'after' && startEvent) {+ const name = sql.string(startEvent.name);+ return {+ sessionFilter: inStartSessions,+ eventsSlice: sql`arraySlice(events_deduped, arrayFirstIndex(x -> x = ${name}, events_deduped), ${stepCount})`,+ };+ }++ if (mode === 'before' && startEvent) {+ const name = sql.string(startEvent.name);+ const first = sql`arrayFirstIndex(x -> x = ${name}, events_deduped)`;+ const from = sql`greatest(1, ${first} - ${stepCount} + 1)`;+ return {+ sessionFilter: inStartSessions,+ eventsSlice: sql`arraySlice(+ events_deduped,+ ${from},+ ${first} - ${from} + 1+ )`,+ };+ }++ if (mode === 'between' && startEvent && endEvent) {+ return {+ sessionFilter: sql`${inStartSessions} AND session_id IN (SELECT session_id FROM end_event_sessions)`,+ eventsSlice: defaultSlice,+ };+ }++ return { sessionFilter: null, eventsSlice: defaultSlice };+}++/**+ * V1's `sessionPathsQuery`: one row per session carrying its deduped, sliced+ * event path and that path's entry event.+ */+export function sankeySessionPathsQuery(input: SankeyPathsInput): SqlFragment {+ const { sessionFilter, eventsSlice } = modeConfig(input);+ const anchorCtes: SqlFragment[] = [];+ if (input.startEvent) {+ anchorCtes.push(+ sql`start_event_sessions AS (${sessionEventCte(input.startEvent, input)})`+ );+ }+ if (input.mode === 'between' && input.endEvent) {+ anchorCtes.push(+ sql`end_event_sessions AS (${sessionEventCte(input.endEvent, input)})`+ );+ }+ const anchors =+ anchorCtes.length > 0 ? sql`${sql.join(anchorCtes)}, ` : sql.empty;++ const having = sessionFilter ?? sql`1 = 1`;+ const eventsExpr =+ input.mode === 'before' ? sql`events_sliced` : TRUNCATE_AT_REPEAT;++ return sql`WITH ${anchors}events_deduped_cte AS (WITH ordered_events AS (SELECT session_id, name as event_name, created_at FROM ${sql.id(CHART_TABLE.events)} WHERE ${dateRange(input)}${eventNameFilter(input)} ORDER BY session_id ASC, created_at ASC) SELECT session_id, ${DEDUPE_CONSECUTIVE} FROM ordered_events GROUP BY session_id), events_sliced_cte AS (SELECT session_id, ${eventsSlice} as events_sliced FROM events_deduped_cte HAVING ${having}) SELECT session_id, ${eventsExpr} as events, events[1] as entry_event FROM events_sliced_cte HAVING length(events) >= ${sql.uint64(MIN_PATH_LENGTH)}`;+}++/** `between` mode narrows each path to the slice from the start to the end event. */+function betweenSessionsQuery(+ input: SankeyPathsInput,+ startEvent: SankeyEvent,+ endEvent: SankeyEvent+): SqlFragment {+ return sql`WITH session_paths AS (${sankeySessionPathsQuery(input)}) SELECT session_id, events, arrayFirstIndex(x -> x = ${sql.string(startEvent.name)}, events) as start_index, arrayFirstIndex(x -> x = ${sql.string(endEvent.name)}, events) as end_index FROM session_paths HAVING start_index > 0 AND end_index > 0 AND start_index < end_index`;+}++const BETWEEN_SLICE = compiledText(+ 'arraySlice(events, start_index, end_index - start_index + 1)'+);++function topEntriesFrom(sessionPaths: SqlFragment): SqlFragment {+ return sql`WITH session_paths AS (${sessionPaths}) SELECT entry_event, count() as count FROM session_paths GROUP BY entry_event ORDER BY count DESC LIMIT ${sql.uint64(TOP_ENTRY_EVENTS)}`;+}++function transitionsFrom(+ baseCteName: string,+ baseQuery: SqlFragment,+ pathsSelect: SqlFragment,+ topEntryEvents: string[]+): SqlFragment {+ return sql`WITH ${compiledText(baseCteName)} AS (${baseQuery}), session_paths AS (SELECT ${pathsSelect} FROM ${compiledText(baseCteName)} HAVING events[1] IN ${sql.array('String', topEntryEvents)}) SELECT pair.1 as source, pair.2 as target, pair.3 as step, count() as value FROM (${TRANSITION_PAIRS}) GROUP BY source, target, step ORDER BY step ASC, value DESC`;+}++/** `after` / `before` modes: top entry events straight off the session paths. */+export function sankeyTopEntriesQuery(input: SankeyPathsInput): SqlFragment {+ return topEntriesFrom(sankeySessionPathsQuery(input));+}++export function sankeyTransitionsQuery(+ input: SankeyPathsInput,+ topEntryEvents: string[]+): SqlFragment {+ return transitionsFrom(+ 'session_paths_base',+ sankeySessionPathsQuery(input),+ sql`session_id, events`,+ topEntryEvents+ );+}++/** `between` mode: the same two statements over the start→end slice. */+export function sankeyBetweenTopEntriesQuery(+ input: SankeyPathsInput,+ startEvent: SankeyEvent,+ endEvent: SankeyEvent+): SqlFragment {+ const betweenPaths = sql`WITH between_sessions AS (${betweenSessionsQuery(input, startEvent, endEvent)}) SELECT session_id, ${BETWEEN_SLICE} as events, events[start_index] as entry_event FROM between_sessions`;+ return topEntriesFrom(betweenPaths);+}++export function sankeyBetweenTransitionsQuery(+ input: SankeyPathsInput,+ startEvent: SankeyEvent,+ endEvent: SankeyEvent,+ topEntryEvents: string[]+): SqlFragment {+ return transitionsFrom(+ 'between_sessions',+ betweenSessionsQuery(input, startEvent, endEvent),+ sql`session_id, ${BETWEEN_SLICE} as events`,+ topEntryEvents+ );+}diff --git a/packages/core/src/modules/mcp/src/tools/analytics/reports.ts b/packages/core/src/modules/mcp/src/tools/analytics/reports.tsindex bbf0865b7..dc4a3d845 100644--- a/packages/core/src/modules/mcp/src/tools/analytics/reports.ts+++ b/packages/core/src/modules/mcp/src/tools/analytics/reports.ts@@ -3,8 +3,8 @@ import {AggregateChartEngine,ChartEngine,db,- funnelService,getChartStartEndDate,+ getFunnel,getReportById,getReportsByDashboardId,getSettingsForProject,@@ -281,7 +281,7 @@ export function registerReportTools(/*** Execute a saved report by ID. Dispatches on chart type:- * - funnel → funnelService.getFunnel+ * - funnel → getFunnel* - metric → AggregateChartEngine.execute* - others → ChartEngine.execute*@@ -343,7 +343,7 @@ export async function runReport(input: {};if (report.chartType === 'funnel') {- return { ...meta, data: await funnelService.getFunnel(chartInput) };+ return { ...meta, data: await getFunnel(chartInput) };}if (report.chartType === 'metric') {return { ...meta, data: await AggregateChartEngine.execute(chartInput) };@@ -392,9 +392,7 @@ export async function runReportFromConfig(input: {if (input.config.chartType === 'funnel') {return {...meta,- data: await funnelService.getFunnel(- chartInput as Parameters<typeof funnelService.getFunnel>[0]- ),+ data: await getFunnel(chartInput as Parameters<typeof getFunnel>[0]),};}if (input.config.chartType === 'metric') {diff --git a/packages/db/src/services/conversion.service.ts b/packages/db/src/services/conversion.service.tsindex fc721722d..d30fd46d9 100644--- a/packages/db/src/services/conversion.service.ts+++ b/packages/db/src/services/conversion.service.ts@@ -1,287 +1,3 @@-import { NOT_SET_VALUE } from '@openpanel/constants';-import type { IReportInput } from '@openpanel/validation';-import { omit } from 'ramda';-import sqlstring from 'sqlstring';-import { TABLE_NAMES, ch } from '../clickhouse/client';-import { clix } from '../clickhouse/query-builder';-import {- buildInlineCohortJoin,- collectBreakdownCohortIds,- extractCohortId,- fetchCohortsMetadata,- getEventFiltersWhereClause,- getSelectPropertyKey,- isKnownEventField,-} from './chart.service';-import { mergeGlobalFilters, onlyReportEvents } from './reports.service';--export class ConversionService {- constructor(private client: typeof ch) {}-- async getConversion({- projectId,- startDate,- endDate,- options,- series,- globalFilters,- breakdowns = [],- limit,- interval,- timezone,- }: Omit<IReportInput, 'range' | 'previous' | 'metric' | 'chartType'> & {- timezone: string;- }) {- series = mergeGlobalFilters(series, globalFilters);- const funnelOptions = options?.type === 'funnel' ? options : undefined;- const funnelGroup = funnelOptions?.funnelGroup;- const funnelWindow = funnelOptions?.funnelWindow ?? 24;- const group = funnelGroup === 'profile_id' ? 'profile_id' : 'session_id';-- // Same guard as FunnelService / getChartSql — drop breakdowns whose name- // can't be resolved against the events schema.- breakdowns = breakdowns.filter((b) => isKnownEventField(b.name));-- const cohortIds = collectBreakdownCohortIds(breakdowns);- const cohortMetadata = await fetchCohortsMetadata(cohortIds);- const cohortJoinsSql = cohortIds- .map((id) => buildInlineCohortJoin(id, projectId, 'events'))- .join(' ');-- const breakdownExpressions = breakdowns.map((b) => {- const bId = extractCohortId(b.name);- const bName = bId ? cohortMetadata.get(bId)?.name : undefined;- return getSelectPropertyKey(b.name, projectId, bId ?? undefined, bName);- });- const breakdownSelects = breakdownExpressions.map(- (expr, index) => `${expr} as b_${index}`,- );- const breakdownGroupBy = breakdowns.map((_, index) => `b_${index}`);-- // Check if any breakdown or filter uses profile fields- const profileBreakdowns = breakdowns.filter((b) =>- b.name.startsWith('profile.'),- );- const needsProfileJoin = profileBreakdowns.length > 0;-- // Build profile JOIN clause if needed- let profileJoin = '';- if (needsProfileJoin) {- const profileFields = new Set<string>();- profileFields.add('id');-- for (const b of profileBreakdowns) {- const fieldName = b.name.replace('profile.', '').split('.')[0];- if (fieldName === 'properties') {- profileFields.add('properties');- } else if (- [- 'email',- 'first_name',- 'last_name',- 'created_at',- 'last_seen_at',- ].includes(fieldName!)- ) {- profileFields.add(fieldName!);- }- }-- // Use simple column names (not aliased) so profile.properties works directly- const selectFields = Array.from(profileFields);-- profileJoin = `LEFT ANY JOIN (- SELECT ${selectFields.join(', ')}- FROM ${TABLE_NAMES.profiles} FINAL- WHERE project_id = ${sqlstring.escape(projectId)}- ) as profile ON profile.id = profile_id`;- }-- const events = onlyReportEvents(series);-- // Check if any breakdown or filter uses group fields- const anyBreakdownOnGroup = breakdowns.some((b) =>- b.name.startsWith('group.'),- );- const anyFilterOnGroup = events.some((e) =>- e.filters?.some((f) => f.name.startsWith('group.')),- );- const needsGroupArrayJoin = anyBreakdownOnGroup || anyFilterOnGroup;-- if (events.length !== 2) {- throw new Error('events must be an array of two events');- }-- if (!startDate || !endDate) {- throw new Error('startDate and endDate are required');- }-- const eventA = events[0]!;- const eventB = events[1]!;- // Qualify with 'events' so event-level `properties[...]` becomes- // `events.properties[...]` — required when the conversion query also- // joins the profiles table (which exposes a `properties` column).- // Without the qualifier ClickHouse fails with "ambiguous identifier- // 'properties'" whenever a step filters on properties.X while a- // breakdown is on profile.properties.Y.- const whereA = Object.values(- getEventFiltersWhereClause(eventA.filters, projectId, 'events'),- ).join(' AND ');- const whereB = Object.values(- getEventFiltersWhereClause(eventB.filters, projectId, 'events'),- ).join(' AND ');-- const funnelWindowSeconds = funnelWindow * 3600;-- // Build funnel conditions- const conditionA = whereA- ? `(events.name = '${eventA.name}' AND ${whereA})`- : `events.name = '${eventA.name}'`;- const conditionB = whereB- ? `(events.name = '${eventB.name}' AND ${whereB})`- : `events.name = '${eventB.name}'`;-- const groupJoin = needsGroupArrayJoin- ? `ARRAY JOIN groups AS _group_id LEFT ANY JOIN (SELECT id, name, type, properties FROM ${TABLE_NAMES.groups} FINAL WHERE project_id = ${sqlstring.escape(projectId)}) AS _g ON _g.id = _group_id`- : '';-- // Use windowFunnel approach - single scan, no JOIN- const query = clix(this.client, timezone)- .select<{- event_day: string;- total_first: number;- conversions: number;- conversion_rate_percentage: number;- [key: string]: string | number;- }>([- 'event_day',- ...breakdownGroupBy,- `uniqExact(${group}) AS total_first`,- 'countIf(steps >= 2) AS conversions',- `round(100.0 * countIf(steps >= 2) / uniqExact(${group}), 2) AS conversion_rate_percentage`,- ])- .from(- clix.exp(`- (SELECT- ${group},- any(${clix.toStartOf('created_at', interval)}) as event_day,- ${breakdownSelects.length ? `${breakdownSelects.join(', ')},` : ''}- windowFunnel(${funnelWindowSeconds})(- toDateTime(created_at),- ${conditionA},- ${conditionB}- ) as steps- FROM ${TABLE_NAMES.events}- ${profileJoin}- ${groupJoin}- ${cohortJoinsSql}- WHERE project_id = '${projectId}'- AND events.name IN ('${eventA.name}', '${eventB.name}')- AND created_at BETWEEN toDateTime('${startDate}') AND toDateTime('${endDate}')- GROUP BY ${group}${breakdownExpressions.length ? `, ${breakdownExpressions.join(', ')}` : ''})- `),- )- .where('steps', '>', 0)- .groupBy(['event_day', ...breakdownGroupBy]);-- for (const order of ['event_day', ...breakdownGroupBy]) {- query.orderBy(order);- }-- const results = await query.execute();- return this.toSeries(results, breakdowns, limit).map(- (serie, serieIndex) => {- return {- ...serie,- data: serie.data.map((d, index) => ({- ...d,- timestamp: new Date(d.date).getTime(),- serieIndex,- index,- serie: omit(['data'], serie),- })),- };- },- );- }-- private toSeries(- data: {- event_day: string;- total_first: number;- conversions: number;- conversion_rate_percentage: number;- [key: string]: string | number;- }[],- breakdowns: { name: string }[] = [],- limit: number | undefined = undefined,- ) {- if (!breakdowns.length) {- return [- {- id: 'conversion',- breakdowns: [],- data: data.map((d) => ({- date: d.event_day,- total: d.total_first,- conversions: d.conversions,- rate: d.conversion_rate_percentage,- })),- },- ];- }-- // Group by breakdown values- const series = data.reduce(- (acc, d) => {- if (limit && Object.keys(acc).length >= limit) {- return acc;- }-- const key =- breakdowns.map((b, index) => d[`b_${index}`]).join('|') ||- NOT_SET_VALUE;- if (!acc[key]) {- acc[key] = {- id: key,- breakdowns: breakdowns.map(- (b, index) => (d[`b_${index}`] || NOT_SET_VALUE) as string,- ),- data: [],- };- }- acc[key]!.data.push({- date: d.event_day,- total: d.total_first,- conversions: d.conversions,- rate: d.conversion_rate_percentage,- });- return acc;- },- {} as Record<- string,- {- id: string;- breakdowns: string[];- data: {- date: string;- total: number;- conversions: number;- rate: number;- }[];- }- >,- );-- return Object.values(series).map((serie, serieIndex) => ({- ...serie,- data: serie.data.map((item, dataIndex) => ({- ...item,- dataIndex,- serieIndex,- })),- }));- }-}--export const conversionService = new ConversionService(ch);+// The conversion service lives in @openpanel/core now (M7-004). Re-exported+// here for existing `@openpanel/db` importers (the barrel).+export { getConversion } from '@openpanel/core';diff --git a/packages/db/src/services/funnel-sql.test.ts b/packages/db/src/services/funnel-sql.test.tsdeleted file mode 100644index 5bd23f0ab..000000000--- a/packages/db/src/services/funnel-sql.test.ts+++ /dev/null@@ -1,419 +0,0 @@-/**- * SQL-syntax tests for the shared funnel query builder.- *- * Same strategy as chart-sql.test.ts: build the SQL string, then run- * `EXPLAIN <sql>` against the local ClickHouse instance. EXPLAIN parses and- * resolves columns without executing, so a breakdown expression referencing a- * join alias that was never added fails here as UNKNOWN_IDENTIFIER — which is- * exactly the class of bug that made funnel "View Users" return "No users- * found" for profile-property and cohort breakdowns.- *- * Requires `pnpm dock:up` (or any locally reachable CH at- * http://localhost:8123/openpanel). All tests auto-skip if CH is unreachable.- */-import type { IChartBreakdown, IReportInput } from '@openpanel/validation';-import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest';--const mockCohortFindMany = vi.hoisted(() => vi.fn());--vi.mock('../prisma-client', () => ({- db: { cohort: { findMany: mockCohortFindMany } },-}));--import { ch } from '../clickhouse/client';-import { funnelService } from './funnel.service';-import { onlyReportEvents } from './reports.service';--const PROJECT_ID = 'test-sql-validation';-const COHORT_ID = 'aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee';-const START = '2026-04-14 00:00:00';-const END = '2026-05-15 00:00:00';--type ISeriesItem = IReportInput['series'][number];--const event = (overrides: Partial<ISeriesItem> = {}): ISeriesItem =>- ({- id: 'A',- type: 'event',- name: 'screen_view',- segment: 'event',- filters: [],- ...overrides,- }) as ISeriesItem;--const breakdown = (name: string): IChartBreakdown => ({ id: name, name });--const SERIES = [event({ id: 'A' }), event({ id: 'B', name: 'sign_up' })];--let chReachable = false;--async function explain(sql: string): Promise<void> {- await ch.command({ query: `EXPLAIN ${sql}` });-}--/** Mirrors what getFunnelProfiles builds on top of the shared base. */-async function buildProfilesSql(breakdowns: IChartBreakdown[]) {- const { query } = await funnelService.buildFunnelBase({- projectId: PROJECT_ID,- startDate: START,- endDate: END,- series: SERIES,- breakdowns,- funnelWindow: 24,- timezone: 'UTC',- });- query.with('funnel', 'SELECT * FROM session_funnel WHERE level != 0');- query.select(['DISTINCT profile_id']).from('funnel');- query.where('level', '>=', 2);- return query.toSQL();-}--/** Mirrors what the funnel chart builds on top of the shared base. */-async function buildChartSql(breakdowns: IChartBreakdown[]) {- const { query, breakdowns: resolved } = await funnelService.buildFunnelBase({- projectId: PROJECT_ID,- startDate: START,- endDate: END,- series: SERIES,- breakdowns,- funnelWindow: 24,- timezone: 'UTC',- });- query.with('funnel', 'SELECT * FROM session_funnel WHERE level != 0');- query- .select([- 'level',- ...resolved.map((_, index) => `b_${index}`),- 'count() as count',- ])- .from('funnel')- .groupBy(['level', ...resolved.map((_, index) => `b_${index}`)]);- return query.toSQL();-}--beforeAll(async () => {- vi.spyOn(console, 'log').mockImplementation(() => {});- mockCohortFindMany.mockResolvedValue([- { id: COHORT_ID, name: 'Power users' },- ]);- try {- await ch.command({ query: 'SELECT 1' });- chReachable = true;- } catch {- chReachable = false;- }-});--afterAll(() => {- vi.restoreAllMocks();-});--// Reports as skipped rather than passed when CH is unreachable — a test that-// returns before asserting anything should not read as green.-const itCH = (name: string, fn: () => Promise<void>) =>- it(name, async (ctx) => {- if (!chReachable) {- ctx.skip('ClickHouse not reachable at CLICKHOUSE_URL');- }- await fn();- });--describe('funnel.service / buildFunnelBase — profile breakdowns', () => {- itCH('adds the profiles join for a profile.properties breakdown', async () => {- const sql = await buildProfilesSql([breakdown('profile.properties.plan')]);-- // The breakdown is narrowed to a scalar alias selected by the profile- // join, so both the join and the aliased column must exist.- expect(sql).toContain("properties['plan'] as `profile.properties.plan`");- expect(sql).toContain('`profile.properties.plan`');- expect(sql).toMatch(/as profile/);- expect(sql).toContain('profile.id = events.profile_id');- await explain(sql);- });-- itCH('selects the properties column, not a bare `properties` field', async () => {- const sql = await buildProfilesSql([breakdown('profile.properties.plan')]);- expect(sql).toMatch(/SELECT [^)]*properties[^)]*FROM .*profiles/);- await explain(sql);- });-- itCH('adds the profiles join for a top-level profile breakdown', async () => {- const sql = await buildProfilesSql([breakdown('profile.email')]);- expect(sql).toContain('profile.id = events.profile_id');- await explain(sql);- });-- itCH('adds the join for a profile filter with no profile breakdown', async () => {- const { query } = await funnelService.buildFunnelBase({- projectId: PROJECT_ID,- startDate: START,- endDate: END,- series: [- event({- filters: [- { id: 'f', name: 'profile.email', operator: 'is', value: ['[email protected]'] },- ],- }),- event({ id: 'B', name: 'sign_up' }),- ],- funnelWindow: 24,- timezone: 'UTC',- });- query.with('funnel', 'SELECT * FROM session_funnel WHERE level != 0');- query.select(['DISTINCT profile_id']).from('funnel');- await explain(query.toSQL());- });-});--describe('funnel.service / buildFunnelBase — cohort breakdowns', () => {- itCH('adds the cohort join for a cohort breakdown', async () => {- const sql = await buildProfilesSql([breakdown(`cohort:${COHORT_ID}`)]);-- // The breakdown renders `cohort_<id>.profile_id`, so its join must exist.- const alias = `cohort_${COHORT_ID.replace(/-/g, '_')}`;- expect(sql).toContain(`${alias}.profile_id`);- expect(sql).toContain(`AS ${alias}`);- expect(sql).toContain('Power users');- await explain(sql);- });-});--describe('funnel.service / buildFunnelBase — unknown breakdowns', () => {- itCH('drops the all-cohorts breakdown, which the funnel cannot render', async () => {- // Bare `cohort` is the chart's all-cohorts breakdown. The funnel has no- // equivalent, and inlining it produced `cohort as b_0 FROM events` —- // UNKNOWN_IDENTIFIER for the chart and the profile list alike.- const sql = await buildProfilesSql([breakdown('cohort')]);- expect(sql).not.toMatch(/\bcohort as b_0\b/);- await explain(sql);- });-- itCH('drops an unrecognised breakdown name entirely', async () => {- const sql = await buildProfilesSql([breakdown('not_a_real_column')]);- expect(sql).not.toContain('not_a_real_column');- await explain(sql);- });-- itCH('drops unsupported breakdowns identically for chart and profiles', async () => {- const breakdowns = [- breakdown('cohort'),- breakdown('profile.properties.plan'),- ];- const chartSql = await buildChartSql(breakdowns);- const profilesSql = await buildProfilesSql(breakdowns);-- // Both sides must resolve the same breakdown to the same b_N index,- // otherwise the breakdownValues filter targets the wrong column.- expect(chartSql).toContain(- 'argMinIf(`profile.properties.plan`, created_at,',- );- expect(profilesSql).toContain(- 'argMinIf(`profile.properties.plan`, created_at,',- );- await explain(chartSql);- await explain(profilesSql);- });-});--describe('funnel.service / buildFunnelCte — step pre-filter', () => {- // Only rows matching at least one step can advance windowFunnel, so the- // funnel CTE must not feed rows that share a step's event name but fail- // its filters into the aggregation. The step conditions therefore appear- // twice: inside windowFunnel and as a row-level pre-filter.- // Each step's COMPLETE condition must appear exactly twice: once inside- // windowFunnel and once in the row-level pre-filter. Counting the full- // condition (not a substring) catches a regression that drops a step from- // either clause.- const countOccurrences = (sql: string, needle: string) =>- sql.split(needle).length - 1;-- it('filters the scan to rows matching at least one step condition', async () => {- const sql = await buildChartSql([]);- expect(sql).toContain(- "((events.name = 'screen_view') OR (events.name = 'sign_up'))",- );- for (const condition of funnelService.getFunnelConditions(- // The same checked narrowing buildFunnelBase applies internally.- onlyReportEvents(SERIES),- PROJECT_ID,- )) {- expect(countOccurrences(sql, condition)).toBe(2);- }- });-- it('includes each step\'s own filters in the pre-filter', async () => {- const series = [- event({- filters: [- { id: 'f', name: 'path', operator: 'is', value: ['/pricing'] },- ],- }),- event({ id: 'B', name: 'sign_up' }),- ];- const { query } = await funnelService.buildFunnelBase({- projectId: PROJECT_ID,- startDate: START,- endDate: END,- series,- funnelWindow: 24,- timezone: 'UTC',- });- query.with('funnel', 'SELECT * FROM session_funnel WHERE level != 0');- query.select(['DISTINCT profile_id']).from('funnel');- const sql = query.toSQL();- // The full filtered condition (path AND name) must gate the scan, not- // only the windowFunnel arm — and so must the unfiltered second step.- for (const condition of funnelService.getFunnelConditions(- onlyReportEvents(series),- PROJECT_ID,- )) {- expect(countOccurrences(sql, condition)).toBe(2);- }- });-- itCH('pre-filtered funnel SQL still parses and resolves', async () => {- const sql = await buildProfilesSql([breakdown('profile.properties.plan')]);- await explain(sql);- });-});--describe('funnel.service / buildFunnelBase — breakdown attribution', () => {- // The breakdown value must be read at the user's FIRST funnel step, not- // used as a windowFunnel GROUP BY key. Per-row grouping splits a user's- // steps across buckets whenever the property isn't identical on every- // step, so the sequence never connects and downstream steps show 0.- it('attributes event-property breakdowns at the entry step', async () => {- const sql = await buildChartSql([breakdown('properties.experiment')]);- expect(sql).toContain("argMinIf(properties['experiment'], created_at,");- // The windowFunnel aggregation groups by the primary key only; b_0 is an- // aggregate, not a grouping key. (The outer chart GROUP BY level, b_0 is- // unaffected.)- expect(sql).toMatch(/GROUP BY session_id\b/);- expect(sql).not.toMatch(/GROUP BY session_id, b_0/);- });-- itCH('entry-step attribution parses and resolves', async () => {- const sql = await buildChartSql([breakdown('properties.experiment')]);- await explain(sql);- });-- it('keeps per-row grouping for group breakdowns (fan-out is intended)', async () => {- // A user in three groups should appear in all three funnels, so group.*- // breakdowns keep the ARRAY JOIN fan-out and per-row GROUP BY.- const sql = await buildChartSql([breakdown('group.plan')]);- expect(sql).not.toContain('argMinIf');- expect(sql).toMatch(/GROUP BY session_id, b_0/);- });-- itCH('group-breakdown SQL parses and resolves', async () => {- const sql = await buildChartSql([breakdown('group.plan')]);- await explain(sql);- });-});--describe('funnel.service / buildFunnelCte — windowFunnel ordering', () => {- afterAll(() => {- vi.unstubAllEnvs();- });-- it('defaults to strict_increase (unchanged behavior)', async () => {- // Stub to '' rather than unstubbing: unstubAllEnvs restores the HOST- // environment, so a dev shell with the variable set would flip this test.- vi.stubEnv('FUNNEL_NON_STRICT_ORDERING', '');- const sql = await buildChartSql([]);- expect(sql).toContain("'strict_increase'");- });-- it('drops strict_increase when FUNNEL_NON_STRICT_ORDERING is set', async () => {- vi.stubEnv('FUNNEL_NON_STRICT_ORDERING', '1');- const sql = await buildChartSql([]);- expect(sql).not.toContain('strict_increase');- expect(sql).toMatch(/windowFunnel\(\d+\)\(/);- });-- itCH('non-strict funnel SQL parses and resolves', async () => {- vi.stubEnv('FUNNEL_NON_STRICT_ORDERING', 'true');- const sql = await buildChartSql([]);- expect(sql).not.toContain('strict_increase');- await explain(sql);- });-});--describe('funnel.service / buildFunnelBase — group breakdowns', () => {- itCH('adds the group array join for a group breakdown', async () => {- const sql = await buildProfilesSql([breakdown('group.plan')]);- expect(sql).toContain('ARRAY JOIN groups AS _group_id');- expect(sql).toContain('LEFT ANY JOIN _g ON _g.id = _group_id');- await explain(sql);- });-});--describe('funnel.service / profile-property narrowing', () => {- const seriesWithProfileFilter = [- event({- filters: [- {- id: 'f1',- name: 'profile.properties.plan',- operator: 'is' as const,- value: ['pro'],- },- ],- }),- event({ id: 'B', name: 'sign_up' }),- ];-- it('joins scalar columns instead of the whole properties Map', async () => {- const { query } = await funnelService.buildFunnelBase({- projectId: PROJECT_ID,- startDate: START,- endDate: END,- series: seriesWithProfileFilter,- breakdowns: [breakdown('profile.properties.experiment')],- funnelWindow: 24,- timezone: 'UTC',- });- query.with('funnel', 'SELECT * FROM session_funnel WHERE level != 0');- query.select(['DISTINCT profile_id']).from('funnel');- const sql = query.toSQL();-- expect(sql).toContain("properties['plan'] as `profile.properties.plan`");- expect(sql).toContain(- "properties['experiment'] as `profile.properties.experiment`",- );- expect(sql).not.toContain('properties as "profile.properties"');- // Conditions (windowFunnel + pre-filter) and the breakdown expression are- // rewritten to the scalar aliases.- expect(sql).not.toContain("profile.properties['plan']");- expect(sql).not.toContain("profile.properties['experiment']");- });-- it('leaves funnels without profile-property refs untouched', async () => {- const sql = await buildChartSql([]);- expect(sql).not.toContain('`profile.properties.');- });-- itCH('narrowed funnel SQL parses and resolves', async () => {- const { query, breakdowns: resolved } = await funnelService.buildFunnelBase(- {- projectId: PROJECT_ID,- startDate: START,- endDate: END,- series: seriesWithProfileFilter,- breakdowns: [breakdown('profile.properties.experiment')],- funnelWindow: 24,- timezone: 'UTC',- },- );- query.with('funnel', 'SELECT * FROM session_funnel WHERE level != 0');- query- .select([- 'level',- ...resolved.map((_, index) => `b_${index}`),- 'count() as count',- ])- .from('funnel')- .groupBy(['level', ...resolved.map((_, index) => `b_${index}`)]);- await explain(query.toSQL());- });-});diff --git a/packages/db/src/services/funnel.service.ts b/packages/db/src/services/funnel.service.tsindex 381ca6a6e..c8091372c 100644--- a/packages/db/src/services/funnel.service.ts+++ b/packages/db/src/services/funnel.service.ts@@ -1,670 +1,14 @@-import { ifNaN } from '@openpanel/common';-import type {- IChartBreakdown,- IChartEvent,- IReportInput,-} from '@openpanel/validation';-import { last, reverse, uniq } from 'ramda';-import sqlstring from 'sqlstring';-import { ch } from '../clickhouse/client';-import { TABLE_NAMES } from '../clickhouse/client';-import { clix } from '../clickhouse/query-builder';-import { createSqlBuilder } from '../sql-builder';-import {- buildInlineCohortJoin,- collectBreakdownCohortIds,- collectProfilePropertyKeys,- extractCohortId,- fetchCohortsMetadata,- getEventFiltersWhereClause,- getSelectPropertyKey,- isAllCohortsBreakdown,- isKnownEventField,- profilePropertiesCteSelect,- rewriteProfilePropertyRefs,-} from './chart.service';-import { mergeGlobalFilters, onlyReportEvents } from './reports.service';--/** Display label for null/empty breakdown values (e.g. property not set). */-export const EMPTY_BREAKDOWN_LABEL = 'Not set';--function normalizeBreakdownValue(value: unknown): string {- if (value == null || value === '') {- return EMPTY_BREAKDOWN_LABEL;- }- const s = String(value).trim();- return s === '' ? EMPTY_BREAKDOWN_LABEL : s;-}--export class FunnelService {- constructor(private client: typeof ch) {}-- /**- * Returns the grouping strategy for the funnel.- * Determines whether windowFunnel is computed per session_id or profile_id.- */- getFunnelGroup(group?: string): 'profile_id' | 'session_id' {- return group === 'profile_id' ? 'profile_id' : 'session_id';- }-- getFunnelConditions(events: IChartEvent[] = [], projectId?: string): string[] {- return events.map((event) => {- const { sb, getWhere } = createSqlBuilder();- // Qualify with 'events' so event-level `properties[...]` becomes- // `events.properties[...]` — required because the funnel CTE may join- // the profiles table (which also exposes a `properties` column).- // Without the qualifier ClickHouse fails with "ambiguous identifier- // 'properties'" whenever a step filters on properties.X while another- // step filters on profile.properties.Y.- sb.where = getEventFiltersWhereClause(event.filters, projectId, 'events');- sb.where.name = `events.name = ${sqlstring.escape(event.name)}`;- return getWhere().replace('WHERE ', '');- });- }-- /**- * Builds the funnel CTE.- * - When group === 'session_id': windowFunnel is computed per session_id.- * profile_id is resolved via argMax to handle identity changes mid-session.- * - When group === 'profile_id': windowFunnel is computed directly per profile_id.- * This correctly handles cross-session funnel completions.- */- buildFunnelCte({- projectId,- startDate,- endDate,- eventSeries,- funnelWindowMilliseconds,- timezone,- additionalSelects = [],- additionalGroupBy = [],- group = 'session_id',- profilePropertyKeys = [],- }: {- projectId: string;- startDate: string;- endDate: string;- eventSeries: IChartEvent[];- funnelWindowMilliseconds: number;- timezone: string;- additionalSelects?: string[];- additionalGroupBy?: string[];- group?: 'session_id' | 'profile_id';- profilePropertyKeys?: string[];- }) {- const funnels = this.getFunnelConditions(eventSeries, projectId).map((c) =>- rewriteProfilePropertyRefs(c, profilePropertyKeys),- );- const primaryKey = group === 'profile_id' ? 'profile_id' : 'session_id';-- // windowFunnel's 'strict_increase' mode requires every step's timestamp- // to be strictly greater than the previous step's, so same-timestamp- // sequences (server-side senders, batched SDKs, imported data with- // coarse timestamps) never connect. Mixpanel/Amplitude count those as- // ordered, so deployments migrating from them can opt into the default- // (>=) mode; strict stays the default here.- const nonStrictOrdering =- process.env.FUNNEL_NON_STRICT_ORDERING === '1' ||- process.env.FUNNEL_NON_STRICT_ORDERING === 'true';- const windowFunnelMode = nonStrictOrdering ? '' : ", 'strict_increase'";-- return clix(this.client, timezone)- .select([- primaryKey,- `windowFunnel(${funnelWindowMilliseconds}${windowFunnelMode})(toUInt64(toUnixTimestamp64Milli(created_at)), ${funnels.join(', ')}) AS level`,- ...(group === 'session_id'- ? ['argMax(profile_id, created_at) AS profile_id']- : []),- ...additionalSelects,- ])- .from(TABLE_NAMES.events, false)- .where('project_id', '=', projectId)- .where('created_at', 'BETWEEN', [- clix.datetime(startDate, 'toDateTime'),- clix.datetime(endDate, 'toDateTime'),- ])- .where(- 'events.name',- 'IN',- eventSeries.map((e) => e.name),- )- // Only rows matching at least one step can advance windowFunnel, so- // rows that share a step's event name but fail its filters are dead- // weight — with filtered steps (e.g. screen_view + a path filter) they- // can be the vast majority of what the name IN(...) lets through.- // Dropping them here shrinks the aggregation input; windowFunnel- // ignores non-matching rows either way, so levels are unchanged.- .rawWhere(`(${funnels.map((f) => `(${f})`).join(' OR ')})`)- .groupBy([primaryKey, ...additionalGroupBy]);- }-- buildSessionsCte({- projectId,- startDate,- endDate,- timezone,- }: {- projectId: string;- startDate: string;- endDate: string;- timezone: string;- }) {- return clix(this.client, timezone)- .select(['profile_id as pid', 'id as sid'])- .from(TABLE_NAMES.sessions)- .where('project_id', '=', projectId)- .where('created_at', 'BETWEEN', [- clix.datetime(startDate, 'toDateTime'),- clix.datetime(endDate, 'toDateTime'),- ]);- }-- private fillFunnel(- funnel: { level: number; count: number }[],- steps: number,- ) {- const filled = Array.from({ length: steps }, (_, index) => {- const level = index + 1;- const matchingResult = funnel.find((res) => res.level === level);- return {- level,- count: matchingResult ? matchingResult.count : 0,- };- });-- // Accumulate counts from top to bottom of the funnel- for (let i = filled.length - 1; i >= 0; i--) {- const step = filled[i];- const prevStep = filled[i + 1];- // If there's a previous step, add the count to the current step- if (step && prevStep) {- step.count += prevStep.count;- }- }- return filled.reverse();- }-- toSeries(- funnel: { level: number; count: number; [key: string]: any }[],- breakdowns: { name: string }[] = [],- limit: number | undefined = undefined,- ) {- if (!breakdowns.length) {- return [- funnel.map((f) => ({- level: f.level,- count: f.count,- id: 'none',- breakdowns: [],- })),- ];- }-- // Group by breakdown values (normalize empty/null to "Not set")- const series = funnel.reduce(- (acc, f) => {- if (limit && Object.keys(acc).length >= limit) {- return acc;- }-- const key = breakdowns- .map((b, index) => normalizeBreakdownValue(f[`b_${index}`]))- .join('|');- if (!acc[key]) {- acc[key] = [];- }- acc[key]!.push({- id: key,- breakdowns: breakdowns.map((b, index) =>- normalizeBreakdownValue(f[`b_${index}`]),- ),- level: f.level,- count: f.count,- });- return acc;- },- {} as Record<- string,- {- id: string;- breakdowns: string[];- level: number;- count: number;- }[]- >,- );-- return Object.values(series);- }-- getProfileFilters(events: IChartEvent[]) {- return events.flatMap((e) =>- e.filters- ?.filter((f) => f.name.startsWith('profile.'))- .map((f) => f.name.replace('profile.', '')),- );- }-- /**- * Builds everything the funnel chart and the funnel profile list share: the- * normalized event series and breakdowns, the `session_funnel` CTE with all- * of its joins wired up, and the outer query those CTEs are registered on.- *- * This exists because the two used to be written out twice and drifted. A- * breakdown expression only works if the join it references was added, and- * the joins depend on the breakdowns — so building the selects in one place- * and the joins in another is exactly the bug waiting to happen. Callers add- * their own `funnel` CTE and final projection on top.- */- async buildFunnelBase({- projectId,- startDate,- endDate,- series,- globalFilters,- breakdowns: initialBreakdowns = [],- funnelWindow = 24,- funnelGroup,- timezone,- }: {- projectId: string;- startDate: string;- endDate: string;- series: IReportInput['series'];- globalFilters?: IReportInput['globalFilters'];- breakdowns?: IChartBreakdown[];- funnelWindow?: number;- funnelGroup?: string;- timezone: string;- }) {- // Drop breakdowns that don't resolve to a known events column, properties- // path, profile path, group path, or specific cohort. The funnel CTE- // inlines each breakdown's name directly via getSelectPropertyKey, so- // anything that doesn't resolve leaks into the SQL verbatim.- //- // `isKnownEventField` accepts the bare `cohort` breakdown because the- // chart's all-cohorts feature uses it, but the funnel has no equivalent —- // it renders as `cohort as b_0 FROM events`, which fails with- // UNKNOWN_IDENTIFIER for the chart and the profile list alike. Exclude it- // explicitly rather than relying on the generic check.- const breakdowns = initialBreakdowns.filter(- (b) => isKnownEventField(b.name) && !isAllCohortsBreakdown(b.name),- );-- const eventSeries = onlyReportEvents(- mergeGlobalFilters(series, globalFilters),- );-- if (eventSeries.length === 0) {- throw new Error('events are required');- }-- const funnelWindowMilliseconds = funnelWindow * 3600 * 1000;- const group = this.getFunnelGroup(funnelGroup);-- const profileFilters = this.getProfileFilters(eventSeries);- const anyFilterOnProfile = profileFilters.length > 0;- const profileBreakdowns = breakdowns.filter((b) =>- b.name.startsWith('profile.'),- );- const anyFilterOnGroup = eventSeries.some((e) =>- e.filters?.some((f) => f.name.startsWith('group.')),- );- const anyBreakdownOnGroup = breakdowns.some((b) =>- b.name.startsWith('group.'),- );- const needsGroupArrayJoin =- anyFilterOnGroup || anyBreakdownOnGroup || funnelGroup === 'group';-- const cohortIds = collectBreakdownCohortIds(breakdowns);- const cohortMetadata = await fetchCohortsMetadata(cohortIds);-- // Attribute each breakdown to its value at the user's FIRST funnel step,- // as a per-group aggregate (argMinIf) — not by adding it to the- // windowFunnel GROUP BY. Grouping the sequence by a per-row value splits- // a user's steps across buckets whenever the value isn't identical on- // every step (e.g. an experiment tag set on the entry event but absent- // on the conversion event): the later step lands in a separate bucket,- // the windowFunnel sequence never connects, and downstream steps show 0.- // Reading the entry-step value keeps each sequence intact in one bucket- // and matches standard funnel-breakdown semantics (segment by entry- // attribute).- //- // `group.*` breakdowns are the exception: their ARRAY JOIN fans each- // event out per group, and grouping by the group value is intentional —- // a user in three groups should appear in all three funnels. Those keep- // the per-row GROUP BY.- // Join only the referenced profile-property keys as scalar columns- // instead of every profile's whole properties Map — the join hash was- // carrying ~1KB of Map per profile and OOMing at scale (same narrowing- // as the chart profile CTE). Funnel conditions and breakdown expressions- // are rewritten to the scalar aliases below.- const profileProps = collectProfilePropertyKeys([- ...eventSeries.flatMap((e) => e.filters ?? []),- ...breakdowns,- ]);-- const firstStepCondition = rewriteProfilePropertyRefs(- this.getFunnelConditions(eventSeries, projectId)[0]!,- profileProps.keys,- );- const breakdownSelects = breakdowns.map((b, index) => {- const bId = extractCohortId(b.name);- const bName = bId ? cohortMetadata.get(bId)?.name : undefined;- const expr = rewriteProfilePropertyRefs(- getSelectPropertyKey(b.name, projectId, bId ?? undefined, bName),- profileProps.keys,- );- if (b.name.startsWith('group.')) {- return `${expr} as b_${index}`;- }- return `argMinIf(${expr}, created_at, ${firstStepCondition}) as b_${index}`;- });- const breakdownGroupBy = breakdowns.flatMap((b, index) =>- b.name.startsWith('group.') ? [`b_${index}`] : [],- );-- const funnelCte = this.buildFunnelCte({- projectId,- startDate,- endDate,- eventSeries,- funnelWindowMilliseconds,- timezone,- additionalSelects: breakdownSelects,- additionalGroupBy: breakdownGroupBy,- group,- profilePropertyKeys: profileProps.keys,- });-- // The profile join has to cover breakdowns as well as filters — a- // `profile.*` breakdown renders `profile.properties[...]` into the select,- // so the alias must exist in scope even when no filter touches profiles.- if (anyFilterOnProfile || profileBreakdowns.length > 0) {- // Scalar columns (email etc.) are selected as-is; the properties Map is- // narrowed to the referenced keys via profilePropertiesCteSelect.- const profileFields = new Set<string>(['id']);- for (const f of profileFilters) {- const fieldName = f.split('.')[0]!;- if (fieldName !== 'properties') {- profileFields.add(fieldName);- }- }- for (const b of profileBreakdowns) {- const fieldName = b.name.replace('profile.', '').split('.')[0];- if (- [- 'email',- 'first_name',- 'last_name',- 'created_at',- 'last_seen_at',- ].includes(fieldName!)- ) {- profileFields.add(fieldName!);- }- }- const selectColumns = Array.from(profileFields);- const referencesProperties =- profileFilters.some((f) => f.startsWith('properties')) ||- profileBreakdowns.some((b) =>- b.name.startsWith('profile.properties'),- );- if (referencesProperties) {- selectColumns.push(- profilePropertiesCteSelect(- profileProps.keys,- profileProps.needsFullMap,- ),- );- }- funnelCte.leftJoin(- `(SELECT ${selectColumns.join(', ')} FROM ${TABLE_NAMES.profiles} FINAL- WHERE project_id = ${sqlstring.escape(projectId)}) as profile`,- 'profile.id = events.profile_id',- );- }-- if (needsGroupArrayJoin) {- funnelCte.rawJoin('ARRAY JOIN groups AS _group_id');- funnelCte.rawJoin('LEFT ANY JOIN _g ON _g.id = _group_id');- }-- // A cohort breakdown renders `cohort_<id>.profile_id`, so every cohort- // referenced by a breakdown needs its join.- for (const cohortId of cohortIds) {- funnelCte.rawJoin(buildInlineCohortJoin(cohortId, projectId, 'events'));- }-- const query = clix(this.client, timezone);-- if (needsGroupArrayJoin) {- query.with(- '_g',- `SELECT id, name, type, properties FROM ${TABLE_NAMES.groups} FINAL WHERE project_id = ${sqlstring.escape(projectId)}`,- );- }-- query.with('session_funnel', funnelCte);-- return { query, eventSeries, breakdowns, group };- }-- async getFunnel({- projectId,- startDate,- endDate,- series,- globalFilters,- options,- breakdowns: initialBreakdowns = [],- limit,- timezone = 'UTC',- }: IReportInput & { timezone: string; events?: IChartEvent[] }) {- if (!startDate || !endDate) {- throw new Error('startDate and endDate are required');- }-- const funnelOptions = options?.type === 'funnel' ? options : undefined;-- const {- query: funnelQuery,- eventSeries,- breakdowns,- } = await this.buildFunnelBase({- projectId,- startDate,- endDate,- series,- globalFilters,- breakdowns: initialBreakdowns,- funnelWindow: funnelOptions?.funnelWindow,- funnelGroup: funnelOptions?.funnelGroup,- timezone,- });-- // windowFunnel is computed per the primary key (profile_id or session_id),- // so we just filter out level=0 rows — no re-aggregation needed.- funnelQuery.with(- 'funnel',- 'SELECT * FROM session_funnel WHERE level != 0',- );-- funnelQuery- .select<{- level: number;- count: number;- [key: string]: any;- }>([- 'level',- ...breakdowns.map((b, index) => `b_${index}`),- 'count() as count',- ])- .from('funnel')- .groupBy(['level', ...breakdowns.map((b, index) => `b_${index}`)])- .orderBy('level', 'DESC');-- const funnelData = await funnelQuery.execute();- const funnelSeries = this.toSeries(funnelData, breakdowns, limit);-- return funnelSeries- .map((data) => {- const maxLevel = eventSeries.length;- const filledFunnelRes = this.fillFunnel(- data.map((d) => ({ level: d.level, count: d.count })),- maxLevel,- );-- const totalSessions = last(filledFunnelRes)?.count ?? 0;- const steps = reverse(filledFunnelRes)- .reduce(- (acc, item, index, list) => {- const prev = list[index - 1] ?? { count: totalSessions };- const next = list[index + 1];- const event = eventSeries[item.level - 1]!;- return [- ...acc,- {- event: {- ...event,- displayName: event.displayName || event.name,- },- count: item.count,- percent: (item.count / totalSessions) * 100,- dropoffCount: next ? item.count - next.count : null,- dropoffPercent: next- ? ((item.count - next.count) / item.count) * 100- : null,- previousCount: prev.count,- nextCount: next?.count ?? null,- },- ];- },- [] as {- event: IChartEvent & { displayName: string };- count: number;- percent: number;- dropoffCount: number | null;- dropoffPercent: number | null;- previousCount: number;- nextCount: number | null;- }[],- )- .map((step, index, list) => {- return {- ...step,- percent: ifNaN(step.percent, 0),- dropoffPercent: ifNaN(step.dropoffPercent, 0),- isHighestDropoff: (() => {- // Skip if current step has no dropoff- if (!step?.dropoffCount) return false;-- // Get maximum dropoff count, excluding 0s- const maxDropoff = Math.max(- ...list- .map((s) => s.dropoffCount || 0)- .filter((count) => count > 0),- );-- // Check if this is the first step with the highest dropoff- return (- step.dropoffCount === maxDropoff &&- list.findIndex((s) => s.dropoffCount === maxDropoff) === index- );- })(),- };- });-- return {- id: data[0]?.id ?? 'none',- breakdowns: data[0]?.breakdowns ?? [],- steps,- totalSessions,- lastStep: last(steps)!,- mostDropoffsStep: steps.find((step) => step.isHighestDropoff)!,- };- })- .sort((a, b) => {- const aTotal = a.steps.reduce((acc, step) => acc + step.count, 0);- const bTotal = b.steps.reduce((acc, step) => acc + step.count, 0);- return bTotal - aTotal;- });- }-}--export const funnelService = new FunnelService(ch);--import { getSettingsForProject } from './organization.service';--export async function getFunnelCore(input: {- projectId: string;- startDate: string;- endDate: string;- steps: string[];- windowHours?: number;- groupBy?: 'session_id' | 'profile_id';-}) {- const { timezone } = await getSettingsForProject(input.projectId);- const eventSeries = input.steps.map((name, index) => ({- id: String(index + 1),- type: 'event' as const,- name,- displayName: name,- segment: 'user' as const,- filters: [],- }));-- const result = await funnelService.getFunnel({- projectId: input.projectId,- startDate: input.startDate,- endDate: input.endDate,- series: eventSeries,- breakdowns: [],- chartType: 'funnel',- interval: 'day',- range: 'custom',- previous: false,- metric: 'sum',- options: {- type: 'funnel',- funnelWindow: input.windowHours ?? 24,- funnelGroup: input.groupBy ?? 'session_id',- },- timezone,- });-- const primarySeries = result[0];- if (!primarySeries) {- return {- steps: [],- totalUsers: 0,- completedUsers: 0,- overallConversionRate: 0,- };- }-- const steps = primarySeries.steps.map((step, index) => ({- step: index + 1,- eventName: step.event.displayName || step.event.name,- users: step.count,- conversionRateFromStart: Math.round(step.percent * 100) / 100,- dropoffPercent:- step.dropoffPercent != null- ? Math.round(step.dropoffPercent * 100) / 100- : null,- isHighestDropoff: step.isHighestDropoff,- }));-- const totalUsers = steps[0]?.users ?? 0;- const completedUsers = steps[steps.length - 1]?.users ?? 0;-- return {- steps,- totalUsers,- completedUsers,- overallConversionRate:- totalUsers > 0- ? Math.round((completedUsers / totalUsers) * 10000) / 100- : 0,- };-}+// The funnel service lives in @openpanel/core now (M7-004). Re-exported here+// for existing `@openpanel/db` importers (the barrel, reports.service's+// `getReportDataCore`, apps/api's insights controller, packages/core's mcp and+// assistant tools).+export {+ buildFunnelBase,+ buildSessionsCte,+ EMPTY_BREAKDOWN_LABEL,+ getFunnel,+ getFunnelCore,+ getFunnelGroup,+ getFunnelProfileIds,+ toSeries as toFunnelSeries,+} from '@openpanel/core';diff --git a/packages/db/src/services/reports.service.ts b/packages/db/src/services/reports.service.tsindex bd25ddc94..de28ecfad 100644--- a/packages/db/src/services/reports.service.ts+++ b/packages/db/src/services/reports.service.ts@@ -148,7 +148,7 @@ export async function getReportById(id: string) {import { AggregateChartEngine, ChartEngine } from '../engine';import { getDashboardById } from './dashboard.service';import { getChartStartEndDate } from './date.service';-import { funnelService } from './funnel.service';+import { getFunnel } from './funnel.service';import { getSettingsForProject } from './organization.service';export async function listReportsCore(input: {@@ -207,7 +207,7 @@ export async function getReportDataCore(input: {};if (report.chartType === 'funnel') {- const result = await funnelService.getFunnel(chartInput);+ const result = await getFunnel(chartInput);return { ...meta, data: result };}diff --git a/packages/db/src/services/retention.service.ts b/packages/db/src/services/retention.service.tsindex 5def2486b..3edcd3ce2 100644--- a/packages/db/src/services/retention.service.ts+++ b/packages/db/src/services/retention.service.ts@@ -1,504 +1,23 @@-import { DateTime, round } from '@openpanel/common';-import type { IChartEventFilter } from '@openpanel/validation';-import { range } from 'ramda';-import sqlstring from 'sqlstring';--import { TABLE_NAMES, chQuery } from '../clickhouse/client';-import { getEventFiltersWhereClause } from './chart.service';--type IGetWeekRetentionInput = {- projectId: string;-};--// Week-over-week retention graph: for each week, how many active users were-// also active the following week.-//-// Instead of self-joining the raw events table (O(events²) per profile), we-// first collapse events to one row per (profile, week) in `weekly_active`,-// then self-join that much smaller set to its next week. Same result, a-// fraction of the work on high-volume projects.-export function getRetentionSeries({ projectId }: IGetWeekRetentionInput) {- const sql = `- WITH weekly_active AS (- SELECT- profile_id,- toStartOfWeek(created_at) AS week- FROM ${TABLE_NAMES.events}- WHERE project_id = ${sqlstring.escape(projectId)}- AND profile_id != device_id- GROUP BY profile_id, week- )- SELECT- cur.week AS date,- countDistinct(cur.profile_id) AS active_users,- countDistinct(nxt.profile_id) AS retained_users,- (100 * (countDistinct(nxt.profile_id) / CAST(countDistinct(cur.profile_id), 'Float64'))) AS retention- FROM weekly_active AS cur- LEFT JOIN weekly_active AS nxt- ON cur.profile_id = nxt.profile_id- AND nxt.week = cur.week + toIntervalWeek(1)- GROUP BY date- ORDER BY date ASC- -- Unmatched LEFT JOIN rows must be NULL (not the empty-string default),- -- otherwise countDistinct(nxt.profile_id) counts '' as a retained user.- SETTINGS join_use_nulls = 1`;-- return chQuery<{- date: string;- active_users: number;- retained_users: number;- retention: number;- }>(sql);-}--// https://medium.com/@andre_bodro/how-to-fast-calculating-mau-in-clickhouse-fd793559b229-// Rolling active users-export type IServiceRetentionRollingActiveUsers = {- date: string;- users: number;-};-export function getRollingActiveUsers({- projectId,- days,-}: IGetWeekRetentionInput & { days: number }) {- const sql = `- SELECT- date,- uniqMerge(profile_id) AS users- FROM- (- SELECT- date + n AS date,- profile_id,- project_id- FROM- (- SELECT *- FROM ${TABLE_NAMES.dau_mv}- WHERE project_id = ${sqlstring.escape(projectId)}- )- ARRAY JOIN range(${days}) AS n- )- WHERE project_id = ${sqlstring.escape(projectId)}- GROUP BY date`;-- return chQuery<IServiceRetentionRollingActiveUsers>(sql);-}--export function getRetentionLastSeenSeries({- projectId,-}: IGetWeekRetentionInput) {- const sql = `- WITH last_active AS (- SELECT- max(created_at) AS last_active,- profile_id- FROM ${TABLE_NAMES.events}- WHERE (project_id = ${sqlstring.escape(projectId)}) AND (device_id != profile_id)- GROUP BY profile_id- )- SELECT- dateDiff('day', last_active, today()) AS days,- countDistinct(profile_id) AS users- FROM last_active- GROUP BY days- ORDER BY days ASC`;-- return chQuery<{- days: number;- users: number;- }>(sql);-}--export async function getRollingActiveUsersCore(input: {- projectId: string;- days: number;-}) {- const data = await getRollingActiveUsers(input);- return {- window_days: input.days,- label:- input.days === 1- ? 'DAU'- : input.days === 7- ? 'WAU'- : input.days === 30- ? 'MAU'- : `${input.days}d active`,- series: data,- };-}--export async function getWeeklyRetentionSeriesCore(projectId: string) {- return getRetentionSeries({ projectId });-}--// Weekly active-user retention cohort over the last 12 weeks, computed by the-// unified getRetentionCohort engine (replaces the legacy, broken-// getRetentionCohortTable). firstEvent/secondEvent are omitted so any-// identified activity counts toward the cohort.-export async function getRetentionCohortCore(projectId: string) {- const end = DateTime.now();- const start = end.minus({ weeks: 12 });- return getRetentionCohort({- projectId,- interval: 'week',- startDate: start.toFormat('yyyy-MM-dd HH:mm:ss'),- endDate: end.toFormat('yyyy-MM-dd HH:mm:ss'),- });-}--export async function getEngagementCore(projectId: string) {- const raw = await getRetentionLastSeenSeries({ projectId });-- let active_0_7 = 0;- let active_8_14 = 0;- let active_15_30 = 0;- let active_31_60 = 0;- let churned_60_plus = 0;-- for (const row of raw) {- if (row.days <= 7) active_0_7 += row.users;- else if (row.days <= 14) active_8_14 += row.users;- else if (row.days <= 30) active_15_30 += row.users;- else if (row.days <= 60) active_31_60 += row.users;- else churned_60_plus += row.users;- }-- const total =- active_0_7 + active_8_14 + active_15_30 + active_31_60 + churned_60_plus;-- return {- summary: {- total_identified_users: total,- active_last_7_days: active_0_7,- active_8_to_14_days: active_8_14,- active_15_to_30_days: active_15_30,- inactive_31_to_60_days: active_31_60,- churned_60_plus_days: churned_60_plus,- },- distribution: raw,- };-}--// ----------------------------------------------------------------------------// Cohort retention matrix-//-// This is the single source of truth for retention cohorts. It powers the-// dashboard retention chart (via the tRPC `cohort` procedure) as well as the-// MCP / agent / REST retention endpoints.-//-// Definition: every user is assigned to exactly ONE cohort, the interval of-// their FIRST `firstEvent` within the window (first-touch). For each cohort we-// count how many of those users performed `secondEvent` 0..N intervals later.-// - criteria 'on' -> active exactly k intervals after (=)-// - criteria 'on_or_after' -> active at least k intervals after (>=, cumulative)-// When `firstEvent` / `secondEvent` is empty the name filter is dropped and the-// query measures retention on ANY activity (active-user retention).-// -----------------------------------------------------------------------------export type IRetentionInterval = 'minute' | 'hour' | 'day' | 'week' | 'month';-export type IRetentionCriteria = 'on' | 'on_or_after';--export type IGetRetentionCohortInput = {- projectId: string;- /** Event name(s) defining cohort entry. Empty/undefined => any event. */- firstEvent?: string[];- /** Event name(s) that count as "returned". Empty/undefined => any event. */- secondEvent?: string[];- criteria?: IRetentionCriteria;- interval?: IRetentionInterval;- /** ISO or `yyyy-MM-dd HH:mm:ss`. */- startDate: string;- /** ISO or `yyyy-MM-dd HH:mm:ss`. */- endDate: string;- /**- * Property and/or cohort filters scoping the analysed events. Cohort- * membership (inCohort/notInCohort) keeps the fast cohort_events_mv path;- * any property/column filter falls back to the raw events table.- */- filters?: IChartEventFilter[];-};--export type IRetentionCohortRow = {- cohort_interval: string;- sum: number;- values: number[];- percentages: number[];-};--const SQL_START_OF: Record<IRetentionInterval, string> = {- minute: 'toDate',- hour: 'toDate',- day: 'toDate',- week: 'toStartOfWeek',- month: 'toStartOfMonth',-};--const SQL_INTERVAL: Record<IRetentionInterval, string> = {- minute: 'DAY',- hour: 'DAY',- day: 'DAY',- week: 'WEEK',- month: 'MONTH',-};--const LUXON_UNIT: Record<IRetentionInterval, 'days' | 'weeks' | 'months'> = {- minute: 'days',- hour: 'days',- day: 'days',- week: 'weeks',- month: 'months',-};--// Normalize an ISO or `yyyy-MM-dd HH:mm:ss` string into ClickHouse date-time form.-function utc(date: string) {- return date.replace('T', ' ').slice(0, 19);-}--// Number of `interval` buckets spanned by [startDate, endDate]; drives the-// number of retention columns (0..diffInterval).-function diffIntervalCount(- startDate: string,- endDate: string,- interval: IRetentionInterval-) {- const unit = LUXON_UNIT[interval];- const start = DateTime.fromFormat(utc(startDate), 'yyyy-MM-dd HH:mm:ss', {- zone: 'utc',- });- const end = DateTime.fromFormat(utc(endDate), 'yyyy-MM-dd HH:mm:ss', {- zone: 'utc',- });- return Math.max(0, Math.floor(end.diff(start, unit).as(unit)));-}--// Build the `name = ... / name IN (...)` predicate; null means "any event".-function eventNameWhere(events: string[] | undefined): string | null {- if (!events || events.length === 0) {- return null;- }- if (events.length === 1) {- return `name = ${sqlstring.escape(events[0])}`;- }- return `name IN (${events.map((event) => sqlstring.escape(event)).join(', ')})`;-}--export async function getRetentionCohort(input: IGetRetentionCohortInput) {- const {- projectId,- firstEvent,- secondEvent,- criteria = 'on_or_after',- interval = 'day',- startDate,- endDate,- filters = [],- } = input;-- const diffInterval = diffIntervalCount(startDate, endDate, interval);- const sqlInterval = SQL_INTERVAL[interval];- const sqlToStartOf = SQL_START_OF[interval];- const countCriteria: '>=' | '=' = criteria === 'on_or_after' ? '>=' : '=';-- const start = utc(startDate);- const end = utc(endDate);- const escProject = sqlstring.escape(projectId);- const firstWhere = eventNameWhere(firstEvent);- const secondWhere = eventNameWhere(secondEvent);-- // Hybrid source: cohort_events_mv (skinny, fast) carries only- // project_id/name/created_at/profile_id, so any filter referencing event- // properties or columns forces a fallback to the raw events table. Cohort- // membership filters only need profile_id, so they stay on the fast path.- const needRawEvents = filters.some(- (filter) =>- filter.operator !== 'inCohort' && filter.operator !== 'notInCohort'- );- const source = needRawEvents ? TABLE_NAMES.events : TABLE_NAMES.cohort_events_mv;-- const baseConditions = [`project_id = ${escProject}`];- if (needRawEvents) {- // cohort_events_mv only stores identified-user rows; replicate that here.- baseConditions.push('profile_id != device_id');- }- if (filters.length > 0) {- baseConditions.push(- ...Object.values(getEventFiltersWhereClause(filters, projectId))- );- }- const baseWhere = baseConditions.join('\n AND ');-- const columns = range(0, diffInterval + 1);- const usersSelect = columns- .map(- (index) =>- `groupUniqArrayIf(profile_id, x_after_cohort ${countCriteria} ${index}) AS interval_${index}_users`- )- .join(',\n ');- const countsSelect = columns- .map(- (index) =>- `length(interval_${index}_users) AS interval_${index}_user_count`- )- .join(',\n ');-- const cohortQuery = `- WITH- cohort_users AS (- SELECT- profile_id AS userID,- ${sqlToStartOf}(min(created_at)) AS cohort_interval- FROM ${source}- WHERE ${baseWhere}- ${firstWhere ? `AND ${firstWhere}` : ''}- AND created_at >= toDateTime('${start}')- AND created_at <= toDateTime('${end}')- GROUP BY profile_id- ),- last_event AS (- SELECT- profile_id,- toDate(created_at) AS event_date- FROM ${source}- WHERE ${baseWhere}- ${secondWhere ? `AND ${secondWhere}` : ''}- AND created_at >= toDateTime('${start}')- AND created_at <= toDateTime('${end}') + INTERVAL ${diffInterval} ${sqlInterval}- ),- retention_matrix AS (- SELECT- f.cohort_interval,- l.profile_id,- dateDiff('${sqlInterval}', f.cohort_interval, ${sqlToStartOf}(l.event_date)) AS x_after_cohort- FROM cohort_users AS f- INNER JOIN last_event AS l ON f.userID = l.profile_id- WHERE l.event_date >= f.cohort_interval- AND dateDiff('${sqlInterval}', f.cohort_interval, ${sqlToStartOf}(l.event_date)) <= ${diffInterval}- ),- interval_users AS (- SELECT- cohort_interval,- ${usersSelect}- FROM retention_matrix- GROUP BY cohort_interval- ),- cohort_sizes AS (- SELECT- cohort_interval,- COUNT(DISTINCT userID) AS total_first_event_count- FROM cohort_users- GROUP BY cohort_interval- )- SELECT- interval_users.cohort_interval AS cohort_interval,- cs.total_first_event_count AS total_first_event_count,- ${countsSelect}- FROM interval_users- LEFT JOIN cohort_sizes AS cs ON interval_users.cohort_interval = cs.cohort_interval- ORDER BY cohort_interval ASC- `;-- const cohortData = await chQuery<{- cohort_interval: string;- total_first_event_count: number;- [key: string]: number | string;- }>(cohortQuery);-- // Reference point for cohort maturity: we only have return data up to "now",- // so periods that haven't elapsed yet are excluded from the weighted average.- const until = DateTime.utc().toFormat('yyyy-MM-dd HH:mm:ss');- return processCohortData(cohortData, diffInterval, interval, until);-}--// Number of fully-elapsed periods between a cohort's start and the reference-// date. Periods beyond this haven't happened yet, so they carry no return data-// and must be excluded from the average. Fails open (treats everything as-// mature) when dates are missing/unparseable.-function maturePeriodCount(- cohortInterval: string,- until: string | undefined,- interval: IRetentionInterval-): number {- if (!until) {- return Number.POSITIVE_INFINITY;- }- const unit = LUXON_UNIT[interval];- const cohort = DateTime.fromFormat(cohortInterval.slice(0, 10), 'yyyy-MM-dd', {- zone: 'utc',- });- const ref = DateTime.fromFormat(until, 'yyyy-MM-dd HH:mm:ss', { zone: 'utc' });- if (!cohort.isValid || !ref.isValid) {- return Number.POSITIVE_INFINITY;- }- return Math.floor(ref.diff(cohort, unit).as(unit));-}--// Shapes the raw ClickHouse matrix into per-cohort rows + a leading weighted--// average row.-//-// The average is a maturity-aware pooled rate: for each period column it pools-// only the cohorts old enough to have actually reached that period-// (cohort_interval + N intervals <= `until`). This keeps not-yet-elapsed cells-// from being read as churn AND keeps genuine zeros in, so the late curve is not-// inflated. Counts are normalised to the representative (mean) cohort size, so-// the row is internally consistent: period 0 equals Total profiles and the-// curve starts at 100%.-export function processCohortData(- data: Array<{- cohort_interval: string;- total_first_event_count: number;- [key: string]: number | string;- }>,- diffInterval: number,- interval: IRetentionInterval = 'day',- until?: string-): IRetentionCohortRow[] {- if (data.length === 0) {- return [];- }-- const columns = range(0, diffInterval + 1);- const processed = data.map((row) => {- const sum = row.total_first_event_count;- const values = columns.map(- (index) => (row[`interval_${index}_user_count`] || 0) as number- );-- return {- cohort_interval: row.cohort_interval,- sum,- values,- percentages: values.map((value) => (sum > 0 ? round(value / sum, 2) : 0)),- };- });-- const maturePeriods = processed.map((row) =>- maturePeriodCount(row.cohort_interval, until, interval)- );- const totalSize = processed.reduce((acc, row) => acc + row.sum, 0);- const representativeSize = round(totalSize / processed.length, 0);-- const averageRow: IRetentionCohortRow = {- cohort_interval: 'Weighted Average',- sum: representativeSize,- values: [],- percentages: [],- };-- for (const index of columns) {- let matureSize = 0;- let retained = 0;- processed.forEach((row, i) => {- if (index <= maturePeriods[i]!) {- matureSize += row.sum;- retained += row.values[index]!;- }- });- const rate = matureSize > 0 ? retained / matureSize : 0;- averageRow.percentages.push(round(rate, 2));- averageRow.values.push(round(rate * representativeSize, 0));- }-- return [averageRow, ...processed];-}+// The retention service — the week-over-week series, the rolling active-user+// series, the last-seen distribution and the cohort matrix — lives in+// @openpanel/core now (M7-004). Re-exported here for existing `@openpanel/db`+// importers (the barrel, apps/api's insights controller, packages/core's mcp+// and assistant tools) — same shape as chart.service.ts since M7-003.+export type {+ IGetRetentionCohortInput,+ IRetentionCohortRow,+ IRetentionCriteria,+ IRetentionInterval,+ IServiceRetentionRollingActiveUsers,+} from '@openpanel/core';+export {+ getEngagementCore,+ getRetentionCohort,+ getRetentionCohortCore,+ getRetentionLastSeenSeries,+ getRetentionSeries,+ getRollingActiveUsers,+ getRollingActiveUsersCore,+ getWeeklyRetentionSeriesCore,+ processCohortData,+} from '@openpanel/core';diff --git a/packages/db/src/services/sankey.service.ts b/packages/db/src/services/sankey.service.tsindex af1844612..3dc69bb41 100644--- a/packages/db/src/services/sankey.service.ts+++ b/packages/db/src/services/sankey.service.ts@@ -1,836 +1,10 @@-import { chartColors } from '@openpanel/constants';-import { type IChartEventFilter, zChartEvent } from '@openpanel/validation';-import { z } from 'zod';-import { TABLE_NAMES, ch } from '../clickhouse/client';-import { clix } from '../clickhouse/query-builder';-import { getEventFiltersWhereClause } from './chart.service';--export const zGetSankeyInput = z.object({- projectId: z.string(),- startDate: z.string(),- endDate: z.string(),- steps: z.number().min(2).max(10).default(5),- mode: z.enum(['between', 'after', 'before']),- startEvent: zChartEvent,- endEvent: zChartEvent.optional(),- exclude: z.array(z.string()).default([]),- include: z.array(z.string()).optional(),-});--export type IGetSankeyInput = z.infer<typeof zGetSankeyInput> & {- timezone: string;-};--export class SankeyService {- constructor(private client: typeof ch) {}-- getRawWhereClause(type: 'events' | 'sessions', filters: IChartEventFilter[]) {- const where = getEventFiltersWhereClause(- filters.map((item) => {- if (type === 'sessions') {- if (item.name === 'path') {- return { ...item, name: 'entry_path' };- }- if (item.name === 'origin') {- return { ...item, name: 'entry_origin' };- }- if (item.name.startsWith('properties.__query.utm_')) {- return {- ...item,- name: item.name.replace('properties.__query.utm_', 'utm_'),- };- }- return item;- }- return item;- }),- );-- return Object.values(where).join(' AND ');- }-- private buildEventNameFilter(- include: string[] | undefined,- exclude: string[],- startEventName: string | undefined,- endEventName: string | undefined,- ) {- if (include && include.length > 0) {- const eventNames = [...include, startEventName, endEventName]- .filter((item) => item !== undefined)- .map((e) => `'${e!.replace(/'/g, "''")}'`)- .join(', ');- return `name IN (${eventNames})`;- }- if (exclude.length > 0) {- const excludedNames = exclude- .map((e) => `'${e.replace(/'/g, "''")}'`)- .join(', ');- return `name NOT IN (${excludedNames})`;- }- return null;- }-- private buildSessionEventCTE(- event: z.infer<typeof zChartEvent>,- projectId: string,- startDate: string,- endDate: string,- timezone: string,- ): ReturnType<typeof clix> {- return clix(this.client, timezone)- .select<{ session_id: string }>(['session_id'])- .from(TABLE_NAMES.events)- .where('project_id', '=', projectId)- .where('name', '=', event.name)- .where('created_at', 'BETWEEN', [- clix.datetime(startDate, 'toDateTime'),- clix.datetime(endDate, 'toDateTime'),- ])- .rawWhere(this.getRawWhereClause('events', event.filters))- .groupBy(['session_id']);- }-- private getModeConfig(- mode: 'after' | 'before' | 'between',- startEvent: z.infer<typeof zChartEvent> | undefined,- endEvent: z.infer<typeof zChartEvent> | undefined,- hasStartEventCTE: boolean,- hasEndEventCTE: boolean,- steps: number,- ): { sessionFilter: string; eventsSliceExpr: string } {- const defaultSliceExpr = `arraySlice(events_deduped, 1, ${steps})`;-- if (mode === 'after' && startEvent) {- const escapedStartEvent = startEvent.name.replace(/'/g, "''");- const sessionFilter = hasStartEventCTE- ? 'session_id IN (SELECT session_id FROM start_event_sessions)'- : `arrayExists(x -> x = '${escapedStartEvent}', events_deduped)`;- const eventsSliceExpr = `arraySlice(events_deduped, arrayFirstIndex(x -> x = '${escapedStartEvent}', events_deduped), ${steps})`;- return { sessionFilter, eventsSliceExpr };- }-- if (mode === 'before' && startEvent) {- const escapedStartEvent = startEvent.name.replace(/'/g, "''");- const sessionFilter = hasStartEventCTE- ? 'session_id IN (SELECT session_id FROM start_event_sessions)'- : `arrayExists(x -> x = '${escapedStartEvent}', events_deduped)`;- const eventsSliceExpr = `arraySlice(- events_deduped,- greatest(1, arrayFirstIndex(x -> x = '${escapedStartEvent}', events_deduped) - ${steps} + 1),- arrayFirstIndex(x -> x = '${escapedStartEvent}', events_deduped) - greatest(1, arrayFirstIndex(x -> x = '${escapedStartEvent}', events_deduped) - ${steps} + 1) + 1- )`;- return { sessionFilter, eventsSliceExpr };- }-- if (mode === 'between' && startEvent && endEvent) {- const escapedStartEvent = startEvent.name.replace(/'/g, "''");- const escapedEndEvent = endEvent.name.replace(/'/g, "''");- let sessionFilter = '';- if (hasStartEventCTE && hasEndEventCTE) {- sessionFilter =- 'session_id IN (SELECT session_id FROM start_event_sessions) AND session_id IN (SELECT session_id FROM end_event_sessions)';- } else if (hasStartEventCTE) {- sessionFilter = `session_id IN (SELECT session_id FROM start_event_sessions) AND arrayExists(x -> x = '${escapedEndEvent}', events_deduped)`;- } else if (hasEndEventCTE) {- sessionFilter = `arrayExists(x -> x = '${escapedStartEvent}', events_deduped) AND session_id IN (SELECT session_id FROM end_event_sessions)`;- } else {- sessionFilter = `arrayExists(x -> x = '${escapedStartEvent}', events_deduped) AND arrayExists(x -> x = '${escapedEndEvent}', events_deduped)`;- }- return { sessionFilter, eventsSliceExpr: defaultSliceExpr };- }-- return { sessionFilter: '', eventsSliceExpr: defaultSliceExpr };- }-- private async executeBetweenMode(- sessionPathsQuery: ReturnType<typeof clix>,- startEvent: z.infer<typeof zChartEvent>,- endEvent: z.infer<typeof zChartEvent>,- steps: number,- COLORS: string[],- timezone: string,- ): Promise<{- nodes: Array<{- id: string;- label: string;- nodeColor: string;- percentage?: number;- value?: number;- step?: number;- }>;- links: Array<{ source: string; target: string; value: number }>;- }> {- // Find sessions where startEvent comes before endEvent- const betweenSessionsQuery = clix(this.client, timezone)- .with('session_paths', sessionPathsQuery)- .select<{- session_id: string;- events: string[];- start_index: number;- end_index: number;- }>([- 'session_id',- 'events',- `arrayFirstIndex(x -> x = '${startEvent.name.replace(/'/g, "''")}', events) as start_index`,- `arrayFirstIndex(x -> x = '${endEvent.name.replace(/'/g, "''")}', events) as end_index`,- ])- .from('session_paths')- .having('start_index', '>', 0)- .having('end_index', '>', 0)- .rawHaving('start_index < end_index');-- // Get the slice between start and end- const betweenPathsQuery = clix(this.client, timezone)- .with('between_sessions', betweenSessionsQuery)- .select<{- session_id: string;- events: string[];- entry_event: string;- }>([- 'session_id',- 'arraySlice(events, start_index, end_index - start_index + 1) as events',- 'events[start_index] as entry_event',- ])- .from('between_sessions');-- // Get top entry events- const topEntriesQuery = clix(this.client, timezone)- .with('session_paths', betweenPathsQuery)- .select<{ entry_event: string; count: number }>([- 'entry_event',- 'count() as count',- ])- .from('session_paths')- .groupBy(['entry_event'])- .orderBy('count', 'DESC')- .limit(3);-- const topEntries = await topEntriesQuery.execute();-- if (topEntries.length === 0) {- return { nodes: [], links: [] };- }-- const topEntryEvents = topEntries.map((e) => e.entry_event);- const totalSessions = topEntries.reduce((sum, e) => sum + e.count, 0);-- // Get transitions for between mode- const transitionsQuery = clix(this.client, timezone)- .with('between_sessions', betweenSessionsQuery)- .with(- 'session_paths',- clix(this.client, timezone)- .select([- 'session_id',- 'arraySlice(events, start_index, end_index - start_index + 1) as events',- ])- .from('between_sessions')- .having('events[1]', 'IN', topEntryEvents),- )- .select<{- source: string;- target: string;- step: number;- value: number;- }>([- 'pair.1 as source',- 'pair.2 as target',- 'pair.3 as step',- 'count() as value',- ])- .from(- clix.exp(- '(SELECT arrayJoin(arrayMap(i -> (events[i], events[i + 1], i), range(1, length(events)))) as pair FROM session_paths WHERE length(events) >= 2)',- ),- )- .groupBy(['source', 'target', 'step'])- .orderBy('step', 'ASC')- .orderBy('value', 'DESC');-- const transitions = await transitionsQuery.execute();-- return this.buildSankeyFromTransitions(- transitions,- topEntries,- totalSessions,- steps,- COLORS,- );- }-- private async executeSimpleMode(- sessionPathsQuery: ReturnType<typeof clix>,- steps: number,- COLORS: string[],- timezone: string,- ): Promise<{- nodes: Array<{- id: string;- label: string;- nodeColor: string;- percentage?: number;- value?: number;- step?: number;- }>;- links: Array<{ source: string; target: string; value: number }>;- }> {- // Get top entry events- const topEntriesQuery = clix(this.client, timezone)- .with('session_paths', sessionPathsQuery)- .select<{ entry_event: string; count: number }>([- 'entry_event',- 'count() as count',- ])- .from('session_paths')- .groupBy(['entry_event'])- .orderBy('count', 'DESC')- .limit(3);-- const topEntries = await topEntriesQuery.execute();-- if (topEntries.length === 0) {- return { nodes: [], links: [] };- }-- const topEntryEvents = topEntries.map((e) => e.entry_event);- const totalSessions = topEntries.reduce((sum, e) => sum + e.count, 0);-- // Get transitions- const transitionsQuery = clix(this.client, timezone)- .with('session_paths_base', sessionPathsQuery)- .with(- 'session_paths',- clix(this.client, timezone)- .select(['session_id', 'events'])- .from('session_paths_base')- .having('events[1]', 'IN', topEntryEvents),- )- .select<{- source: string;- target: string;- step: number;- value: number;- }>([- 'pair.1 as source',- 'pair.2 as target',- 'pair.3 as step',- 'count() as value',- ])- .from(- clix.exp(- '(SELECT arrayJoin(arrayMap(i -> (events[i], events[i + 1], i), range(1, length(events)))) as pair FROM session_paths WHERE length(events) >= 2)',- ),- )- .groupBy(['source', 'target', 'step'])- .orderBy('step', 'ASC')- .orderBy('value', 'DESC');-- const transitions = await transitionsQuery.execute();-- return this.buildSankeyFromTransitions(- transitions,- topEntries,- totalSessions,- steps,- COLORS,- );- }-- async getSankey({- projectId,- startDate,- endDate,- steps = 5,- mode,- startEvent,- endEvent,- exclude = [],- include,- timezone,- }: IGetSankeyInput): Promise<{- nodes: Array<{- id: string;- label: string;- nodeColor: string;- percentage?: number;- value?: number;- step?: number;- }>;- links: Array<{ source: string; target: string; value: number }>;- }> {- const COLORS = chartColors.map((color) => color.main);-- // 1. Build event name filter- const eventNameFilter = this.buildEventNameFilter(- include,- exclude,- startEvent?.name,- endEvent?.name,- );-- // 2. Build ordered events query- // For screen_view events, use the path instead of the event name for more meaningful flow visualization- const orderedEventsQuery = clix(this.client, timezone)- .select<{- session_id: string;- event_name: string;- created_at: string;- }>([- 'session_id',- // "if(name = 'screen_view', path, name) as event_name",- 'name as event_name',- 'created_at',- ])- .from(TABLE_NAMES.events)- .where('project_id', '=', projectId)- .where('created_at', 'BETWEEN', [- clix.datetime(startDate, 'toDateTime'),- clix.datetime(endDate, 'toDateTime'),- ])- .orderBy('session_id', 'ASC')- .orderBy('created_at', 'ASC');-- if (eventNameFilter) {- orderedEventsQuery.rawWhere(eventNameFilter);- }-- // 3. Build session event CTEs- const startEventCTE = startEvent- ? this.buildSessionEventCTE(- startEvent,- projectId,- startDate,- endDate,- timezone,- )- : null;- const endEventCTE =- mode === 'between' && endEvent- ? this.buildSessionEventCTE(- endEvent,- projectId,- startDate,- endDate,- timezone,- )- : null;-- // 4. Build deduped events CTE- const eventsDedupedCTE = clix(this.client, timezone)- .with('ordered_events', orderedEventsQuery)- .select<{- session_id: string;- events_deduped: string[];- }>([- 'session_id',- `arrayFilter(- (x, i) -> i = 1 OR x != events_raw[i - 1],- groupArray(event_name) as events_raw,- arrayEnumerate(events_raw)- ) as events_deduped`,- ])- .from('ordered_events')- .groupBy(['session_id']);-- // 5. Get mode-specific config- const { sessionFilter, eventsSliceExpr } = this.getModeConfig(- mode,- startEvent,- endEvent,- startEventCTE !== null,- endEventCTE !== null,- steps,- );-- // 6. Build truncate expression (for 'after' mode)- const truncateAtRepeatExpr = `if(- arrayFirstIndex(x -> x > 1, arrayEnumerateUniq(events_sliced)) = 0,- events_sliced,- arraySlice(- events_sliced,- 1,- arrayFirstIndex(x -> x > 1, arrayEnumerateUniq(events_sliced)) - 1- )- )`;- const eventsExpr =- mode === 'before' ? 'events_sliced' : truncateAtRepeatExpr;-- // 7. Build session paths query with conditional CTEs- const eventCTEs: Array<{ name: string; query: ReturnType<typeof clix> }> =- [];- if (startEventCTE) {- eventCTEs.push({ name: 'start_event_sessions', query: startEventCTE });- }- if (endEventCTE) {- eventCTEs.push({ name: 'end_event_sessions', query: endEventCTE });- }-- const sessionPathsQuery = eventCTEs- .reduce(- (builder, cte) => builder.with(cte.name, cte.query),- clix(this.client, timezone),- )- .with('events_deduped_cte', eventsDedupedCTE)- .with(- 'events_sliced_cte',- clix(this.client, timezone)- .select<{- session_id: string;- events_sliced: string[];- }>(['session_id', `${eventsSliceExpr} as events_sliced`])- .from('events_deduped_cte')- .rawHaving(sessionFilter || '1 = 1'),- )- .select<{- session_id: string;- entry_event: string;- events: string[];- }>(['session_id', `${eventsExpr} as events`, 'events[1] as entry_event'])- .from('events_sliced_cte')- .having('length(events)', '>=', 2);-- // 8. Execute mode-specific logic- if (mode === 'between' && startEvent && endEvent) {- return this.executeBetweenMode(- sessionPathsQuery,- startEvent,- endEvent,- steps,- COLORS,- timezone,- );- }-- return this.executeSimpleMode(sessionPathsQuery, steps, COLORS, timezone);- }-- private buildSankeyFromTransitions(- transitions: Array<{- source: string;- target: string;- step: number;- value: number;- }>,- topEntries: Array<{ entry_event: string; count: number }>,- totalSessions: number,- steps: number,- COLORS: string[],- ) {- if (transitions.length === 0) {- return { nodes: [], links: [] };- }-- const TOP_DESTINATIONS_PER_NODE = 3;-- // Build the sankey progressively step by step- const nodes = new Map<- string,- { event: string; value: number; step: number; color: string }- >();- const links: Array<{ source: string; target: string; value: number }> = [];-- // Helper to create unique node ID- const getNodeId = (event: string, step: number) => `${event}::step${step}`;-- // Group transitions by step- const transitionsByStep = new Map<number, typeof transitions>();- for (const t of transitions) {- if (!transitionsByStep.has(t.step)) {- transitionsByStep.set(t.step, []);- }- transitionsByStep.get(t.step)!.push(t);- }-- // Initialize with entry events (step 1)- const activeNodes = new Map<string, string>(); // event -> nodeId- topEntries.forEach((entry, idx) => {- const nodeId = getNodeId(entry.entry_event, 1);- nodes.set(nodeId, {- event: entry.entry_event,- value: entry.count,- step: 1,- color: COLORS[idx % COLORS.length]!,- });- activeNodes.set(entry.entry_event, nodeId);- });-- // Process each step: from active nodes, find top destinations- for (let step = 1; step < steps; step++) {- const stepTransitions = transitionsByStep.get(step) || [];- const nextActiveNodes = new Map<string, string>();-- // For each currently active node, find its top destinations- for (const [sourceEvent, sourceNodeId] of activeNodes) {- // Get transitions FROM this source event- const fromSource = stepTransitions- .filter((t) => t.source === sourceEvent)- .sort((a, b) => b.value - a.value)- .slice(0, TOP_DESTINATIONS_PER_NODE);-- for (const t of fromSource) {- // Skip self-loops- if (t.source === t.target) continue;-- const targetNodeId = getNodeId(t.target, step + 1);-- // Add link using unique node IDs- links.push({- source: sourceNodeId,- target: targetNodeId,- value: t.value,- });-- // Add/update target node- const existing = nodes.get(targetNodeId);- if (existing) {- existing.value += t.value;- } else {- // Inherit color from source or assign new- const sourceData = nodes.get(sourceNodeId);- nodes.set(targetNodeId, {- event: t.target,- value: t.value,- step: step + 1,- color: sourceData?.color || COLORS[nodes.size % COLORS.length]!,- });- }-- nextActiveNodes.set(t.target, targetNodeId);- }- }-- // Update active nodes for next iteration- activeNodes.clear();- for (const [event, nodeId] of nextActiveNodes) {- activeNodes.set(event, nodeId);- }-- // Stop if no more nodes to process- if (activeNodes.size === 0) break;- }-- // Filter links by threshold (0.25% of total sessions)- const MIN_LINK_PERCENT = 0.25;- const minLinkValue = Math.ceil((totalSessions * MIN_LINK_PERCENT) / 100);- const filteredLinks = links.filter((link) => link.value >= minLinkValue);-- // Find all nodes referenced by remaining links- const referencedNodeIds = new Set<string>();- filteredLinks.forEach((link) => {- referencedNodeIds.add(link.source);- referencedNodeIds.add(link.target);- });-- // Recompute node values from filtered links- const nodeValuesFromLinks = new Map<string, number>();- filteredLinks.forEach((link) => {- const current = nodeValuesFromLinks.get(link.target) || 0;- nodeValuesFromLinks.set(link.target, current + link.value);- });-- // For entry nodes (step 1), only keep them if they have outgoing links after filtering- nodes.forEach((nodeData, nodeId) => {- if (nodeData.step === 1) {- const hasOutgoing = filteredLinks.some((l) => l.source === nodeId);- if (!hasOutgoing) {- referencedNodeIds.delete(nodeId);- }- }- });-- // Build final nodes array sorted by step then value- const finalNodes = Array.from(nodes.entries())- .filter(([id]) => referencedNodeIds.has(id))- .map(([id, data]) => {- const value =- data.step === 1- ? data.value- : nodeValuesFromLinks.get(id) || data.value;- return {- id,- label: data.event,- nodeColor: data.color,- percentage: (value / totalSessions) * 100,- value,- step: data.step,- };- })- .sort((a, b) => {- if (a.step !== b.step) return a.step - b.step;- return b.value - a.value;- });-- // Sanity check: Ensure all link endpoints exist in nodes- const nodeIds = new Set(finalNodes.map((n) => n.id));- const validLinks = filteredLinks.filter(- (link) => nodeIds.has(link.source) && nodeIds.has(link.target),- );-- // Combine final nodes with the same event name- // A final node is one that has no outgoing links- const nodesWithOutgoing = new Set(validLinks.map((l) => l.source));- const finalNodeIds = new Set(- finalNodes.filter((n) => !nodesWithOutgoing.has(n.id)).map((n) => n.id),- );-- // Group final nodes by event name- const finalNodesByEvent = new Map<string, typeof finalNodes>();- finalNodes.forEach((node) => {- if (finalNodeIds.has(node.id)) {- if (!finalNodesByEvent.has(node.label)) {- finalNodesByEvent.set(node.label, []);- }- finalNodesByEvent.get(node.label)!.push(node);- }- });-- // Create merged nodes and remap links- const nodeIdRemap = new Map<string, string>(); // old nodeId -> new merged nodeId- const mergedNodes = new Map<string, (typeof finalNodes)[0]>(); // merged nodeId -> node data-- finalNodesByEvent.forEach((nodesToMerge, eventName) => {- if (nodesToMerge.length > 1) {- // Merge multiple final nodes with same event name- const maxStep = Math.max(...nodesToMerge.map((n) => n.step || 0));- const totalValue = nodesToMerge.reduce(- (sum, n) => sum + (n.value || 0),- 0,- );- const mergedNodeId = `${eventName}::final`;- const firstNode = nodesToMerge[0]!;-- // Create merged node at the maximum step- mergedNodes.set(mergedNodeId, {- id: mergedNodeId,- label: eventName,- nodeColor: firstNode.nodeColor,- percentage: (totalValue / totalSessions) * 100,- value: totalValue,- step: maxStep,- });-- // Map all old node IDs to the merged node ID- nodesToMerge.forEach((node) => {- nodeIdRemap.set(node.id, mergedNodeId);- });- }- });-- // Update links to point to merged nodes- const remappedLinks = validLinks.map((link) => {- const newSource = nodeIdRemap.get(link.source) || link.source;- const newTarget = nodeIdRemap.get(link.target) || link.target;- return {- source: newSource,- target: newTarget,- value: link.value,- };- });-- // Combine merged nodes with non-final nodes- const nonFinalNodes = finalNodes.filter((n) => !finalNodeIds.has(n.id));- const finalNodesList = Array.from(mergedNodes.values());-- // Remove old final nodes that were merged- const mergedOldNodeIds = new Set(nodeIdRemap.keys());- const remainingNodes = nonFinalNodes.filter(- (n) => !mergedOldNodeIds.has(n.id),- );-- // Combine all nodes and sort- const allNodes = [...remainingNodes, ...finalNodesList].sort((a, b) => {- if (a.step !== b.step) return a.step! - b.step!;- return b.value! - a.value!;- });-- // Aggregate links that now point to the same merged target- const linkMap = new Map<string, number>(); // "source->target" -> value- remappedLinks.forEach((link) => {- const key = `${link.source}->${link.target}`;- linkMap.set(key, (linkMap.get(key) || 0) + link.value);- });-- const aggregatedLinks = Array.from(linkMap.entries())- .map(([key, value]) => {- const parts = key.split('->');- if (parts.length !== 2) return null;- return { source: parts[0]!, target: parts[1]!, value };- })- .filter(- (link): link is { source: string; target: string; value: number } =>- link !== null,- );-- // Final sanity check: Ensure all link endpoints exist in nodes- const finalNodeIdsSet = new Set(allNodes.map((n) => n.id));- const finalValidLinks: Array<{- source: string;- target: string;- value: number;- }> = aggregatedLinks.filter(- (link) =>- finalNodeIdsSet.has(link.source) && finalNodeIdsSet.has(link.target),- );-- return {- nodes: allNodes,- links: finalValidLinks,- };- }-}--export const sankeyService = new SankeyService(ch);--import { getSettingsForProject } from './organization.service';--function toChartEvent(name: string) {- return {- id: name,- name,- displayName: name,- type: 'event' as const,- segment: 'event' as const,- filters: [],- };-}--export async function getUserFlowCore(input: {- projectId: string;- startDate: string;- endDate: string;- startEvent: string;- endEvent?: string;- mode: 'after' | 'before' | 'between';- steps?: number;- exclude?: string[];- include?: string[];-}) {- if (input.mode === 'between' && !input.endEvent) {- throw new Error('endEvent is required when mode is "between"');- }-- const { timezone } = await getSettingsForProject(input.projectId);- const result = await sankeyService.getSankey({- projectId: input.projectId,- startDate: input.startDate,- endDate: input.endDate,- steps: input.steps ?? 5,- mode: input.mode,- startEvent: toChartEvent(input.startEvent),- endEvent: input.endEvent ? toChartEvent(input.endEvent) : undefined,- exclude: input.exclude ?? [],- include: input.include,- timezone,- });-- return {- mode: input.mode,- startEvent: input.startEvent,- endEvent: input.endEvent,- node_count: result.nodes.length,- link_count: result.links.length,- nodes: result.nodes,- links: result.links,- };-}+// The sankey (user-flow) service lives in @openpanel/core now (M7-004).+// Re-exported here for existing `@openpanel/db` importers (the barrel,+// apps/api's insights controller, packages/core's mcp and assistant tools).+export type { IGetSankeyInput } from '@openpanel/core';+export {+ getRawWhereClause,+ getSankey,+ getUserFlowCore,+ zGetSankeyInput,+} from '@openpanel/core';