OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.
fix(funnel): compute "View Users" from the same query as the chart (#434)
Carl-Gerhard Lindesvärd716c59e6fcc2
diff --git a/apps/start/src/modals/view-chart-users.tsx b/apps/start/src/modals/view-chart-users.tsxindex 69d6ed369..f6b03be0d 100644--- a/apps/start/src/modals/view-chart-users.tsx+++ b/apps/start/src/modals/view-chart-users.tsx@@ -149,6 +149,39 @@ function ProfileList({ profiles }: { profiles: any[] }) {);}+/**+ * Renders the query's outcome. A failed request must not look like an empty+ * result — `data ?? []` used to render backend errors as "No users found",+ * which hid real failures (e.g. a funnel breakdown whose SQL didn't compile).+ */+function ProfileListState({+ query,+}: {+ query: { isLoading: boolean; isError: boolean; data?: any[] };+}) {+ if (query.isLoading) {+ return (+ <div className="flex items-center justify-center py-8">+ <div className="text-muted-foreground">Loading users...</div>+ </div>+ );+ }++ if (query.isError) {+ return (+ <div className="col items-center justify-center gap-1 py-8 text-center">+ <div className="font-medium">Could not load users</div>+ <div className="text-muted-foreground text-sm">+ Something went wrong running this query. Try again, or adjust the+ report.+ </div>+ </div>+ );+ }++ return <ProfileList profiles={query.data ?? []} />;+}+// Chart-specific props and componentinterface ChartUsersViewProps {chartData: IChartData;@@ -213,8 +246,6 @@ function ChartUsersView({ chartData, report, date }: ChartUsersViewProps) {),);- const profiles = profilesQuery.data ?? [];-return (<ScrollableModalheader={@@ -273,13 +304,7 @@ function ChartUsersView({ chartData, report, date }: ChartUsersViewProps) {}><div className="col">- {profilesQuery.isLoading ? (- <div className="flex items-center justify-center py-8">- <div className="text-muted-foreground">Loading users...</div>- </div>- ) : (- <ProfileList profiles={profiles} />- )}+ <ProfileListState query={profilesQuery} /></div></ScrollableModal>);@@ -330,7 +355,6 @@ function FunnelUsersView({ report, stepIndex, breakdownValues }: FunnelUsersView),);- const profiles = profilesQuery.data ?? [];const isLastStep = stepIndex === report.series.length - 1;return (@@ -377,13 +401,7 @@ function FunnelUsersView({ report, stepIndex, breakdownValues }: FunnelUsersView}><div className="flex flex-col gap-4">- {profilesQuery.isLoading ? (- <div className="flex items-center justify-center py-8">- <div className="text-muted-foreground">Loading users...</div>- </div>- ) : (- <ProfileList profiles={profiles} />- )}+ <ProfileListState query={profilesQuery} /></div></ScrollableModal>);diff --git a/packages/db/src/services/funnel-sql.test.ts b/packages/db/src/services/funnel-sql.test.tsnew file mode 100644index 000000000..35fc8e097--- /dev/null+++ b/packages/db/src/services/funnel-sql.test.ts@@ -0,0 +1,218 @@+/**+ * SQL-syntax tests for the shared funnel query builder.+ *+ * Same strategy as chart-sql.test.ts: build the SQL string, then run+ * `EXPLAIN <sql>` against the local ClickHouse instance. EXPLAIN parses and+ * resolves columns without executing, so a breakdown expression referencing a+ * join alias that was never added fails here as UNKNOWN_IDENTIFIER — which is+ * exactly the class of bug that made funnel "View Users" return "No users+ * found" for profile-property and cohort breakdowns.+ *+ * Requires `pnpm dock:up` (or any locally reachable CH at+ * http://localhost:8123/openpanel). All tests auto-skip if CH is unreachable.+ */+import type { IChartBreakdown, IReportInput } from '@openpanel/validation';+import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest';++const mockCohortFindMany = vi.hoisted(() => vi.fn());++vi.mock('../prisma-client', () => ({+ db: { cohort: { findMany: mockCohortFindMany } },+}));++import { ch } from '../clickhouse/client';+import { funnelService } from './funnel.service';++const PROJECT_ID = 'test-sql-validation';+const COHORT_ID = 'aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee';+const START = '2026-04-14 00:00:00';+const END = '2026-05-15 00:00:00';++type ISeriesItem = IReportInput['series'][number];++const event = (overrides: Partial<ISeriesItem> = {}): ISeriesItem =>+ ({+ id: 'A',+ type: 'event',+ name: 'screen_view',+ segment: 'event',+ filters: [],+ ...overrides,+ }) as ISeriesItem;++const breakdown = (name: string): IChartBreakdown => ({ id: name, name });++const SERIES = [event({ id: 'A' }), event({ id: 'B', name: 'sign_up' })];++let chReachable = false;++async function explain(sql: string): Promise<void> {+ await ch.command({ query: `EXPLAIN ${sql}` });+}++/** Mirrors what getFunnelProfiles builds on top of the shared base. */+async function buildProfilesSql(breakdowns: IChartBreakdown[]) {+ const { query } = await funnelService.buildFunnelBase({+ projectId: PROJECT_ID,+ startDate: START,+ endDate: END,+ series: SERIES,+ breakdowns,+ funnelWindow: 24,+ timezone: 'UTC',+ });+ query.with('funnel', 'SELECT * FROM session_funnel WHERE level != 0');+ query.select(['DISTINCT profile_id']).from('funnel');+ query.where('level', '>=', 2);+ return query.toSQL();+}++/** Mirrors what the funnel chart builds on top of the shared base. */+async function buildChartSql(breakdowns: IChartBreakdown[]) {+ const { query, breakdowns: resolved } = await funnelService.buildFunnelBase({+ projectId: PROJECT_ID,+ startDate: START,+ endDate: END,+ series: SERIES,+ breakdowns,+ funnelWindow: 24,+ timezone: 'UTC',+ });+ query.with('funnel', 'SELECT * FROM session_funnel WHERE level != 0');+ query+ .select([+ 'level',+ ...resolved.map((_, index) => `b_${index}`),+ 'count() as count',+ ])+ .from('funnel')+ .groupBy(['level', ...resolved.map((_, index) => `b_${index}`)]);+ return query.toSQL();+}++beforeAll(async () => {+ vi.spyOn(console, 'log').mockImplementation(() => {});+ mockCohortFindMany.mockResolvedValue([+ { id: COHORT_ID, name: 'Power users' },+ ]);+ try {+ await ch.command({ query: 'SELECT 1' });+ chReachable = true;+ } catch {+ chReachable = false;+ }+});++afterAll(() => {+ vi.restoreAllMocks();+});++// Reports as skipped rather than passed when CH is unreachable — a test that+// returns before asserting anything should not read as green.+const itCH = (name: string, fn: () => Promise<void>) =>+ it(name, async (ctx) => {+ if (!chReachable) {+ ctx.skip('ClickHouse not reachable at CLICKHOUSE_URL');+ }+ await fn();+ });++describe('funnel.service / buildFunnelBase — profile breakdowns', () => {+ itCH('adds the profiles join for a profile.properties breakdown', async () => {+ const sql = await buildProfilesSql([breakdown('profile.properties.plan')]);++ // The breakdown renders `profile.properties['plan']`, so the alias must exist.+ expect(sql).toContain("profile.properties['plan']");+ expect(sql).toMatch(/as profile/);+ expect(sql).toContain('profile.id = events.profile_id');+ await explain(sql);+ });++ itCH('selects the properties column, not a bare `properties` field', async () => {+ const sql = await buildProfilesSql([breakdown('profile.properties.plan')]);+ expect(sql).toMatch(/SELECT [^)]*properties[^)]*FROM .*profiles/);+ await explain(sql);+ });++ itCH('adds the profiles join for a top-level profile breakdown', async () => {+ const sql = await buildProfilesSql([breakdown('profile.email')]);+ expect(sql).toContain('profile.id = events.profile_id');+ await explain(sql);+ });++ itCH('adds the join for a profile filter with no profile breakdown', async () => {+ const { query } = await funnelService.buildFunnelBase({+ projectId: PROJECT_ID,+ startDate: START,+ endDate: END,+ series: [+ event({+ filters: [+ { id: 'f', name: 'profile.email', operator: 'is', value: ['[email protected]'] },+ ],+ }),+ event({ id: 'B', name: 'sign_up' }),+ ],+ funnelWindow: 24,+ timezone: 'UTC',+ });+ query.with('funnel', 'SELECT * FROM session_funnel WHERE level != 0');+ query.select(['DISTINCT profile_id']).from('funnel');+ await explain(query.toSQL());+ });+});++describe('funnel.service / buildFunnelBase — cohort breakdowns', () => {+ itCH('adds the cohort join for a cohort breakdown', async () => {+ const sql = await buildProfilesSql([breakdown(`cohort:${COHORT_ID}`)]);++ // The breakdown renders `cohort_<id>.profile_id`, so its join must exist.+ const alias = `cohort_${COHORT_ID.replace(/-/g, '_')}`;+ expect(sql).toContain(`${alias}.profile_id`);+ expect(sql).toContain(`AS ${alias}`);+ expect(sql).toContain('Power users');+ await explain(sql);+ });+});++describe('funnel.service / buildFunnelBase — unknown breakdowns', () => {+ itCH('drops the all-cohorts breakdown, which the funnel cannot render', async () => {+ // Bare `cohort` is the chart's all-cohorts breakdown. The funnel has no+ // equivalent, and inlining it produced `cohort as b_0 FROM events` —+ // UNKNOWN_IDENTIFIER for the chart and the profile list alike.+ const sql = await buildProfilesSql([breakdown('cohort')]);+ expect(sql).not.toMatch(/\bcohort as b_0\b/);+ await explain(sql);+ });++ itCH('drops an unrecognised breakdown name entirely', async () => {+ const sql = await buildProfilesSql([breakdown('not_a_real_column')]);+ expect(sql).not.toContain('not_a_real_column');+ await explain(sql);+ });++ itCH('drops unsupported breakdowns identically for chart and profiles', async () => {+ const breakdowns = [+ breakdown('cohort'),+ breakdown('profile.properties.plan'),+ ];+ const chartSql = await buildChartSql(breakdowns);+ const profilesSql = await buildProfilesSql(breakdowns);++ // Both sides must resolve the same breakdown to the same b_N index,+ // otherwise the breakdownValues filter targets the wrong column.+ expect(chartSql).toContain("profile.properties['plan'] as b_0");+ expect(profilesSql).toContain("profile.properties['plan'] as b_0");+ await explain(chartSql);+ await explain(profilesSql);+ });+});++describe('funnel.service / buildFunnelBase — group breakdowns', () => {+ itCH('adds the group array join for a group breakdown', async () => {+ const sql = await buildProfilesSql([breakdown('group.plan')]);+ expect(sql).toContain('ARRAY JOIN groups AS _group_id');+ expect(sql).toContain('LEFT ANY JOIN _g ON _g.id = _group_id');+ await explain(sql);+ });+});diff --git a/packages/db/src/services/funnel.service.ts b/packages/db/src/services/funnel.service.tsindex 442c34df5..8f61b9ab3 100644--- a/packages/db/src/services/funnel.service.ts+++ b/packages/db/src/services/funnel.service.ts@@ -1,5 +1,9 @@import { ifNaN } from '@openpanel/common';-import type { IChartEvent, IReportInput } from '@openpanel/validation';+import type {+ IChartBreakdown,+ IChartEvent,+ IReportInput,+} from '@openpanel/validation';import { last, reverse, uniq } from 'ramda';import sqlstring from 'sqlstring';import { ch } from '../clickhouse/client';@@ -13,6 +17,7 @@ import {fetchCohortsMetadata,getEventFiltersWhereClause,getSelectPropertyKey,+ isAllCohortsBreakdown,isKnownEventField,} from './chart.service';import { mergeGlobalFilters, onlyReportEvents } from './reports.service';@@ -209,47 +214,66 @@ export class FunnelService {);}- async getFunnel({+ /**+ * Builds everything the funnel chart and the funnel profile list share: the+ * normalized event series and breakdowns, the `session_funnel` CTE with all+ * of its joins wired up, and the outer query those CTEs are registered on.+ *+ * This exists because the two used to be written out twice and drifted. A+ * breakdown expression only works if the join it references was added, and+ * the joins depend on the breakdowns — so building the selects in one place+ * and the joins in another is exactly the bug waiting to happen. Callers add+ * their own `funnel` CTE and final projection on top.+ */+ async buildFunnelBase({projectId,startDate,endDate,series,globalFilters,- options,- breakdowns = [],- limit,- timezone = 'UTC',- }: IReportInput & { timezone: string; events?: IChartEvent[] }) {- if (!startDate || !endDate) {- throw new Error('startDate and endDate are required');- }-- series = mergeGlobalFilters(series, globalFilters);-- const funnelOptions = options?.type === 'funnel' ? options : undefined;- const funnelWindow = funnelOptions?.funnelWindow ?? 24;- const funnelGroup = funnelOptions?.funnelGroup;-+ breakdowns: initialBreakdowns = [],+ funnelWindow = 24,+ funnelGroup,+ timezone,+ }: {+ projectId: string;+ startDate: string;+ endDate: string;+ series: IReportInput['series'];+ globalFilters?: IReportInput['globalFilters'];+ breakdowns?: IChartBreakdown[];+ funnelWindow?: number;+ funnelGroup?: string;+ timezone: string;+ }) {// Drop breakdowns that don't resolve to a known events column, properties- // path, profile path, group path, or cohort. The funnel CTE inlines each- // breakdown's name directly via getSelectPropertyKey — a saved report with- // breakdown `cohort` (intended for the chart's all-cohorts feature; the- // funnel doesn't support it) used to produce `cohort as b_0 FROM events- // GROUP BY b_0`, failing parse.- breakdowns = breakdowns.filter((b) => isKnownEventField(b.name));+ // path, profile path, group path, or specific cohort. The funnel CTE+ // inlines each breakdown's name directly via getSelectPropertyKey, so+ // anything that doesn't resolve leaks into the SQL verbatim.+ //+ // `isKnownEventField` accepts the bare `cohort` breakdown because the+ // chart's all-cohorts feature uses it, but the funnel has no equivalent —+ // it renders as `cohort as b_0 FROM events`, which fails with+ // UNKNOWN_IDENTIFIER for the chart and the profile list alike. Exclude it+ // explicitly rather than relying on the generic check.+ const breakdowns = initialBreakdowns.filter(+ (b) => isKnownEventField(b.name) && !isAllCohortsBreakdown(b.name),+ );- const eventSeries = onlyReportEvents(series);+ const eventSeries = onlyReportEvents(+ mergeGlobalFilters(series, globalFilters),+ );if (eventSeries.length === 0) {throw new Error('events are required');}- const funnelWindowSeconds = funnelWindow * 3600;- const funnelWindowMilliseconds = funnelWindowSeconds * 1000;+ const funnelWindowMilliseconds = funnelWindow * 3600 * 1000;const group = this.getFunnelGroup(funnelGroup);+const profileFilters = this.getProfileFilters(eventSeries);const anyFilterOnProfile = profileFilters.length > 0;- const anyBreakdownOnProfile = breakdowns.some((b) =>+ const profileBreakdowns = breakdowns.filter((b) =>b.name.startsWith('profile.'),);const anyFilterOnGroup = eventSeries.some((e) =>@@ -264,13 +288,12 @@ export class FunnelService {const cohortIds = collectBreakdownCohortIds(breakdowns);const cohortMetadata = await fetchCohortsMetadata(cohortIds);- // Create the funnel CTE (session-level)const breakdownSelects = breakdowns.map((b, index) => {const bId = extractCohortId(b.name);const bName = bId ? cohortMetadata.get(bId)?.name : undefined;return `${getSelectPropertyKey(b.name, projectId, bId ?? undefined, bName)} as b_${index}`;});- const breakdownGroupBy = breakdowns.map((b, index) => `b_${index}`);+ const breakdownGroupBy = breakdowns.map((_, index) => `b_${index}`);const funnelCte = this.buildFunnelCte({projectId,@@ -284,13 +307,15 @@ export class FunnelService {group,});- if (anyFilterOnProfile || anyBreakdownOnProfile) {- // Collect profile columns needed for filters and breakdowns (same as conversion.service)+ // The profile join has to cover breakdowns as well as filters — a+ // `profile.*` breakdown renders `profile.properties[...]` into the select,+ // so the alias must exist in scope even when no filter touches profiles.+ if (anyFilterOnProfile || profileBreakdowns.length > 0) {const profileFields = new Set<string>(['id']);for (const f of profileFilters) {profileFields.add(f.split('.')[0]!);}- for (const b of breakdowns.filter((x) => x.name.startsWith('profile.'))) {+ for (const b of profileBreakdowns) {const fieldName = b.name.replace('profile.', '').split('.')[0];if (fieldName === 'properties') {profileFields.add('properties');@@ -319,21 +344,58 @@ export class FunnelService {funnelCte.rawJoin('LEFT ANY JOIN _g ON _g.id = _group_id');}+ // A cohort breakdown renders `cohort_<id>.profile_id`, so every cohort+ // referenced by a breakdown needs its join.for (const cohortId of cohortIds) {funnelCte.rawJoin(buildInlineCohortJoin(cohortId, projectId, 'events'));}- // Base funnel query with CTEs- const funnelQuery = clix(this.client, timezone);+ const query = clix(this.client, timezone);if (needsGroupArrayJoin) {- funnelQuery.with(+ query.with('_g',`SELECT id, name, type, properties FROM ${TABLE_NAMES.groups} FINAL WHERE project_id = ${sqlstring.escape(projectId)}`,);}- funnelQuery.with('session_funnel', funnelCte);+ query.with('session_funnel', funnelCte);++ return { query, eventSeries, breakdowns, group };+ }++ async getFunnel({+ projectId,+ startDate,+ endDate,+ series,+ globalFilters,+ options,+ breakdowns: initialBreakdowns = [],+ limit,+ timezone = 'UTC',+ }: IReportInput & { timezone: string; events?: IChartEvent[] }) {+ if (!startDate || !endDate) {+ throw new Error('startDate and endDate are required');+ }++ const funnelOptions = options?.type === 'funnel' ? options : undefined;++ const {+ query: funnelQuery,+ eventSeries,+ breakdowns,+ } = await this.buildFunnelBase({+ projectId,+ startDate,+ endDate,+ series,+ globalFilters,+ breakdowns: initialBreakdowns,+ funnelWindow: funnelOptions?.funnelWindow,+ funnelGroup: funnelOptions?.funnelGroup,+ timezone,+ });// windowFunnel is computed per the primary key (profile_id or session_id),// so we just filter out level=0 rows — no re-aggregation needed.diff --git a/packages/trpc/src/routers/chart.ts b/packages/trpc/src/routers/chart.tsindex fbf06d3a0..0c641e79e 100644--- a/packages/trpc/src/routers/chart.ts+++ b/packages/trpc/src/routers/chart.ts@@ -823,7 +823,7 @@ export const chartRouter = createTRPCRouter({showDropoffs = false,funnelWindow,funnelGroup,- breakdowns = [],+ breakdowns: inputBreakdowns = [],breakdownValues = [],} = input;@@ -832,94 +832,27 @@ export const chartRouter = createTRPCRouter({// stepIndex is 0-based, but level is 1-based, so we need level >= stepIndex + 1const targetLevel = stepIndex + 1;- const eventSeries = onlyReportEvents(series);-- if (eventSeries.length === 0) {- throw new Error('At least one event series is required');- }-- const funnelWindowSeconds = (funnelWindow || 24) * 3600;- const funnelWindowMilliseconds = funnelWindowSeconds * 1000;-- // Get the grouping strategy (profile_id or session_id)- const group = funnelService.getFunnelGroup(funnelGroup);-- const anyFilterOnGroup = (eventSeries as IChartEvent[]).some((e) =>- e.filters?.some((f) => f.name.startsWith('group.'))- );- const anyBreakdownOnGroup = breakdowns.some((b) =>- b.name.startsWith('group.')- );- const needsGroupArrayJoin = anyFilterOnGroup || anyBreakdownOnGroup;-- // Breakdown selects/groupBy so we can filter by specific breakdown values- const breakdownSelects = breakdowns.map(- (b, index) => `${getSelectPropertyKey(b.name, projectId)} as b_${index}`- );- const breakdownGroupBy = breakdowns.map((_, index) => `b_${index}`);-- // Create funnel CTE using funnel service- const funnelCte = funnelService.buildFunnelCte({+ // Reuse the chart's own funnel builder rather than re-deriving the CTE+ // here. The two copies used to drift — breakdown expressions referencing+ // a `profile` or `cohort_<id>` alias whose join this side never added,+ // which failed with UNKNOWN_IDENTIFIER and surfaced as "No users found".+ const { query, breakdowns } = await funnelService.buildFunnelBase({projectId,startDate,endDate,- eventSeries: eventSeries as IChartEvent[],- funnelWindowMilliseconds,+ series,+ breakdowns: inputBreakdowns,+ funnelWindow,+ funnelGroup,timezone,- additionalSelects: breakdownSelects,- additionalGroupBy: breakdownGroupBy,- group,});- // Profile JOIN must cover both profile filters AND profile breakdowns —- // breakdownSelects reference `profile.properties[...]` etc. via- // getSelectPropertyKey, so the alias has to exist in scope even when- // no filter touches the profiles table. Mirrors the same guard in- // funnelService.getFunnel.- const profileFilters = funnelService.getProfileFilters(- eventSeries as IChartEvent[]- );- const profileBreakdownNames = breakdowns- .filter((b) => b.name.startsWith('profile.'))- .map((b) => b.name.replace('profile.', ''));- if (profileFilters.length > 0 || profileBreakdownNames.length > 0) {- const fieldsToSelect = uniq([- ...profileFilters.map((f) => f.split('.')[0]!),- ...profileBreakdownNames.map((f) => f.split('.')[0]!),- ]).join(', ');- funnelCte.leftJoin(- `(SELECT id, ${fieldsToSelect} FROM ${TABLE_NAMES.profiles} FINAL WHERE project_id = ${sqlstring.escape(projectId)}) as profile`,- 'profile.id = events.profile_id'- );- }-- if (needsGroupArrayJoin) {- funnelCte.rawJoin('ARRAY JOIN groups AS _group_id');- funnelCte.rawJoin('LEFT ANY JOIN _g ON _g.id = _group_id');- }-- // Build main query- const query = clix(ch, timezone);- if (needsGroupArrayJoin) {- query.with(- '_g',- `SELECT id, name, type, properties FROM ${TABLE_NAMES.groups} FINAL WHERE project_id = ${sqlstring.escape(projectId)}`- );- }- query.with('session_funnel', funnelCte);-- if (group === 'profile_id') {- const breakdownAggregates =- breakdowns.length > 0- ? `, ${breakdowns.map((_, index) => `any(b_${index}) AS b_${index}`).join(', ')}`- : '';- query.with(- 'funnel',- `SELECT profile_id, max(level) AS level${breakdownAggregates} FROM (SELECT * FROM session_funnel WHERE level != 0) GROUP BY profile_id`- );- } else {- query.with('funnel', 'SELECT * FROM session_funnel WHERE level != 0');- }+ // Same shape as the chart's `funnel` CTE: windowFunnel is already+ // computed per primary key, so drop level=0 and select distinct+ // profiles. Re-aggregating with max(level)/any(b_0) here would collapse+ // the breakdown column and make the breakdownValues filter pick an+ // arbitrary value for profiles that appear under more than one.+ query.with('funnel', 'SELECT * FROM session_funnel WHERE level != 0');query.select(['DISTINCT profile_id']).from('funnel');