Skip to content

OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.

perf(funnel): pre-filter the funnel scan to rows matching a step (#437)

Ritik Jain769c0615b2fd
diff --git a/packages/db/src/services/funnel-sql.test.ts b/packages/db/src/services/funnel-sql.test.ts
index 36bb2c8e9..dbd95f1ba 100644
--- a/packages/db/src/services/funnel-sql.test.ts
+++ b/packages/db/src/services/funnel-sql.test.ts
@@ -22,6 +22,7 @@ vi.mock('../prisma-client', () => ({
import { ch } from '../clickhouse/client';
import { funnelService } from './funnel.service';
+import { onlyReportEvents } from './reports.service';
const PROJECT_ID = 'test-sql-validation';
const COHORT_ID = 'aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee';
@@ -212,6 +213,68 @@ describe('funnel.service / buildFunnelBase — unknown breakdowns', () => {
});
});
+describe('funnel.service / buildFunnelCte — step pre-filter', () => {
+ // Only rows matching at least one step can advance windowFunnel, so the
+ // funnel CTE must not feed rows that share a step's event name but fail
+ // its filters into the aggregation. The step conditions therefore appear
+ // twice: inside windowFunnel and as a row-level pre-filter.
+ // Each step's COMPLETE condition must appear exactly twice: once inside
+ // windowFunnel and once in the row-level pre-filter. Counting the full
+ // condition (not a substring) catches a regression that drops a step from
+ // either clause.
+ const countOccurrences = (sql: string, needle: string) =>
+ sql.split(needle).length - 1;
+
+ it('filters the scan to rows matching at least one step condition', async () => {
+ const sql = await buildChartSql([]);
+ expect(sql).toContain(
+ "((events.name = 'screen_view') OR (events.name = 'sign_up'))",
+ );
+ for (const condition of funnelService.getFunnelConditions(
+ // The same checked narrowing buildFunnelBase applies internally.
+ onlyReportEvents(SERIES),
+ PROJECT_ID,
+ )) {
+ expect(countOccurrences(sql, condition)).toBe(2);
+ }
+ });
+
+ it('includes each step\'s own filters in the pre-filter', async () => {
+ const series = [
+ event({
+ filters: [
+ { id: 'f', name: 'path', operator: 'is', value: ['/pricing'] },
+ ],
+ }),
+ event({ id: 'B', name: 'sign_up' }),
+ ];
+ const { query } = await funnelService.buildFunnelBase({
+ projectId: PROJECT_ID,
+ startDate: START,
+ endDate: END,
+ series,
+ funnelWindow: 24,
+ timezone: 'UTC',
+ });
+ query.with('funnel', 'SELECT * FROM session_funnel WHERE level != 0');
+ query.select(['DISTINCT profile_id']).from('funnel');
+ const sql = query.toSQL();
+ // The full filtered condition (path AND name) must gate the scan, not
+ // only the windowFunnel arm — and so must the unfiltered second step.
+ for (const condition of funnelService.getFunnelConditions(
+ onlyReportEvents(series),
+ PROJECT_ID,
+ )) {
+ expect(countOccurrences(sql, condition)).toBe(2);
+ }
+ });
+
+ itCH('pre-filtered funnel SQL still parses and resolves', async () => {
+ const sql = await buildProfilesSql([breakdown('profile.properties.plan')]);
+ await explain(sql);
+ });
+});
+
describe('funnel.service / buildFunnelBase — breakdown attribution', () => {
// The breakdown value must be read at the user's FIRST funnel step, not
// used as a windowFunnel GROUP BY key. Per-row grouping splits a user's
diff --git a/packages/db/src/services/funnel.service.ts b/packages/db/src/services/funnel.service.ts
index 9790de67b..f42c8cc64 100644
--- a/packages/db/src/services/funnel.service.ts
+++ b/packages/db/src/services/funnel.service.ts
@@ -104,6 +104,13 @@ export class FunnelService {
'IN',
eventSeries.map((e) => e.name),
)
+ // Only rows matching at least one step can advance windowFunnel, so
+ // rows that share a step's event name but fail its filters are dead
+ // weight — with filtered steps (e.g. screen_view + a path filter) they
+ // can be the vast majority of what the name IN(...) lets through.
+ // Dropping them here shrinks the aggregation input; windowFunnel
+ // ignores non-matching rows either way, so levels are unchanged.
+ .rawWhere(`(${funnels.map((f) => `(${f})`).join(' OR ')})`)
.groupBy([primaryKey, ...additionalGroupBy]);
}