OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.
perf(api): profile filters no longer join profiles in charts and funnels
Carl-Gerhard Lindesvärdaba280256ce0
diff --git a/packages/core/src/modules/chart/chart.service.test.ts b/packages/core/src/modules/chart/chart.service.test.tsindex 464136114..b0c79dcf1 100644--- a/packages/core/src/modules/chart/chart.service.test.ts+++ b/packages/core/src/modules/chart/chart.service.test.ts@@ -435,18 +435,8 @@ describe('getChartBucketProfiles', () => {projectId: TEST_PROJECT_ID,date: twoDaysAgo,interval: 'day',- series: [- pageView({- filters: [- {- id: 'p',- name: 'profile.evil; DROP TABLE profiles',- operator: 'is',- value: ['x'],- },- ],- }),- ],+ series: [pageView()],+ breakdowns: { 'profile.evil; DROP TABLE profiles': 'x' },})).rejects.toThrow(/Refusing to inline identifier/);});diff --git a/packages/core/src/modules/chart/src/filter-where.ts b/packages/core/src/modules/chart/src/filter-where.tsindex 7cd0cd630..18c9687cb 100644--- a/packages/core/src/modules/chart/src/filter-where.ts+++ b/packages/core/src/modules/chart/src/filter-where.ts@@ -39,6 +39,16 @@ export type CompiledEventFilters = Record<string, SqlFragment>;// anything outside the set by the time it gets here.const EVENT_COLUMN_ALLOWLIST = Array.from(EVENT_TOP_LEVEL_COLUMNS);+/**+ * Whether the compiled clause reads the caller's `profile` join. Only a+ * wildcard `profile.*` filter does; every other profile filter compiles to+ * its own id subquery below, so joining profiles for it is wasted work.+ */+export function filterReadsProfileJoin(filter: IChartEventFilter): boolean {+ const name = normalizeEventField(filter.name);+ return name.startsWith('profile.') && isWildcardPropertyKey(name);+}+function filterValue(value: unknown) {return sql.string(String(value).trim());}diff --git a/packages/core/src/modules/chart/src/funnel.sql.test.ts b/packages/core/src/modules/chart/src/funnel.sql.test.tsindex 776939e85..7d9b0aaaf 100644--- a/packages/core/src/modules/chart/src/funnel.sql.test.ts+++ b/packages/core/src/modules/chart/src/funnel.sql.test.ts@@ -209,7 +209,7 @@ describe('funnel.sql / buildFunnelBase — profile breakdowns', () => {await explain(statement);});- it('adds the join for a profile filter with no profile breakdown', async () => {+ it('does not join profiles for a profile filter with no profile breakdown', async () => {const statement = await profilesStatement({series: [event({@@ -225,7 +225,9 @@ describe('funnel.sql / buildFunnelBase — profile breakdowns', () => {event({ id: 'B', name: 'sign_up' }),],});- expect(render(statement).sql).toContain('profile.id = events.profile_id');+ const { sql } = render(statement);+ expect(sql).not.toContain('profile.id = events.profile_id');+ expect(sql).toContain('HAVING argMax(email, last_seen_at)');await explain(statement);});});@@ -419,9 +421,9 @@ describe('funnel.sql / profile-property narrowing', () => {});const { text } = render(statement);- expect(text).toContain(- "argMax(properties['plan'], last_seen_at) as `profile.properties.plan`"- );+ // The filter compiles to its own id subquery, so only the breakdown's+ // key is joined.+ expect(text).not.toContain('as `profile.properties.plan`');expect(text).toContain("argMax(properties['experiment'], last_seen_at) as `profile.properties.experiment`");diff --git a/packages/core/src/modules/chart/src/funnel.sql.ts b/packages/core/src/modules/chart/src/funnel.sql.tsindex fa87cb289..f1f6a8da9 100644--- a/packages/core/src/modules/chart/src/funnel.sql.ts+++ b/packages/core/src/modules/chart/src/funnel.sql.ts@@ -13,6 +13,7 @@ import { type SqlFragment, sql } from '@openpanel/db/src/clickhouse/sql';import type {IChartBreakdown,IChartEvent,+ IChartEventFilter,} from '../../report/report.constants';import { JOINABLE_PROFILE_COLUMNS } from '../chart.constants';import { compiledText, fragmentWithProfileRefs } from './compiled';@@ -31,7 +32,10 @@ import {profileIdsInWindow,profilePropertiesCteSelect,} from './field-resolution';-import { getEventFiltersWhereClause } from './filter-where';+import {+ filterReadsProfileJoin,+ getEventFiltersWhereClause,+} from './filter-where';/** The two keys windowFunnel may be computed per — a closed set. */const FUNNEL_GROUPS = ['session_id', 'profile_id'] as const;@@ -235,13 +239,10 @@ function profileCteColumns(return columns;}-/** The `profile.`-prefixed filter names, unprefixed. */-function profileFilterNames(eventSeries: IChartEvent[]): string[] {+/** Filters whose compiled clause reads the profile join. */+function profileJoinFilters(eventSeries: IChartEvent[]): IChartEventFilter[] {return eventSeries.flatMap(- (event) =>- event.filters- ?.filter((filter) => filter.name.startsWith('profile.'))- .map((filter) => filter.name.replace('profile.', '')) ?? []+ (event) => event.filters?.filter(filterReadsProfileJoin) ?? []);}@@ -252,7 +253,10 @@ export function funnelBase(input: FunnelBaseInput): FunnelBase {throw new Error('events are required');}- const profileFilters = profileFilterNames(eventSeries);+ const joinFilters = profileJoinFilters(eventSeries);+ const profileFilters = joinFilters.map((filter) =>+ filter.name.replace('profile.', '')+ );const anyFilterOnGroup = eventSeries.some((event) =>event.filters?.some((filter) => filter.name.startsWith('group.')));@@ -263,7 +267,7 @@ export function funnelBase(input: FunnelBaseInput): FunnelBase {anyFilterOnGroup || anyBreakdownOnGroup || input.funnelGroup === 'group';const profileKeys = collectProfilePropertyKeys([- ...eventSeries.flatMap((event) => event.filters ?? []),+ ...joinFilters,...breakdowns,]);diff --git a/packages/core/src/modules/chart/src/sql.test.ts b/packages/core/src/modules/chart/src/sql.test.tsindex 3c93c5d6d..c33003705 100644--- a/packages/core/src/modules/chart/src/sql.test.ts+++ b/packages/core/src/modules/chart/src/sql.test.ts@@ -455,12 +455,14 @@ describe('profile-property narrowing', () => {...base,});- // The CTE selects one scalar column per referenced key...+ // The CTE selects one scalar column per key the join is read for...expect(text).toContain(- "argMax(properties['plan'], last_seen_at) as `profile.properties.plan`"+ "argMax(properties['experiment'], last_seen_at) as `profile.properties.experiment`");+ // ...not for the filter, which compiles to its own id subquery.+ expect(text).not.toContain('as `profile.properties.plan`');expect(text).toContain(- "argMax(properties['experiment'], last_seen_at) as `profile.properties.experiment`"+ "HAVING argMax(properties['plan'], last_seen_at) = 'pro'");// ...instead of every profile's whole Map...expect(text).not.toContain('as "profile.properties"');@@ -469,6 +471,20 @@ describe('profile-property narrowing', () => {expect(text).not.toContain("profile.properties['experiment']");});+ it('does not join profiles for a profile filter alone', async () => {+ const { text } = await getChartSql({+ event: event({ filters: [profileFilter] }),+ breakdowns: [],+ ...base,+ });++ expect(text).not.toContain('LEFT ANY JOIN profile');+ expect(text).not.toContain('WITH profile AS');+ expect(text).toContain(+ "HAVING argMax(properties['plan'], last_seen_at) = 'pro'"+ );+ });+it('falls back to the full Map for wildcard refs', async () => {const { text } = await getChartSql({event: event({@@ -496,17 +512,8 @@ describe('profile-property narrowing', () => {it('never narrows identifier-unsafe keys (backtick falls back to the Map)', async () => {const { text } = await getChartSql({- event: event({- filters: [- {- id: 'f1',- name: 'profile.properties.plan`tier',- operator: 'is' as const,- value: ['a'],- },- ],- }),- breakdowns: [],+ event: event(),+ breakdowns: [breakdown('profile.properties.plan`tier')],...base,});diff --git a/packages/core/src/modules/chart/src/sql.ts b/packages/core/src/modules/chart/src/sql.tsindex 0f665bee6..79dbe4413 100644--- a/packages/core/src/modules/chart/src/sql.ts+++ b/packages/core/src/modules/chart/src/sql.ts@@ -42,7 +42,10 @@ import {profileIdsInWindow,profilePropertiesCteSelect,} from './field-resolution';-import { getEventFiltersWhereClause } from './filter-where';+import {+ filterReadsProfileJoin,+ getEventFiltersWhereClause,+} from './filter-where';const EVENTS_ALIAS = 'e';const ALL_COHORTS_ALIAS = '_all_cohorts';@@ -263,11 +266,12 @@ function chartBody({const metricRef = event.property ? [{ name: event.property }] : [];// Math metrics reference event.property too — missing it here would strip// the Map the metric still reads from.- const profileProps = collectProfilePropertyKeys([- ...event.filters,+ const profileJoinRefs = [+ ...event.filters.filter(filterReadsProfileJoin),...breakdowns,...metricRef,- ]);+ ];+ const profileProps = collectProfilePropertyKeys(profileJoinRefs);const ctes: Parts = {};const joins: Parts = {};@@ -306,10 +310,10 @@ function chartBody({const dateRange = dateRangeWhere(startDate, endDate);Object.assign(where, dateRange);- const profileRefs = [...event.filters, ...breakdowns, ...metricRef];- const needsProfileJoin = anyRefOn('profile.', profileRefs);+ const needsProfileJoin = anyRefOn('profile.', profileJoinRefs);const needsGroupArrayJoin =- anyRefOn('group.', profileRefs) || event.segment === 'group';+ anyRefOn('group.', [...event.filters, ...breakdowns, ...metricRef]) ||+ event.segment === 'group';if (needsGroupArrayJoin) {ctes._g = buildGroupsQuery(projectId);@@ -318,7 +322,7 @@ function chartBody({}if (needsProfileJoin) {- const selectFields = collectProfileCteFields(profileRefs).map((field) =>+ const selectFields = collectProfileCteFields(profileJoinRefs).map((field) =>profileCteSelectField(field, profileProps.keys, profileProps.needsFullMap));ctes.profile = profileCte(@@ -713,7 +717,9 @@ export function chartBucketProfilesQuery(const joins: Parts = {};const profileRefs = [- ...event.filters.map((filter) => filter.name),+ ...event.filters+ .filter(filterReadsProfileJoin)+ .map((filter) => filter.name),...breakdownKeys,].filter((name) => name.startsWith('profile.'));if (profileRefs.length > 0) {