Skip to content

OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.

perf(api): profile filters no longer join profiles in charts and funnels

Carl-Gerhard Lindesvärdaba280256ce0
diff --git a/packages/core/src/modules/chart/chart.service.test.ts b/packages/core/src/modules/chart/chart.service.test.ts
index 464136114..b0c79dcf1 100644
--- a/packages/core/src/modules/chart/chart.service.test.ts
+++ b/packages/core/src/modules/chart/chart.service.test.ts
@@ -435,18 +435,8 @@ describe('getChartBucketProfiles', () => {
projectId: TEST_PROJECT_ID,
date: twoDaysAgo,
interval: 'day',
- series: [
- pageView({
- filters: [
- {
- id: 'p',
- name: 'profile.evil; DROP TABLE profiles',
- operator: 'is',
- value: ['x'],
- },
- ],
- }),
- ],
+ series: [pageView()],
+ breakdowns: { 'profile.evil; DROP TABLE profiles': 'x' },
})
).rejects.toThrow(/Refusing to inline identifier/);
});
diff --git a/packages/core/src/modules/chart/src/filter-where.ts b/packages/core/src/modules/chart/src/filter-where.ts
index 7cd0cd630..18c9687cb 100644
--- a/packages/core/src/modules/chart/src/filter-where.ts
+++ b/packages/core/src/modules/chart/src/filter-where.ts
@@ -39,6 +39,16 @@ export type CompiledEventFilters = Record<string, SqlFragment>;
// anything outside the set by the time it gets here.
const EVENT_COLUMN_ALLOWLIST = Array.from(EVENT_TOP_LEVEL_COLUMNS);
+/**
+ * Whether the compiled clause reads the caller's `profile` join. Only a
+ * wildcard `profile.*` filter does; every other profile filter compiles to
+ * its own id subquery below, so joining profiles for it is wasted work.
+ */
+export function filterReadsProfileJoin(filter: IChartEventFilter): boolean {
+ const name = normalizeEventField(filter.name);
+ return name.startsWith('profile.') && isWildcardPropertyKey(name);
+}
+
function filterValue(value: unknown) {
return sql.string(String(value).trim());
}
diff --git a/packages/core/src/modules/chart/src/funnel.sql.test.ts b/packages/core/src/modules/chart/src/funnel.sql.test.ts
index 776939e85..7d9b0aaaf 100644
--- a/packages/core/src/modules/chart/src/funnel.sql.test.ts
+++ b/packages/core/src/modules/chart/src/funnel.sql.test.ts
@@ -209,7 +209,7 @@ describe('funnel.sql / buildFunnelBase — profile breakdowns', () => {
await explain(statement);
});
- it('adds the join for a profile filter with no profile breakdown', async () => {
+ it('does not join profiles for a profile filter with no profile breakdown', async () => {
const statement = await profilesStatement({
series: [
event({
@@ -225,7 +225,9 @@ describe('funnel.sql / buildFunnelBase — profile breakdowns', () => {
event({ id: 'B', name: 'sign_up' }),
],
});
- expect(render(statement).sql).toContain('profile.id = events.profile_id');
+ const { sql } = render(statement);
+ expect(sql).not.toContain('profile.id = events.profile_id');
+ expect(sql).toContain('HAVING argMax(email, last_seen_at)');
await explain(statement);
});
});
@@ -419,9 +421,9 @@ describe('funnel.sql / profile-property narrowing', () => {
});
const { text } = render(statement);
- expect(text).toContain(
- "argMax(properties['plan'], last_seen_at) as `profile.properties.plan`"
- );
+ // The filter compiles to its own id subquery, so only the breakdown's
+ // key is joined.
+ expect(text).not.toContain('as `profile.properties.plan`');
expect(text).toContain(
"argMax(properties['experiment'], last_seen_at) as `profile.properties.experiment`"
);
diff --git a/packages/core/src/modules/chart/src/funnel.sql.ts b/packages/core/src/modules/chart/src/funnel.sql.ts
index fa87cb289..f1f6a8da9 100644
--- a/packages/core/src/modules/chart/src/funnel.sql.ts
+++ b/packages/core/src/modules/chart/src/funnel.sql.ts
@@ -13,6 +13,7 @@ import { type SqlFragment, sql } from '@openpanel/db/src/clickhouse/sql';
import type {
IChartBreakdown,
IChartEvent,
+ IChartEventFilter,
} from '../../report/report.constants';
import { JOINABLE_PROFILE_COLUMNS } from '../chart.constants';
import { compiledText, fragmentWithProfileRefs } from './compiled';
@@ -31,7 +32,10 @@ import {
profileIdsInWindow,
profilePropertiesCteSelect,
} from './field-resolution';
-import { getEventFiltersWhereClause } from './filter-where';
+import {
+ filterReadsProfileJoin,
+ getEventFiltersWhereClause,
+} from './filter-where';
/** The two keys windowFunnel may be computed per — a closed set. */
const FUNNEL_GROUPS = ['session_id', 'profile_id'] as const;
@@ -235,13 +239,10 @@ function profileCteColumns(
return columns;
}
-/** The `profile.`-prefixed filter names, unprefixed. */
-function profileFilterNames(eventSeries: IChartEvent[]): string[] {
+/** Filters whose compiled clause reads the profile join. */
+function profileJoinFilters(eventSeries: IChartEvent[]): IChartEventFilter[] {
return eventSeries.flatMap(
- (event) =>
- event.filters
- ?.filter((filter) => filter.name.startsWith('profile.'))
- .map((filter) => filter.name.replace('profile.', '')) ?? []
+ (event) => event.filters?.filter(filterReadsProfileJoin) ?? []
);
}
@@ -252,7 +253,10 @@ export function funnelBase(input: FunnelBaseInput): FunnelBase {
throw new Error('events are required');
}
- const profileFilters = profileFilterNames(eventSeries);
+ const joinFilters = profileJoinFilters(eventSeries);
+ const profileFilters = joinFilters.map((filter) =>
+ filter.name.replace('profile.', '')
+ );
const anyFilterOnGroup = eventSeries.some((event) =>
event.filters?.some((filter) => filter.name.startsWith('group.'))
);
@@ -263,7 +267,7 @@ export function funnelBase(input: FunnelBaseInput): FunnelBase {
anyFilterOnGroup || anyBreakdownOnGroup || input.funnelGroup === 'group';
const profileKeys = collectProfilePropertyKeys([
- ...eventSeries.flatMap((event) => event.filters ?? []),
+ ...joinFilters,
...breakdowns,
]);
diff --git a/packages/core/src/modules/chart/src/sql.test.ts b/packages/core/src/modules/chart/src/sql.test.ts
index 3c93c5d6d..c33003705 100644
--- a/packages/core/src/modules/chart/src/sql.test.ts
+++ b/packages/core/src/modules/chart/src/sql.test.ts
@@ -455,12 +455,14 @@ describe('profile-property narrowing', () => {
...base,
});
- // The CTE selects one scalar column per referenced key...
+ // The CTE selects one scalar column per key the join is read for...
expect(text).toContain(
- "argMax(properties['plan'], last_seen_at) as `profile.properties.plan`"
+ "argMax(properties['experiment'], last_seen_at) as `profile.properties.experiment`"
);
+ // ...not for the filter, which compiles to its own id subquery.
+ expect(text).not.toContain('as `profile.properties.plan`');
expect(text).toContain(
- "argMax(properties['experiment'], last_seen_at) as `profile.properties.experiment`"
+ "HAVING argMax(properties['plan'], last_seen_at) = 'pro'"
);
// ...instead of every profile's whole Map...
expect(text).not.toContain('as "profile.properties"');
@@ -469,6 +471,20 @@ describe('profile-property narrowing', () => {
expect(text).not.toContain("profile.properties['experiment']");
});
+ it('does not join profiles for a profile filter alone', async () => {
+ const { text } = await getChartSql({
+ event: event({ filters: [profileFilter] }),
+ breakdowns: [],
+ ...base,
+ });
+
+ expect(text).not.toContain('LEFT ANY JOIN profile');
+ expect(text).not.toContain('WITH profile AS');
+ expect(text).toContain(
+ "HAVING argMax(properties['plan'], last_seen_at) = 'pro'"
+ );
+ });
+
it('falls back to the full Map for wildcard refs', async () => {
const { text } = await getChartSql({
event: event({
@@ -496,17 +512,8 @@ describe('profile-property narrowing', () => {
it('never narrows identifier-unsafe keys (backtick falls back to the Map)', async () => {
const { text } = await getChartSql({
- event: event({
- filters: [
- {
- id: 'f1',
- name: 'profile.properties.plan`tier',
- operator: 'is' as const,
- value: ['a'],
- },
- ],
- }),
- breakdowns: [],
+ event: event(),
+ breakdowns: [breakdown('profile.properties.plan`tier')],
...base,
});
diff --git a/packages/core/src/modules/chart/src/sql.ts b/packages/core/src/modules/chart/src/sql.ts
index 0f665bee6..79dbe4413 100644
--- a/packages/core/src/modules/chart/src/sql.ts
+++ b/packages/core/src/modules/chart/src/sql.ts
@@ -42,7 +42,10 @@ import {
profileIdsInWindow,
profilePropertiesCteSelect,
} from './field-resolution';
-import { getEventFiltersWhereClause } from './filter-where';
+import {
+ filterReadsProfileJoin,
+ getEventFiltersWhereClause,
+} from './filter-where';
const EVENTS_ALIAS = 'e';
const ALL_COHORTS_ALIAS = '_all_cohorts';
@@ -263,11 +266,12 @@ function chartBody({
const metricRef = event.property ? [{ name: event.property }] : [];
// Math metrics reference event.property too — missing it here would strip
// the Map the metric still reads from.
- const profileProps = collectProfilePropertyKeys([
- ...event.filters,
+ const profileJoinRefs = [
+ ...event.filters.filter(filterReadsProfileJoin),
...breakdowns,
...metricRef,
- ]);
+ ];
+ const profileProps = collectProfilePropertyKeys(profileJoinRefs);
const ctes: Parts = {};
const joins: Parts = {};
@@ -306,10 +310,10 @@ function chartBody({
const dateRange = dateRangeWhere(startDate, endDate);
Object.assign(where, dateRange);
- const profileRefs = [...event.filters, ...breakdowns, ...metricRef];
- const needsProfileJoin = anyRefOn('profile.', profileRefs);
+ const needsProfileJoin = anyRefOn('profile.', profileJoinRefs);
const needsGroupArrayJoin =
- anyRefOn('group.', profileRefs) || event.segment === 'group';
+ anyRefOn('group.', [...event.filters, ...breakdowns, ...metricRef]) ||
+ event.segment === 'group';
if (needsGroupArrayJoin) {
ctes._g = buildGroupsQuery(projectId);
@@ -318,7 +322,7 @@ function chartBody({
}
if (needsProfileJoin) {
- const selectFields = collectProfileCteFields(profileRefs).map((field) =>
+ const selectFields = collectProfileCteFields(profileJoinRefs).map((field) =>
profileCteSelectField(field, profileProps.keys, profileProps.needsFullMap)
);
ctes.profile = profileCte(
@@ -713,7 +717,9 @@ export function chartBucketProfilesQuery(
const joins: Parts = {};
const profileRefs = [
- ...event.filters.map((filter) => filter.name),
+ ...event.filters
+ .filter(filterReadsProfileJoin)
+ .map((filter) => filter.name),
...breakdownKeys,
].filter((name) => name.startsWith('profile.'));
if (profileRefs.length > 0) {