OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.
perf(charts): narrow profile joins to the referenced property keys (#447)
Ritik Jain51eda3ca3835
diff --git a/packages/db/src/services/chart-sql.test.ts b/packages/db/src/services/chart-sql.test.tsindex 7a7ecf2f3..64f286d13 100644--- a/packages/db/src/services/chart-sql.test.ts+++ b/packages/db/src/services/chart-sql.test.ts@@ -464,3 +464,188 @@ describe('overview.service / getRawWhereClause (UTM remapping)', () => {);});});++describe('chart.service / profile-property narrowing', () => {+ const profileFilter = {+ id: 'f1',+ name: 'profile.properties.plan',+ operator: 'is' as const,+ value: ['pro'],+ };++ it('projects only the referenced keys as scalar columns in the profile CTE', async () => {+ const sql = await getChartSql({+ event: event({ filters: [profileFilter] }),+ breakdowns: [breakdown('profile.properties.experiment')],+ interval: 'day',+ startDate: START,+ endDate: END,+ projectId: PROJECT_ID,+ timezone: 'UTC',+ });++ // The CTE selects one scalar column per referenced key...+ expect(sql).toContain("properties['plan'] as `profile.properties.plan`");+ expect(sql).toContain(+ "properties['experiment'] as `profile.properties.experiment`",+ );+ // ...instead of every profile's whole Map...+ expect(sql).not.toContain('properties as "profile.properties"');+ // ...and every ref in the query is rewritten to the scalar alias.+ expect(sql).not.toContain("profile.properties['plan']");+ expect(sql).not.toContain("profile.properties['experiment']");+ });++ it('falls back to the full Map for wildcard refs', async () => {+ const sql = await getChartSql({+ event: event({+ filters: [+ {+ id: 'f1',+ name: 'profile.properties.experiments.*.name',+ operator: 'is' as const,+ value: ['a'],+ },+ ],+ }),+ breakdowns: [],+ interval: 'day',+ startDate: START,+ endDate: END,+ projectId: PROJECT_ID,+ timezone: 'UTC',+ });++ // mapExtractKeyLike needs the whole Map, so it must stay selected.+ expect(sql).toContain('properties as "profile.properties"');+ });++ it('never narrows identifier-unsafe keys (backtick falls back to the Map)', async () => {+ const sql = await getChartSql({+ event: event({+ filters: [+ {+ id: 'f1',+ name: 'profile.properties.plan`tier',+ operator: 'is' as const,+ value: ['a'],+ },+ ],+ }),+ breakdowns: [],+ interval: 'day',+ startDate: START,+ endDate: END,+ projectId: PROJECT_ID,+ timezone: 'UTC',+ });++ // The unsafe key keeps its original Map access against the full Map —+ // it must never be embedded in a backtick-quoted alias.+ expect(sql).toContain('properties as "profile.properties"');+ expect(sql).not.toContain('as `profile.properties.plan`tier`');+ });++ it('collects the math-metric property too', async () => {+ const sql = await getChartSql({+ event: event({+ segment: 'property_average',+ property: 'profile.properties.age',+ filters: [profileFilter],+ }),+ breakdowns: [],+ interval: 'day',+ startDate: START,+ endDate: END,+ projectId: PROJECT_ID,+ timezone: 'UTC',+ });++ // The metric's key must be narrowed alongside the filter's — otherwise+ // the metric keeps reading the Map that narrowing just removed.+ expect(sql).toContain("properties['age'] as `profile.properties.age`");+ expect(sql).not.toContain("profile.properties['age']");+ });++ it('creates the profile join for a metric-only profile property', async () => {+ // No profile filter or breakdown — the metric alone must still create+ // the CTE and join, or its scalar alias resolves against nothing.+ const sql = await getChartSql({+ event: event({+ segment: 'property_average',+ property: 'profile.properties.age',+ filters: [],+ }),+ breakdowns: [],+ interval: 'day',+ startDate: START,+ endDate: END,+ projectId: PROJECT_ID,+ timezone: 'UTC',+ });++ expect(sql).toContain('LEFT ANY JOIN profile ON profile.id = profile_id');+ expect(sql).toContain("properties['age'] as `profile.properties.age`");+ });++ itCH('metric-only profile property parses and resolves', async () => {+ const sql = await getChartSql({+ event: event({+ segment: 'property_average',+ property: 'profile.properties.age',+ filters: [],+ }),+ breakdowns: [],+ interval: 'day',+ startDate: START,+ endDate: END,+ projectId: PROJECT_ID,+ timezone: 'UTC',+ });+ await explain(sql);+ });++ itCH('math metric on a narrowed profile property parses and resolves', async () => {+ const sql = await getChartSql({+ event: event({+ segment: 'property_average',+ property: 'profile.properties.age',+ filters: [profileFilter],+ }),+ breakdowns: [],+ interval: 'day',+ startDate: START,+ endDate: END,+ projectId: PROJECT_ID,+ timezone: 'UTC',+ });+ await explain(sql);+ });++ itCH('narrowed chart SQL parses and resolves', async () => {+ const sql = await getChartSql({+ event: event({ filters: [profileFilter] }),+ breakdowns: [breakdown('profile.properties.experiment')],+ interval: 'day',+ startDate: START,+ endDate: END,+ projectId: PROJECT_ID,+ timezone: 'UTC',+ });+ await explain(sql);+ });++ itCH('narrowed aggregate chart SQL parses and resolves', async () => {+ const sql = await getAggregateChartSql({+ event: event({ filters: [profileFilter] }),+ breakdowns: [breakdown('profile.properties.experiment')],+ interval: 'day',+ startDate: START,+ endDate: END,+ projectId: PROJECT_ID,+ timezone: 'UTC',+ });+ expect(sql).not.toContain("profile.properties['plan']");+ await explain(sql);+ });+});diff --git a/packages/db/src/services/chart.service.ts b/packages/db/src/services/chart.service.tsindex ab00f319b..59dff37cd 100644--- a/packages/db/src/services/chart.service.ts+++ b/packages/db/src/services/chart.service.ts@@ -380,6 +380,76 @@ export function getSelectPropertyKey(return `${aliasPrefix}${match}['${property.replace(new RegExp(`^${match}.`), '')}']`;}++// --- profile-property CTE narrowing (perf) ---------------------------------+// profile.properties.<key> refs render as Map lookups `profile.properties['<key>']`.+// Pulling the whole `properties` Map into the profile CTE makes the LEFT ANY+// JOIN hash carry the full Map per profile — roughly a kilobyte each on real+// data — and OOMs at scale. Instead we project ONLY the referenced keys as+// scalar columns in the CTE and rewrite the refs to those columns — identical+// results at a fraction of the memory. Wildcard refs (mapExtractKeyLike) still+// need the full Map, so those fall back to selecting it.++const PROFILE_PROP_PREFIX = 'profile.properties.';++export function collectProfilePropertyKeys(refs: { name: string }[]): {+ keys: string[];+ needsFullMap: boolean;+} {+ const keys = new Set<string>();+ let needsFullMap = false;+ for (const { name } of refs) {+ if (!name.startsWith(PROFILE_PROP_PREFIX)) {+ continue;+ }+ // Wildcard refs render as mapExtractKeyLike over the whole Map.+ if (name.includes('*')) {+ needsFullMap = true;+ continue;+ }+ const key = name.slice(PROFILE_PROP_PREFIX.length);+ // A backtick or backslash in the key can't be embedded in the+ // backtick-quoted scalar alias. Such keys are never narrowed: the full+ // Map stays selected and their refs keep the original Map access.+ if (/[`\\]/.test(key)) {+ needsFullMap = true;+ continue;+ }+ keys.add(key);+ }+ return { keys: Array.from(keys), needsFullMap };+}++// The profile-CTE SELECT expression for the `properties` field: one scalar+// column per referenced key, plus the full Map only when a wildcard ref needs+// it (or when nothing specific was referenced).+export function profilePropertiesCteSelect(+ keys: string[],+ needsFullMap: boolean,+): string {+ const cols = keys.map(+ (k) => `properties[${sqlstring.escape(k)}] as \`profile.properties.${k}\``,+ );+ if (needsFullMap || cols.length === 0) {+ cols.push('properties as "profile.properties"');+ }+ return cols.join(', ');+}++// Rewrite `profile.properties['<key>']` -> `` `profile.properties.<key>` ``+// for the narrowed keys. Matches the raw render from getSelectPropertyKey /+// the filter builders; never matches the CTE's own `properties['<key>']`,+// which has no `profile.` prefix. No-op when keys is empty.+export function rewriteProfilePropertyRefs(sql: string, keys: string[]): string {+ let out = sql;+ for (const k of keys) {+ out = out+ .split(`profile.properties['${k}']`)+ .join(`\`profile.properties.${k}\``);+ }+ return out;+}+export async function getChartSql({event,breakdowns: initialBreakdowns,@@ -428,6 +498,14 @@ export async function getChartSql({const cohortIds = collectBreakdownCohortIds(breakdowns);const cohortMetadata = await fetchCohortsMetadata(cohortIds);+ const profileProps = collectProfilePropertyKeys([+ ...event.filters,+ ...breakdowns,+ // Math metrics (property_sum/avg/min/max) reference event.property too —+ // missing it here would strip the Map the metric still reads from.+ ...(event.property ? [{ name: event.property }] : []),+ ]);+// Add CTE + JOIN for "all cohorts" breakdownif (hasAllCohortsBreakdown) {addCte('_all_cohorts', buildAllCohortsMembershipQuery(projectId));@@ -461,6 +539,10 @@ export async function getChartSql({const anyBreakdownOnProfile = breakdowns.some((breakdown) =>breakdown.name.startsWith('profile.'));+ // Math metrics (property_sum/avg/min/max) can target a profile property+ // too — the join must exist for the metric alone, not only for filters+ // and breakdowns.+ const anyMetricOnProfile = !!event.property?.startsWith('profile.');const anyFilterOnGroup = event.filters.some((filter) =>filter.name.startsWith('group.'));@@ -543,24 +625,46 @@ export async function getChartSql({}});+ // Collect from the math metric+ if (event.property?.startsWith('profile.')) {+ const fieldName = event.property.replace('profile.', '').split('.')[0];+ if (fieldName && fieldName === 'properties') {+ fields.add('properties');+ } else if (+ fieldName &&+ [+ 'email',+ 'first_name',+ 'last_name',+ 'created_at',+ 'last_seen_at',+ ].includes(fieldName)+ ) {+ fields.add(fieldName);+ }+ }+return Array.from(fields);};// Create profiles CTE if profiles are needed (to avoid duplicating the heavy profile join)// Only select the fields that are actually usedconst profilesJoinRef =- anyFilterOnProfile || anyBreakdownOnProfile+ anyFilterOnProfile || anyBreakdownOnProfile || anyMetricOnProfile? 'LEFT ANY JOIN profile ON profile.id = profile_id': '';- if (anyFilterOnProfile || anyBreakdownOnProfile) {+ if (anyFilterOnProfile || anyBreakdownOnProfile || anyMetricOnProfile) {const profileFields = getProfileFields();const selectFields = profileFields.map((field) => {if (field === 'id') {return 'id as "profile.id"';}if (field === 'properties') {- return 'properties as "profile.properties"';+ return profilePropertiesCteSelect(+ profileProps.keys,+ profileProps.needsFullMap,+ );}if (field === 'email') {return 'email as "profile.email"';@@ -719,7 +823,10 @@ export async function getChartSql({// "Unknown identifier `e.name`". Clear it.sb.where = {};- const sql = `${getWith()}${getSelect()} ${getFrom()} ${getJoins()} ${getWhere()} ${getGroupBy()} ${getOrderBy()} ${getFill()}`;+ const sql = rewriteProfilePropertyRefs(+ `${getWith()}${getSelect()} ${getFrom()} ${getJoins()} ${getWhere()} ${getGroupBy()} ${getOrderBy()} ${getFill()}`,+ profileProps.keys,+ );console.log('-- Report --');console.log(sql.replaceAll(/[\n\r]/g, ' '));console.log('-- End --');@@ -805,7 +912,10 @@ export async function getChartSql({'(SELECT total_count FROM _uc) as total_count';}- const sql = `${getWith()}${getSelect()} ${getFrom()} ${getJoins()} ${getWhere()} ${getGroupBy()} ${getOrderBy()} ${getFill()}`;+ const sql = rewriteProfilePropertyRefs(+ `${getWith()}${getSelect()} ${getFrom()} ${getJoins()} ${getWhere()} ${getGroupBy()} ${getOrderBy()} ${getFill()}`,+ profileProps.keys,+ );console.log('-- Report --');console.log(sql.replaceAll(/[\n\r]/g, ' '));console.log('-- End --');@@ -846,6 +956,14 @@ export async function getAggregateChartSql({const cohortIds = collectBreakdownCohortIds(breakdowns);const cohortMetadata = await fetchCohortsMetadata(cohortIds);+ const profileProps = collectProfilePropertyKeys([+ ...event.filters,+ ...breakdowns,+ // Math metrics (property_sum/avg/min/max) reference event.property too —+ // missing it here would strip the Map the metric still reads from.+ ...(event.property ? [{ name: event.property }] : []),+ ]);+// Add CTE + JOIN for "all cohorts" breakdownif (hasAllCohortsBreakdown) {addCte('_all_cohorts', buildAllCohortsMembershipQuery(projectId));@@ -879,6 +997,10 @@ export async function getAggregateChartSql({const anyBreakdownOnProfile = breakdowns.some((breakdown) =>breakdown.name.startsWith('profile.'));+ // Math metrics (property_sum/avg/min/max) can target a profile property+ // too — the join must exist for the metric alone, not only for filters+ // and breakdowns.+ const anyMetricOnProfile = !!event.property?.startsWith('profile.');const anyFilterOnGroup = event.filters.some((filter) =>filter.name.startsWith('group.'));@@ -950,23 +1072,45 @@ export async function getAggregateChartSql({}});+ // Collect from the math metric+ if (event.property?.startsWith('profile.')) {+ const fieldName = event.property.replace('profile.', '').split('.')[0];+ if (fieldName && fieldName === 'properties') {+ fields.add('properties');+ } else if (+ fieldName &&+ [+ 'email',+ 'first_name',+ 'last_name',+ 'created_at',+ 'last_seen_at',+ ].includes(fieldName)+ ) {+ fields.add(fieldName);+ }+ }+return Array.from(fields);};// Create profiles CTE if profiles are neededconst profilesJoinRef =- anyFilterOnProfile || anyBreakdownOnProfile+ anyFilterOnProfile || anyBreakdownOnProfile || anyMetricOnProfile? 'LEFT ANY JOIN profile ON profile.id = profile_id': '';- if (anyFilterOnProfile || anyBreakdownOnProfile) {+ if (anyFilterOnProfile || anyBreakdownOnProfile || anyMetricOnProfile) {const profileFields = getProfileFields();const selectFields = profileFields.map((field) => {if (field === 'id') {return 'id as "profile.id"';}if (field === 'properties') {- return 'properties as "profile.properties"';+ return profilePropertiesCteSelect(+ profileProps.keys,+ profileProps.needsFullMap,+ );}if (field === 'email') {return 'email as "profile.email"';@@ -1086,7 +1230,7 @@ export async function getAggregateChartSql({) as subQuery`;sb.joins = {};- const sql = getSql();+ const sql = rewriteProfilePropertyRefs(getSql(), profileProps.keys);console.log('-- Aggregate Chart --');console.log(sql.replaceAll(/[\n\r]/g, ' '));console.log('-- End --');@@ -1101,7 +1245,7 @@ export async function getAggregateChartSql({sb.limit = limit;}- const sql = getSql();+ const sql = rewriteProfilePropertyRefs(getSql(), profileProps.keys);console.log('-- Aggregate Chart --');console.log(sql.replaceAll(/[\n\r]/g, ' '));console.log('-- End --');diff --git a/packages/db/src/services/funnel-sql.test.ts b/packages/db/src/services/funnel-sql.test.tsindex 737c6f23a..5bd23f0ab 100644--- a/packages/db/src/services/funnel-sql.test.ts+++ b/packages/db/src/services/funnel-sql.test.ts@@ -122,8 +122,10 @@ describe('funnel.service / buildFunnelBase — profile breakdowns', () => {itCH('adds the profiles join for a profile.properties breakdown', async () => {const sql = await buildProfilesSql([breakdown('profile.properties.plan')]);- // The breakdown renders `profile.properties['plan']`, so the alias must exist.- expect(sql).toContain("profile.properties['plan']");+ // The breakdown is narrowed to a scalar alias selected by the profile+ // join, so both the join and the aliased column must exist.+ expect(sql).toContain("properties['plan'] as `profile.properties.plan`");+ expect(sql).toContain('`profile.properties.plan`');expect(sql).toMatch(/as profile/);expect(sql).toContain('profile.id = events.profile_id');await explain(sql);@@ -203,10 +205,10 @@ describe('funnel.service / buildFunnelBase — unknown breakdowns', () => {// Both sides must resolve the same breakdown to the same b_N index,// otherwise the breakdownValues filter targets the wrong column.expect(chartSql).toContain(- "argMinIf(profile.properties['plan'], created_at,",+ 'argMinIf(`profile.properties.plan`, created_at,',);expect(profilesSql).toContain(- "argMinIf(profile.properties['plan'], created_at,",+ 'argMinIf(`profile.properties.plan`, created_at,',);await explain(chartSql);await explain(profilesSql);@@ -345,3 +347,73 @@ describe('funnel.service / buildFunnelBase — group breakdowns', () => {await explain(sql);});});++describe('funnel.service / profile-property narrowing', () => {+ const seriesWithProfileFilter = [+ event({+ filters: [+ {+ id: 'f1',+ name: 'profile.properties.plan',+ operator: 'is' as const,+ value: ['pro'],+ },+ ],+ }),+ event({ id: 'B', name: 'sign_up' }),+ ];++ it('joins scalar columns instead of the whole properties Map', async () => {+ const { query } = await funnelService.buildFunnelBase({+ projectId: PROJECT_ID,+ startDate: START,+ endDate: END,+ series: seriesWithProfileFilter,+ breakdowns: [breakdown('profile.properties.experiment')],+ funnelWindow: 24,+ timezone: 'UTC',+ });+ query.with('funnel', 'SELECT * FROM session_funnel WHERE level != 0');+ query.select(['DISTINCT profile_id']).from('funnel');+ const sql = query.toSQL();++ expect(sql).toContain("properties['plan'] as `profile.properties.plan`");+ expect(sql).toContain(+ "properties['experiment'] as `profile.properties.experiment`",+ );+ expect(sql).not.toContain('properties as "profile.properties"');+ // Conditions (windowFunnel + pre-filter) and the breakdown expression are+ // rewritten to the scalar aliases.+ expect(sql).not.toContain("profile.properties['plan']");+ expect(sql).not.toContain("profile.properties['experiment']");+ });++ it('leaves funnels without profile-property refs untouched', async () => {+ const sql = await buildChartSql([]);+ expect(sql).not.toContain('`profile.properties.');+ });++ itCH('narrowed funnel SQL parses and resolves', async () => {+ const { query, breakdowns: resolved } = await funnelService.buildFunnelBase(+ {+ projectId: PROJECT_ID,+ startDate: START,+ endDate: END,+ series: seriesWithProfileFilter,+ breakdowns: [breakdown('profile.properties.experiment')],+ funnelWindow: 24,+ timezone: 'UTC',+ },+ );+ query.with('funnel', 'SELECT * FROM session_funnel WHERE level != 0');+ query+ .select([+ 'level',+ ...resolved.map((_, index) => `b_${index}`),+ 'count() as count',+ ])+ .from('funnel')+ .groupBy(['level', ...resolved.map((_, index) => `b_${index}`)]);+ await explain(query.toSQL());+ });+});diff --git a/packages/db/src/services/funnel.service.ts b/packages/db/src/services/funnel.service.tsindex ec3f43b6c..dbadc39ec 100644--- a/packages/db/src/services/funnel.service.ts+++ b/packages/db/src/services/funnel.service.ts@@ -13,12 +13,15 @@ import { createSqlBuilder } from '../sql-builder';import {buildInlineCohortJoin,collectBreakdownCohortIds,+ collectProfilePropertyKeys,extractCohortId,fetchCohortsMetadata,getEventFiltersWhereClause,getSelectPropertyKey,isAllCohortsBreakdown,isKnownEventField,+ profilePropertiesCteSelect,+ rewriteProfilePropertyRefs,} from './chart.service';import { mergeGlobalFilters, onlyReportEvents } from './reports.service';@@ -70,6 +73,7 @@ export class FunnelService {additionalSelects = [],additionalGroupBy = [],group = 'session_id',+ profilePropertyKeys = [],}: {projectId: string;startDate: string;@@ -80,8 +84,11 @@ export class FunnelService {additionalSelects?: string[];additionalGroupBy?: string[];group?: 'session_id' | 'profile_id';+ profilePropertyKeys?: string[];}) {- const funnels = this.getFunnelConditions(eventSeries, projectId);+ const funnels = this.getFunnelConditions(eventSeries, projectId).map((c) =>+ rewriteProfilePropertyRefs(c, profilePropertyKeys),+ );const primaryKey = group === 'profile_id' ? 'profile_id' : 'session_id';// windowFunnel's 'strict_increase' mode requires every step's timestamp@@ -321,18 +328,26 @@ export class FunnelService {// event out per group, and grouping by the group value is intentional —// a user in three groups should appear in all three funnels. Those keep// the per-row GROUP BY.- const firstStepCondition = this.getFunnelConditions(- eventSeries,- projectId,- )[0]!;+ // Join only the referenced profile-property keys as scalar columns+ // instead of every profile's whole properties Map — the join hash was+ // carrying ~1KB of Map per profile and OOMing at scale (same narrowing+ // as the chart profile CTE). Funnel conditions and breakdown expressions+ // are rewritten to the scalar aliases below.+ const profileProps = collectProfilePropertyKeys([+ ...eventSeries.flatMap((e) => e.filters ?? []),+ ...breakdowns,+ ]);++ const firstStepCondition = rewriteProfilePropertyRefs(+ this.getFunnelConditions(eventSeries, projectId)[0]!,+ profileProps.keys,+ );const breakdownSelects = breakdowns.map((b, index) => {const bId = extractCohortId(b.name);const bName = bId ? cohortMetadata.get(bId)?.name : undefined;- const expr = getSelectPropertyKey(- b.name,- projectId,- bId ?? undefined,- bName,+ const expr = rewriteProfilePropertyRefs(+ getSelectPropertyKey(b.name, projectId, bId ?? undefined, bName),+ profileProps.keys,);if (b.name.startsWith('group.')) {return `${expr} as b_${index}`;@@ -353,21 +368,25 @@ export class FunnelService {additionalSelects: breakdownSelects,additionalGroupBy: breakdownGroupBy,group,+ profilePropertyKeys: profileProps.keys,});// The profile join has to cover breakdowns as well as filters — a// `profile.*` breakdown renders `profile.properties[...]` into the select,// so the alias must exist in scope even when no filter touches profiles.if (anyFilterOnProfile || profileBreakdowns.length > 0) {+ // Scalar columns (email etc.) are selected as-is; the properties Map is+ // narrowed to the referenced keys via profilePropertiesCteSelect.const profileFields = new Set<string>(['id']);for (const f of profileFilters) {- profileFields.add(f.split('.')[0]!);+ const fieldName = f.split('.')[0]!;+ if (fieldName !== 'properties') {+ profileFields.add(fieldName);+ }}for (const b of profileBreakdowns) {const fieldName = b.name.replace('profile.', '').split('.')[0];- if (fieldName === 'properties') {- profileFields.add('properties');- } else if (+ if (['email','first_name',@@ -379,9 +398,22 @@ export class FunnelService {profileFields.add(fieldName!);}}- const profileSelectColumns = Array.from(profileFields).join(', ');+ const selectColumns = Array.from(profileFields);+ const referencesProperties =+ profileFilters.some((f) => f.startsWith('properties')) ||+ profileBreakdowns.some((b) =>+ b.name.startsWith('profile.properties'),+ );+ if (referencesProperties) {+ selectColumns.push(+ profilePropertiesCteSelect(+ profileProps.keys,+ profileProps.needsFullMap,+ ),+ );+ }funnelCte.leftJoin(- `(SELECT ${profileSelectColumns} FROM ${TABLE_NAMES.profiles} FINAL+ `(SELECT ${selectColumns.join(', ')} FROM ${TABLE_NAMES.profiles} FINALWHERE project_id = ${sqlstring.escape(projectId)}) as profile`,'profile.id = events.profile_id',);