OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.
perf(api): charts only load profiles active in the chart window
Carl-Gerhard Lindesvärd9f3b78a87caa
diff --git a/apps/start/e2e/verify-b-profile-breakdown.spec.ts b/apps/start/e2e/verify-b-profile-breakdown.spec.tsnew file mode 100644index 000000000..10537800c--- /dev/null+++ b/apps/start/e2e/verify-b-profile-breakdown.spec.ts@@ -0,0 +1,125 @@+import type { Page } from '@playwright/test';+import { expect, test } from './fixtures';+import {+ eventSeries,+ hideFeedbackPrompt,+ reportInput,+ trpcMutation,+ trpcQuery,+ waitForChart,+} from './reports-helpers';+import { shot } from './verify-b-helpers';++const SAAS = 'acme-saas';+const CHART_ERROR = 'There was an error loading this chart.';+const PLAN_BREAKDOWN = [{ id: 'plan', name: 'profile.properties.plan' }];++interface ChartSeries {+ names: string[];+ metrics: { sum: number };+}++interface FunnelSeries {+ breakdowns: string[];+ steps: { count: number }[];+}++/** Saves a report, opens it, and returns the rendered editor text. */+async function openReport(+ page: Page,+ name: string,+ input: Record<string, unknown>+) {+ const dashboard = await trpcMutation<{ id: string }>(+ page,+ 'dashboard.create',+ { name: `E2E profile breakdown ${Date.now()}`, projectId: SAAS }+ );+ const { projectId: _projectId, ...report } = input;+ const created = await trpcMutation<{ id: string }>(page, 'report.create', {+ dashboardId: dashboard.id,+ report: { ...report, name },+ });+ await page.goto(`/acme/${SAAS}/reports/${created.id}`);+ await page.waitForLoadState('networkidle');+ await waitForChart(page);+ const text = (await page.locator('#report-editor').innerText()).replace(+ /\s+/g,+ ' '+ );+ return { dashboardId: dashboard.id, text };+}++function deleteDashboard(page: Page, id: string) {+ return trpcMutation(page, 'dashboard.delete', { id, forceDelete: true });+}++test.use({ actionTimeout: 15_000 });++test.beforeEach(async ({ page }) => {+ test.setTimeout(180_000);+ await hideFeedbackPrompt(page);+ await page.setViewportSize({ width: 1440, height: 1200 });+ await page.goto('/login');+});++test('bar chart breaks unique users down by a profile property', async ({+ page,+}) => {+ const input = reportInput(SAAS, {+ chartType: 'bar',+ series: eventSeries(['screen_view'], { segment: 'user' }),+ breakdowns: PLAN_BREAKDOWN,+ });+ const api = await trpcQuery<{ series: ChartSeries[] }>(+ page,+ 'chart.aggregate',+ input+ );+ const plans = api.series+ .map((serie) => serie.names.at(-1))+ .filter((plan): plan is string => !!plan && plan !== 'screen_view');+ expect(plans).toContain('free');++ const opened = await openReport(page, 'E2E users by plan', input);+ try {+ await shot(page, 'profile-breakdown-bar');+ expect(opened.text).not.toContain(CHART_ERROR);+ for (const plan of plans) {+ expect(opened.text).toContain(plan);+ }+ } finally {+ await deleteDashboard(page, opened.dashboardId);+ }+});++// The breakdown is read from step 1, so both steps happen after identification:+// signup_started still belongs to the anonymous device and has no plan.+test('funnel breaks down by a profile property', async ({ page }) => {+ const input = reportInput(SAAS, {+ chartType: 'funnel',+ series: eventSeries(['onboarding_step_completed', 'project_created']),+ breakdowns: PLAN_BREAKDOWN,+ options: { type: 'funnel', funnelGroup: 'profile_id' },+ });+ const api = await trpcQuery<{ current: FunnelSeries[] }>(+ page,+ 'chart.funnel',+ input+ );+ const completedPlans = api.current+ .filter((serie) => (serie.steps.at(-1)?.count ?? 0) > 0)+ .map((serie) => serie.breakdowns[0]);+ expect(completedPlans).toContain('free');++ const opened = await openReport(page, 'E2E onboarding funnel by plan', input);+ try {+ await shot(page, 'profile-breakdown-funnel');+ expect(opened.text).not.toContain(CHART_ERROR);+ for (const plan of completedPlans) {+ expect(opened.text).toContain(plan);+ }+ } finally {+ await deleteDashboard(page, opened.dashboardId);+ }+});diff --git a/packages/core/src/modules/chart/chart.service.test.ts b/packages/core/src/modules/chart/chart.service.test.tsindex cf097f904..464136114 100644--- a/packages/core/src/modules/chart/chart.service.test.ts+++ b/packages/core/src/modules/chart/chart.service.test.ts@@ -233,6 +233,18 @@ describe('executeChart', () => {expect(chart.metrics.sum).toBe(1);expect(seriesNamed(chart, 'US')?.metrics.sum).toBe(1);});++ it('joins profiles created before the window that have events inside it', async () => {+ const chart = await service.execute(+ report({+ range: '7d',+ breakdowns: [{ id: 'b', name: 'profile.properties.browser' }],+ })+ );++ expect(seriesNamed(chart, 'Chrome')?.metrics.sum).toBe(1);+ expect(seriesNamed(chart, 'Firefox')?.metrics.sum).toBe(1);+ });});describe('executeAggregateChart', () => {diff --git a/packages/core/src/modules/chart/src/conversion.sql.ts b/packages/core/src/modules/chart/src/conversion.sql.tsindex 01e14fba8..efa959d3a 100644--- a/packages/core/src/modules/chart/src/conversion.sql.ts+++ b/packages/core/src/modules/chart/src/conversion.sql.ts@@ -2,12 +2,16 @@//// Cluster note: `events`, `profiles` and `groups` are Distributed on Cloud. The// profile / group / cohort joins are LEFT ANY JOIN and run under the client's-// `distributed_product_mode: 'allow'`; no `IN (subquery)` is involved.+// `distributed_product_mode: 'allow'`; the profile join's id set is `GLOBAL IN`.import { type SqlFragment, sql } from '@openpanel/db/src/clickhouse/sql';import type { IInterval } from '../../report/report.constants';import { compiledText } from './compiled';-import { buildGroupsQuery, CHART_TABLE } from './field-resolution';+import {+ buildGroupsQuery,+ CHART_TABLE,+ profileIdsInWindow,+} from './field-resolution';/** windowFunnel counts a conversion when both steps fired. */const CONVERTED_STEP_COUNT = 2;@@ -62,7 +66,11 @@ function stepCondition(name: string, where: SqlFragment | null): SqlFragment {: sql`events.name = ${sql.string(name)}`;}-function profileJoin(fields: string[], projectId: string): SqlFragment {+function profileJoin(+ fields: string[],+ projectId: string,+ activeProfiles: SqlFragment+): SqlFragment {if (fields.length === 0) {return sql.empty;}@@ -70,7 +78,7 @@ function profileJoin(fields: string[], projectId: string): SqlFragment {return sql`LEFT ANY JOIN (SELECT ${sql.join(fields.map((field) => sql.id(field)))}FROM ${sql.id(CHART_TABLE.profiles)} FINAL- WHERE project_id = ${sql.string(projectId)}+ WHERE project_id = ${sql.string(projectId)} AND ${activeProfiles}) as profile ON profile.id = profile_id`;}@@ -106,6 +114,13 @@ export function conversionQuery(input: ConversionQueryInput): SqlFragment {);const outerKeys = [sql`event_day`, ...breakdownAliases];const converted = sql`countIf(steps >= ${sql.uint64(CONVERTED_STEP_COUNT)})`;+ const eventNames = [firstEventName, secondEventName];+ const dateRange = sql`created_at BETWEEN toDateTime(${sql.string(startDate)}) AND toDateTime(${sql.string(endDate)})`;+ const activeProfiles = profileIdsInWindow({+ projectId,+ eventNames,+ window: [dateRange],+ });const outerSelect = sql.join([sql`event_day`,@@ -135,12 +150,12 @@ export function conversionQuery(input: ConversionQueryInput): SqlFragment {${stepCondition(secondEventName, secondEventWhere)}) as stepsFROM ${sql.id(CHART_TABLE.events)}- ${profileJoin(profileJoinFields, projectId)}+ ${profileJoin(profileJoinFields, projectId, activeProfiles)}${groupJoin(needsGroupArrayJoin, projectId)}${sql.join(cohortJoins, ' ')}WHERE project_id = ${sql.string(projectId)}- AND events.name IN ${sql.array('String', [firstEventName, secondEventName])}- AND created_at BETWEEN toDateTime(${sql.string(startDate)}) AND toDateTime(${sql.string(endDate)})+ AND events.name IN ${sql.array('String', eventNames)}+ AND ${dateRange}GROUP BY ${groupKey}${innerBreakdownGroupBy})) WHERE steps > 0 GROUP BY ${sql.join(outerKeys)} ORDER BY ${sql.join(outerKeys.map((key) => sql`${key} ASC`))}`;}diff --git a/packages/core/src/modules/chart/src/field-resolution.ts b/packages/core/src/modules/chart/src/field-resolution.tsindex ccc2322fa..d8b5f74bb 100644--- a/packages/core/src/modules/chart/src/field-resolution.ts+++ b/packages/core/src/modules/chart/src/field-resolution.ts@@ -319,6 +319,33 @@ export function buildGroupsQuery(projectId: string): SqlFragment {return sql`SELECT id, name, type, properties FROM ${sql.id(CHART_TABLE.groups)} FINAL WHERE project_id = ${sql.string(projectId)}`;}+/**+ * Restricts a `profiles FINAL` join to profiles with events in the statement's+ * window, so its memory scales with active profiles, not the project's.+ * `window` must be the outer query's date predicates and no event filters: the+ * set has to be a superset of every `profile_id` the query joins.+ * Hashes, because a String `IN` set costs ~300 bytes per id; a collision only+ * adds a profile the join then ignores. `GLOBAL IN` so the set is built once,+ * not once per shard from the Distributed `events`.+ */+export function profileIdsInWindow({+ projectId,+ eventNames,+ window,+}: {+ projectId: string;+ /** Empty, or containing `'*'`, means every event name. */+ eventNames: string[];+ window: SqlFragment[];+}): SqlFragment {+ const conditions = [sql`project_id = ${sql.string(projectId)}`, ...window];+ const isEveryEvent = eventNames.length === 0 || eventNames.includes('*');+ if (!isEveryEvent) {+ conditions.push(sql`name IN ${sql.array('String', eventNames)}`);+ }+ return sql`cityHash64(id) GLOBAL IN (SELECT cityHash64(profile_id) FROM ${sql.id(CHART_TABLE.events)} WHERE ${sql.join(conditions, ' AND ')})`;+}+// Returns a SQL expression for a group property via the _g JOIN alias// property format: "group.name", "group.type", "group.properties.plan"export function getGroupPropertySql(property: string): SqlFragment {diff --git a/packages/core/src/modules/chart/src/funnel.sql.test.ts b/packages/core/src/modules/chart/src/funnel.sql.test.tsindex 9d7d8060d..431b74047 100644--- a/packages/core/src/modules/chart/src/funnel.sql.test.ts+++ b/packages/core/src/modules/chart/src/funnel.sql.test.ts@@ -191,6 +191,16 @@ describe('funnel.sql / buildFunnelBase — profile breakdowns', () => {await explain(statement);});+ it('loads only profiles with step events in the funnel range', async () => {+ const statement = await chartStatement({+ breakdowns: [breakdown('profile.properties.plan')],+ });+ expect(render(statement).text).toContain(+ `cityHash64(id) GLOBAL IN (SELECT cityHash64(profile_id) FROM events WHERE project_id = '${PROJECT_ID}' AND created_at BETWEEN toDateTime('${START}') AND toDateTime('${END}') AND name IN ('screen_view', 'sign_up'))`+ );+ await explain(statement);+ });+it('adds the profiles join for a top-level profile breakdown', async () => {const statement = await profilesStatement({breakdowns: [breakdown('profile.email')],diff --git a/packages/core/src/modules/chart/src/funnel.sql.ts b/packages/core/src/modules/chart/src/funnel.sql.tsindex dc22460ea..8ded7c2ef 100644--- a/packages/core/src/modules/chart/src/funnel.sql.ts+++ b/packages/core/src/modules/chart/src/funnel.sql.ts@@ -7,7 +7,7 @@//// Cluster note: `events`, `profiles`, `groups` and `cohort_members` are// Distributed on Cloud. Every join runs under the client's-// `distributed_product_mode: 'allow'`; no `IN (subquery)` is involved.+// `distributed_product_mode: 'allow'`; the profile join's id set is `GLOBAL IN`.import { type SqlFragment, sql } from '@openpanel/db/src/clickhouse/sql';import type {@@ -27,6 +27,7 @@ import {getSelectPropertyKey,isAllCohortsBreakdown,isKnownEventField,+ profileIdsInWindow,profilePropertiesCteSelect,} from './field-resolution';import { getEventFiltersWhereClause } from './filter-where';@@ -292,11 +293,18 @@ export function funnelBase(input: FunnelBaseInput): FunnelBase {}});+ const dateRange = sql`created_at BETWEEN toDateTime(${sql.string(input.startDate)}) AND toDateTime(${sql.string(input.endDate)})`;+ const eventNames = eventSeries.map((event) => event.name);const columns = profileCteColumns(input, profileFilters, profileKeys);+ const activeProfiles = profileIdsInWindow({+ projectId,+ eventNames,+ window: [dateRange],+ });const profileJoin =columns.length > 0? sql` LEFT JOIN (SELECT ${sql.join(columns, ', ')} FROM ${sql.id(CHART_TABLE.profiles)} FINAL- WHERE project_id = ${sql.string(projectId)}) as profile ON profile.id = events.profile_id`+ WHERE project_id = ${sql.string(projectId)} AND ${activeProfiles}) as profile ON profile.id = events.profile_id`: sql.empty;const groupJoin = needsGroupArrayJoin? sql` ARRAY JOIN groups AS _group_id LEFT ANY JOIN _g ON _g.id = _group_id`@@ -325,10 +333,7 @@ export function funnelBase(input: FunnelBaseInput): FunnelBase {' OR ')})`;- const funnelCte = sql`SELECT ${sql.join(cteSelects)} FROM ${sql.id(CHART_TABLE.events)}${profileJoin}${groupJoin}${cohortJoins} WHERE project_id = ${sql.string(projectId)} AND created_at BETWEEN toDateTime(${sql.string(input.startDate)}) AND toDateTime(${sql.string(input.endDate)}) AND events.name IN ${sql.array(- 'String',- eventSeries.map((event) => event.name)- )} AND ${stepPreFilter} GROUP BY ${sql.join(groupByKeys)}`;+ const funnelCte = sql`SELECT ${sql.join(cteSelects)} FROM ${sql.id(CHART_TABLE.events)}${profileJoin}${groupJoin}${cohortJoins} WHERE project_id = ${sql.string(projectId)} AND ${dateRange} AND events.name IN ${sql.array('String', eventNames)} AND ${stepPreFilter} GROUP BY ${sql.join(groupByKeys)}`;const groupsCte = needsGroupArrayJoin? sql`_g AS (${buildGroupsQuery(projectId)}), `diff --git a/packages/core/src/modules/chart/src/sql.test.ts b/packages/core/src/modules/chart/src/sql.test.tsindex ec1255599..6a2ce6f7d 100644--- a/packages/core/src/modules/chart/src/sql.test.ts+++ b/packages/core/src/modules/chart/src/sql.test.ts@@ -14,7 +14,7 @@*/import { afterAll, beforeAll, describe, expect, it, mock } from 'bun:test';-import type { SqlFragment } from '@openpanel/db/src/clickhouse/sql';+import { type SqlFragment, sql } from '@openpanel/db/src/clickhouse/sql';import type {IChartBreakdown,IChartEvent,@@ -592,6 +592,86 @@ describe('profile-property narrowing', () => {});});+describe('profile join window', () => {+ const window =+ "created_at >= toDateTime('2026-04-14 00:00:00') AND created_at <= toDateTime('2026-05-15 00:00:00')";+ const profileBreakdown = [breakdown('profile.properties.plan')];++ it('loads only profiles with events of the series in the chart window', async () => {+ const rendered = await getChartSql({+ event: event(),+ breakdowns: profileBreakdown,+ ...base,+ });+ expect(rendered.text).toContain(+ `cityHash64(id) GLOBAL IN (SELECT cityHash64(profile_id) FROM events WHERE project_id = '${PROJECT_ID}' AND ${window} AND name IN ('screen_view'))`+ );+ await explain(rendered);+ });++ it('bounds a `*` series by the window alone', async () => {+ const rendered = await getAggregateChartSql({+ event: event({ name: '*' }),+ breakdowns: profileBreakdown,+ ...base,+ });+ expect(rendered.text).toContain(+ `cityHash64(id) GLOBAL IN (SELECT cityHash64(profile_id) FROM events WHERE project_id = '${PROJECT_ID}' AND ${window})`+ );+ await explain(rendered);+ });++ it('bounds the drill-down by its bucket', async () => {+ const { chartBucketProfilesQuery } = await import('./sql');+ const rendered = render(+ chartBucketProfilesQuery({+ projectId: PROJECT_ID,+ bucketDate: START,+ interval: 'day',+ event: { name: 'screen_view', filters: [] },+ breakdowns: { 'profile.properties.plan': 'pro' },+ limit: 10,+ })+ );+ expect(rendered.text).toContain(+ `cityHash64(id) GLOBAL IN (SELECT cityHash64(profile_id) FROM events WHERE project_id = '${PROJECT_ID}' AND toStartOfDay(created_at) = toDate('${START}') AND name IN ('screen_view'))`+ );+ await explain(rendered);+ });++ it('bounds the conversion join by both steps and the range', async () => {+ const { conversionQuery } = await import('./conversion.sql');+ const { getSelectPropertyKey } = await import('./field-resolution');+ const expression = getSelectPropertyKey(+ 'profile.properties.plan',+ PROJECT_ID+ );+ const rendered = render(+ conversionQuery({+ projectId: PROJECT_ID,+ startDate: START,+ endDate: END,+ interval: 'day',+ group: 'profile_id',+ funnelWindowSeconds: 3600,+ firstEventName: 'screen_view',+ secondEventName: 'sign_up',+ firstEventWhere: null,+ secondEventWhere: null,+ breakdownSelects: [sql`${expression} as b_0`],+ breakdownExpressions: [expression],+ profileJoinFields: ['id', 'properties'],+ needsGroupArrayJoin: false,+ cohortJoins: [],+ })+ );+ expect(rendered.text).toContain(+ `cityHash64(id) GLOBAL IN (SELECT cityHash64(profile_id) FROM events WHERE project_id = '${PROJECT_ID}' AND created_at BETWEEN toDateTime('${START}') AND toDateTime('${END}') AND name IN ('screen_view', 'sign_up'))`+ );+ await explain(rendered);+ });+});+describe('single-pass total_count', () => {it('scans events exactly once and merges uniq states globally (no breakdown)', async () => {const { sql } = await getChartSql({diff --git a/packages/core/src/modules/chart/src/sql.ts b/packages/core/src/modules/chart/src/sql.tsindex b9f872266..604a97ef4 100644--- a/packages/core/src/modules/chart/src/sql.ts+++ b/packages/core/src/modules/chart/src/sql.ts@@ -38,6 +38,7 @@ import {isKnownEventField,isNumericColumn,PROFILE_CTE_FIELDS,+ profileIdsInWindow,profilePropertiesCteSelect,} from './field-resolution';import { getEventFiltersWhereClause } from './filter-where';@@ -213,11 +214,12 @@ function profileCteSelectField(function profileCte(selectFields: SqlFragment[],- projectId: string+ projectId: string,+ activeProfiles: SqlFragment): SqlFragment {return sql`SELECT ${sql.join(selectFields, ', ')}FROM ${sql.id(CHART_TABLE.profiles)} FINAL- WHERE project_id = ${sql.string(projectId)}`;+ WHERE project_id = ${sql.string(projectId)} AND ${activeProfiles}`;}function anyRefOn(prefix: string, refs: { name: string }[]): boolean {@@ -234,19 +236,23 @@ interface ChartBody {/*** The CTEs, JOINs, WHERE and `label_0` both chart shapes share, in a fixed- * clause order: cohort CTEs/joins, filters + project + event name, group- * ARRAY JOIN, profile CTE.+ * clause order: cohort CTEs/joins, filters + project + event name + date+ * range, group ARRAY JOIN, profile CTE.*/function chartBody({event,breakdowns,allCohorts,projectId,+ startDate,+ endDate,}: {event: ChartQueryEvent;breakdowns: IChartBreakdown[];allCohorts: CohortMetadata[];projectId: string;+ startDate: string;+ endDate: string;}): ChartBody {const hasAllCohortsBreakdown =requestsAllCohortsBreakdown(breakdowns) && allCohorts.length > 0;@@ -295,6 +301,8 @@ function chartBody({} else {select.label_0 = sql`'*' as label_0`;}+ const dateRange = dateRangeWhere(startDate, endDate);+ Object.assign(where, dateRange);const profileRefs = [...event.filters, ...breakdowns, ...metricRef];const needsProfileJoin = anyRefOn('profile.', profileRefs);@@ -311,7 +319,15 @@ function chartBody({const selectFields = collectProfileCteFields(profileRefs).map((field) =>profileCteSelectField(field, profileProps.keys, profileProps.needsFullMap));- ctes.profile = profileCte(selectFields, projectId);+ ctes.profile = profileCte(+ selectFields,+ projectId,+ profileIdsInWindow({+ projectId,+ eventNames: [event.name],+ window: Object.values(dateRange),+ })+ );joins.profiles = sql`LEFT ANY JOIN profile ON profile.id = profile_id`;}@@ -427,17 +443,15 @@ function oneEventPerUserFrom(body: ChartBody): SqlFragment {) as e`;}-function dateRangeWhere(- where: Parts,- startDate: string,- endDate: string-): void {+function dateRangeWhere(startDate: string, endDate: string): Parts {+ const where: Parts = {};if (startDate) {where.startDate = sql`created_at >= toDateTime(${sql.string(formatClickhouseDate(startDate))})`;}if (endDate) {where.endDate = sql`created_at <= toDateTime(${sql.string(formatClickhouseDate(endDate))})`;}+ return where;}function intervalBucket(@@ -501,7 +515,6 @@ export function chartSeriesQuery(input: ChartSeriesQueryInput): SqlFragment {groupBy.date = sql`date`;orderBy.date = sql`date ASC`;- dateRangeWhere(body.where, startDate, endDate);addBreakdownLabels(input, body, groupBy);body.select.count = countExpression(event, body, undefined);@@ -532,12 +545,11 @@ export function chartSeriesQuery(input: ChartSeriesQueryInput): SqlFragment {export function aggregateChartQuery(input: AggregateChartQueryInput): SqlFragment {- const { event, startDate, endDate, projectId, limit } = input;+ const { event, startDate, projectId, limit } = input;const body = chartBody(input);const groupBy: Parts = {};const orderBy: Parts = {};- dateRangeWhere(body.where, startDate, endDate);// A constant date field: groupByLabels expects one, and the whole range// aggregates into it.body.select.date = sql`${sql.string(startDate)} as date`;@@ -688,7 +700,8 @@ export function chartBucketProfilesQuery(where[key] = clause;}where.projectId = sql`project_id = ${sql.string(projectId)}`;- where.dateRange = bucketEquals(interval, bucketDate);+ const dateRange = bucketEquals(interval, bucketDate);+ where.dateRange = dateRange;if (event.name !== '*') {where.eventName = sql`name = ${sql.string(event.name)}`;}@@ -700,7 +713,12 @@ export function chartBucketProfilesQuery(].filter((name) => name.startsWith('profile.'));if (profileRefs.length > 0) {const fields = sql.join(profileFieldsToSelect(profileRefs), ', ');- joins.profiles = sql`LEFT ANY JOIN (SELECT id, ${fields} FROM ${sql.id(CHART_TABLE.profiles)} FINAL WHERE project_id = ${sql.string(projectId)}) as profile on profile.id = profile_id`;+ const activeProfiles = profileIdsInWindow({+ projectId,+ eventNames: [event.name],+ window: [dateRange],+ });+ joins.profiles = sql`LEFT ANY JOIN (SELECT id, ${fields} FROM ${sql.id(CHART_TABLE.profiles)} FINAL WHERE project_id = ${sql.string(projectId)} AND ${activeProfiles}) as profile on profile.id = profile_id`;}const needsGroupJoin = [