Skip to content

OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.

perf(charts): single-pass total_count via uniqState + window merge (#448)

Ritik Jain1e14db8c0897
diff --git a/packages/db/src/services/chart-sql.test.ts b/packages/db/src/services/chart-sql.test.ts
index 64f286d13..f9c997a7e 100644
--- a/packages/db/src/services/chart-sql.test.ts
+++ b/packages/db/src/services/chart-sql.test.ts
@@ -649,3 +649,67 @@ describe('chart.service / profile-property narrowing', () => {
await explain(sql);
});
});
+
+describe('chart.service / single-pass total_count', () => {
+ const base = {
+ interval: 'day',
+ startDate: START,
+ endDate: END,
+ projectId: PROJECT_ID,
+ timezone: 'UTC',
+ };
+
+ it('scans events exactly once and merges uniq states globally (no breakdown)', async () => {
+ const sql = await getChartSql({ event: event(), breakdowns: [], ...base });
+ expect(sql).not.toContain('_uc AS (');
+ expect(sql).toContain('uniqState(profile_id) as _uc_state');
+ expect(sql).toContain('uniqMerge(_uc_state) OVER () as total_count');
+ expect(sql).toContain('* EXCEPT (_uc_state)');
+ // exactly one scan of the events table
+ expect(sql.match(/FROM events e/g)).toHaveLength(1);
+ });
+
+ it('partitions the merged uniq states by breakdown labels', async () => {
+ const sql = await getChartSql({
+ event: event(),
+ breakdowns: [breakdown('properties.experiment')],
+ ...base,
+ });
+ expect(sql).not.toContain('_uc AS (');
+ expect(sql).not.toContain('LEFT ANY JOIN _uc');
+ expect(sql).toContain(
+ 'uniqMerge(_uc_state) OVER (PARTITION BY label_1) as total_count',
+ );
+ expect(sql.match(/FROM events e/g)).toHaveLength(1);
+ });
+
+ it('keeps ORDER BY and WITH FILL outside the aggregation subquery', async () => {
+ const sql = await getChartSql({ event: event(), breakdowns: [], ...base });
+ // GROUP BY belongs to the inner scan; ORDER BY/FILL follow its closing paren
+ expect(sql).toMatch(/GROUP BY[^)]*\)\s*ORDER BY date ASC/);
+ expect(sql).toContain('WITH FILL');
+ });
+
+ itCH('single-pass chart SQL parses and resolves (no breakdown)', async () => {
+ const sql = await getChartSql({ event: event(), breakdowns: [], ...base });
+ await explain(sql);
+ });
+
+ itCH('single-pass chart SQL parses and resolves (breakdown + cohort + profile)', async () => {
+ const sql = await getChartSql({
+ event: event({
+ filters: [
+ {
+ id: 'f1',
+ name: 'profile.properties.plan',
+ operator: 'is' as const,
+ value: ['pro'],
+ },
+ ],
+ }),
+ breakdowns: [breakdown('properties.experiment')],
+ ...base,
+ });
+ await explain(sql);
+ });
+});
diff --git a/packages/db/src/services/chart.service.ts b/packages/db/src/services/chart.service.ts
index 59dff37cd..68ab1c217 100644
--- a/packages/db/src/services/chart.service.ts
+++ b/packages/db/src/services/chart.service.ts
@@ -565,16 +565,6 @@ export async function getChartSql({
sb.joins.groups_table = 'LEFT ANY JOIN _g ON _g.id = _group_id';
}
- // Build WHERE clause without the bar filter (for use in subqueries and CTEs)
- // Define this early so we can use it in CTE definitions
- const getWhereWithoutBar = () => {
- const whereWithoutBar = { ...sb.where };
- delete whereWithoutBar.bar;
- return Object.keys(whereWithoutBar).length
- ? `WHERE ${join(whereWithoutBar, ' AND ')}`
- : '';
- };
-
// Collect all profile fields used in filters and breakdowns
// Extract top-level field names (e.g., 'properties' from 'profile.properties.os')
const getProfileFields = () => {
@@ -833,89 +823,26 @@ export async function getChartSql({
return sql;
}
- // Note: The profile CTE (if it exists) is available in subqueries, so we can reference it directly.
- // Cohort CTEs cannot be referenced from nested CTEs in ClickHouse, so we inline them.
- const subqueryGroupJoins = needsGroupArrayJoin
- ? 'ARRAY JOIN groups AS _group_id LEFT ANY JOIN _g ON _g.id = _group_id '
- : '';
- const inlineCohortJoinsSql = cohortIds
- .map((id) => buildInlineCohortJoin(id, projectId, 'e'))
- .join(' ');
- // Inline all-cohorts join for use in _uc CTE (can't reference CTEs from nested CTEs)
- const inlineAllCohortsJoin = hasAllCohortsBreakdown
- ? `INNER JOIN (${buildAllCohortsMembershipQuery(projectId)}) AS _all_cohorts ON _all_cohorts.profile_id = e.profile_id `
- : '';
-
- if (breakdowns.length > 0) {
- // Pre-compute unique counts per breakdown group in a CTE, then JOIN it.
- // We can't use a correlated subquery because:
- // 1. ClickHouse expands label_X aliases to their underlying expressions,
- // which resolve in the subquery's scope, making the condition a tautology.
- // 2. Correlated subqueries aren't supported on distributed/remote tables.
- const ucSelectParts: string[] = breakdowns.map((breakdown, index) => {
- if (isAllCohortsBreakdown(breakdown.name)) {
- return `${buildAllCohortsLabelExpr(allCohorts)} as _uc_label_${index + 1}`;
- }
- const bId = extractCohortId(breakdown.name);
- const bName = bId ? cohortMetadata.get(bId)?.name : undefined;
- const propertyKey = getSelectPropertyKey(
- breakdown.name,
- projectId,
- bId ?? undefined,
- bName,
- 'e',
- );
- return `${propertyKey} as _uc_label_${index + 1}`;
- });
- ucSelectParts.push('uniq(profile_id) as total_count');
-
- const ucGroupByParts = breakdowns.map(
- (_, index) => `_uc_label_${index + 1}`
- );
-
- const ucWhere = getWhereWithoutBar();
-
- addCte(
- '_uc',
- `SELECT ${ucSelectParts.join(', ')} FROM ${TABLE_NAMES.events} e ${subqueryGroupJoins}${profilesJoinRef ? `${profilesJoinRef} ` : ''}${inlineCohortJoinsSql ? `${inlineCohortJoinsSql} ` : ''}${inlineAllCohortsJoin}${ucWhere} GROUP BY ${ucGroupByParts.join(', ')}`
- );
-
- const ucJoinConditions = breakdowns
- .map((b, index) => {
- if (isAllCohortsBreakdown(b.name)) {
- return `_uc._uc_label_${index + 1} = ${buildAllCohortsLabelExpr(allCohorts)}`;
- }
- const bId = extractCohortId(b.name);
- const bName = bId ? cohortMetadata.get(bId)?.name : undefined;
- const propertyKey = getSelectPropertyKey(
- b.name,
- projectId,
- bId ?? undefined,
- bName,
- 'e',
- );
- return `_uc._uc_label_${index + 1} = ${propertyKey}`;
- })
- .join(' AND ');
-
- sb.joins.unique_counts = `LEFT ANY JOIN _uc ON ${ucJoinConditions}`;
- sb.select.total_unique_count = 'any(_uc.total_count) as total_count';
- } else {
- const ucWhere = getWhereWithoutBar();
-
- addCte(
- '_uc',
- `SELECT uniq(profile_id) as total_count FROM ${TABLE_NAMES.events} e ${subqueryGroupJoins}${profilesJoinRef ? `${profilesJoinRef} ` : ''}${inlineCohortJoinsSql ? `${inlineCohortJoinsSql} ` : ''}${ucWhere}`
- );
-
- sb.select.total_unique_count =
- '(SELECT total_count FROM _uc) as total_count';
- }
+ // Single-pass total_count: aggregate uniqState(profile_id) alongside the
+ // series in the same scan, then merge the per-group states with a window
+ // aggregate in an outer select — PARTITION BY the breakdown labels, or an
+ // empty partition for the global total. The previous shape re-ran the
+ // chart's entire WHERE in a second full scan (a `_uc` CTE joined back per
+ // label / injected as a scalar subquery), doubling every chart's read
+ // cost. uniq states merge losslessly, so the result matches the two-scan
+ // form exactly. (The old _uc scan nominally excluded a `bar` filter, but
+ // nothing sets sb.where.bar anymore — the exclusion was dead code.)
+ sb.select.uc_state = 'uniqState(profile_id) as _uc_state';
+
+ const totalCountPartition = breakdowns
+ .map((_, index) => `label_${index + 1}`)
+ .join(', ');
+ const totalCountSelect = `uniqMerge(_uc_state) OVER (${totalCountPartition ? `PARTITION BY ${totalCountPartition}` : ''}) as total_count`;
const sql = rewriteProfilePropertyRefs(
- `${getWith()}${getSelect()} ${getFrom()} ${getJoins()} ${getWhere()} ${getGroupBy()} ${getOrderBy()} ${getFill()}`,
- profileProps.keys,
- );
+ `${getWith()}SELECT * EXCEPT (_uc_state), ${totalCountSelect} FROM (${getSelect()} ${getFrom()} ${getJoins()} ${getWhere()} ${getGroupBy()}) ${getOrderBy()} ${getFill()}`,
+ profileProps.keys,
+ );
console.log('-- Report --');
console.log(sql.replaceAll(/[\n\r]/g, ' '));
console.log('-- End --');