OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.
fix: view profiles, improve chart service
Carl-Gerhard Lindesvärd620904b4d458
diff --git a/packages/db/src/clickhouse/client.ts b/packages/db/src/clickhouse/client.tsindex 2a792d63f..a4d048c3f 100644--- a/packages/db/src/clickhouse/client.ts+++ b/packages/db/src/clickhouse/client.ts@@ -95,6 +95,7 @@ export const CLICKHOUSE_OPTIONS: NodeClickHouseClientConfigOptions = {},clickhouse_settings: {date_time_input_format: 'best_effort',+ query_plan_convert_any_join_to_semi_or_anti_join: 0,},log: {LoggerClass: CustomLogger,diff --git a/packages/db/src/services/chart.service.ts b/packages/db/src/services/chart.service.tsindex 989c3730d..a89d0470d 100644--- a/packages/db/src/services/chart.service.ts+++ b/packages/db/src/services/chart.service.ts@@ -1,3 +1,4 @@+import { uniq } from 'ramda';import sqlstring from 'sqlstring';import { DateTime, stripLeadingAndTrailingSlashes } from '@openpanel/common';@@ -74,6 +75,8 @@ export function getChartSql({getOrderBy,getGroupBy,getFill,+ getWith,+ with: addCte,} = createSqlBuilder();sb.where = getEventFiltersWhereClause(event.filters);@@ -93,14 +96,95 @@ export function getChartSql({breakdown.name.startsWith('profile.'),);+ // Build WHERE clause without the bar filter (for use in subqueries and CTEs)+ // Define this early so we can use it in CTE definitions+ const getWhereWithoutBar = () => {+ const whereWithoutBar = { ...sb.where };+ delete whereWithoutBar.bar;+ return Object.keys(whereWithoutBar).length+ ? `WHERE ${join(whereWithoutBar, ' AND ')}`+ : '';+ };++ // Collect all profile fields used in filters and breakdowns+ // Extract top-level field names (e.g., 'properties' from 'profile.properties.os')+ const getProfileFields = () => {+ const fields = new Set<string>();++ // Always need id for the join+ fields.add('id');++ // Collect from filters+ event.filters+ .filter((f) => f.name.startsWith('profile.'))+ .forEach((f) => {+ const fieldName = f.name.replace('profile.', '').split('.')[0];+ if (fieldName && fieldName === 'properties') {+ fields.add('properties');+ } else if (+ fieldName &&+ ['email', 'first_name', 'last_name'].includes(fieldName)+ ) {+ fields.add(fieldName);+ }+ });++ // Collect from breakdowns+ breakdowns+ .filter((b) => b.name.startsWith('profile.'))+ .forEach((b) => {+ const fieldName = b.name.replace('profile.', '').split('.')[0];+ if (fieldName && fieldName === 'properties') {+ fields.add('properties');+ } else if (+ fieldName &&+ ['email', 'first_name', 'last_name'].includes(fieldName)+ ) {+ fields.add(fieldName);+ }+ });++ return Array.from(fields);+ };++ // Create profiles CTE if profiles are needed (to avoid duplicating the heavy profile join)+ // Only select the fields that are actually used+ const profilesJoinRef =+ anyFilterOnProfile || anyBreakdownOnProfile+ ? 'LEFT ANY JOIN profile ON profile.id = profile_id'+ : '';+if (anyFilterOnProfile || anyBreakdownOnProfile) {- sb.joins.profiles = `LEFT ANY JOIN (SELECT- id as "profile.id",- email as "profile.email",- first_name as "profile.first_name",- last_name as "profile.last_name",- properties as "profile.properties"- FROM ${TABLE_NAMES.profiles} FINAL WHERE project_id = ${sqlstring.escape(projectId)}) as profile on profile.id = profile_id`;+ const profileFields = getProfileFields();+ const selectFields = profileFields.map((field) => {+ if (field === 'id') {+ return 'id as "profile.id"';+ }+ if (field === 'properties') {+ return 'properties as "profile.properties"';+ }+ if (field === 'email') {+ return 'email as "profile.email"';+ }+ if (field === 'first_name') {+ return 'first_name as "profile.first_name"';+ }+ if (field === 'last_name') {+ return 'last_name as "profile.last_name"';+ }+ return field;+ });++ // Add profiles CTE using the builder+ addCte(+ 'profile',+ `SELECT ${selectFields.join(', ')}+ FROM ${TABLE_NAMES.profiles} FINAL+ WHERE project_id = ${sqlstring.escape(projectId)}`,+ );++ // Use the CTE reference in the main query+ sb.joins.profiles = profilesJoinRef;}sb.select.count = 'count(*) as count';@@ -142,16 +226,25 @@ export function getChartSql({sb.where.endDate = `created_at <= toDateTime('${formatClickhouseDate(endDate)}')`;}+ // Use CTE to define top breakdown values once, then reference in WHERE clauseif (breakdowns.length > 0 && limit) {- sb.where.bar = `(${breakdowns.map((b) => getSelectPropertyKey(b.name)).join(',')}) IN (- SELECT ${breakdowns.map((b) => getSelectPropertyKey(b.name)).join(',')}- FROM ${TABLE_NAMES.events}- ${getJoins()}- ${getWhere()}- GROUP BY ${breakdowns.map((b) => getSelectPropertyKey(b.name)).join(',')}+ const breakdownSelects = breakdowns+ .map((b) => getSelectPropertyKey(b.name))+ .join(', ');++ // Add top_breakdowns CTE using the builder+ addCte(+ 'top_breakdowns',+ `SELECT ${breakdownSelects}+ FROM ${TABLE_NAMES.events} e+ ${profilesJoinRef ? `${profilesJoinRef} ` : ''}${getWhereWithoutBar()}+ GROUP BY ${breakdownSelects}ORDER BY count(*) DESC- LIMIT ${limit}- )`;+ LIMIT ${limit}`,+ );++ // Filter main query to only include top breakdown values+ sb.where.bar = `(${breakdowns.map((b) => getSelectPropertyKey(b.name)).join(',')}) IN (SELECT * FROM top_breakdowns)`;}breakdowns.forEach((breakdown, index) => {@@ -224,69 +317,26 @@ export function getChartSql({) as subQuery`;sb.joins = {};- const sql = `${getSelect()} ${getFrom()} ${getJoins()} ${getWhere()} ${getGroupBy()} ${getOrderBy()} ${getFill()}`;+ const sql = `${getWith()}${getSelect()} ${getFrom()} ${getJoins()} ${getWhere()} ${getGroupBy()} ${getOrderBy()} ${getFill()}`;console.log('-- Report --');console.log(sql.replaceAll(/[\n\r]/g, ' '));console.log('-- End --');return sql;}- // Build total_count calculation that accounts for breakdowns- // When breakdowns exist, we need to calculate total_count per breakdown groupif (breakdowns.length > 0) {- // Create a subquery that calculates total_count per breakdown group (without date grouping)- // Then reference it in the main query via JOIN- const breakdownSelects = breakdowns- .map((breakdown, index) => {- const key = `label_${index + 1}`;- const breakdownExpr = getSelectPropertyKey(breakdown.name);- return `${breakdownExpr} as ${key}`;- })- .join(', ');-- // GROUP BY needs to use the actual expressions, not aliases- const breakdownGroupByExprs = breakdowns- .map((breakdown) => getSelectPropertyKey(breakdown.name))+ const breakdownPartitionKeys = breakdowns+ .map((_, index) => `label_${index + 1}`).join(', ');- // Build the total_count subquery grouped only by breakdowns (no date)- // Extract the count expression without the alias (remove "as count")- const countExpression = sb.select.count.replace(/\s+as\s+count$/i, '');- const totalCountSubquery = `(- SELECT- ${breakdownSelects},- ${countExpression} as total_count- FROM ${sb.from}- ${getJoins()}- ${getWhere()}- GROUP BY ${breakdownGroupByExprs}- ) as total_counts`;-- // Join the total_counts subquery to get total_count per breakdown- // Match on the breakdown column values- const joinConditions = breakdowns- .map((_, index) => {- const outerKey = `label_${index + 1}`;- return `${outerKey} = total_counts.label_${index + 1}`;- })- .join(' AND ');-- sb.joins.total_counts = `LEFT JOIN ${totalCountSubquery} ON ${joinConditions}`;- // Use any() aggregate since total_count is the same for all rows in a breakdown group- sb.select.total_unique_count =- 'any(total_counts.total_count) as total_count';+ sb.select.total_unique_count = `sum(count) OVER (PARTITION BY ${breakdownPartitionKeys}) as total_count`;} else {- // No breakdowns - use a simple subquery for total count- const totalUniqueSubquery = `(- SELECT ${sb.select.count}- FROM ${sb.from}- ${getJoins()}- ${getWhere()}- )`;- sb.select.total_unique_count = `${totalUniqueSubquery} as total_count`;+ // No breakdowns - use window function without partition to get total across all rows+ // Sum the count values across all grouped rows+ sb.select.total_unique_count = 'sum(count) OVER () as total_count';}- const sql = `${getSelect()} ${getFrom()} ${getJoins()} ${getWhere()} ${getGroupBy()} ${getOrderBy()} ${getFill()}`;+ const sql = `${getWith()}${getSelect()} ${getFrom()} ${getJoins()} ${getWhere()} ${getGroupBy()} ${getOrderBy()} ${getFill()}`;console.log('-- Report --');console.log(sql.replaceAll(/[\n\r]/g, ' '));console.log('-- End --');diff --git a/packages/db/src/sql-builder.ts b/packages/db/src/sql-builder.tsindex ac1e0170e..d20d85dac 100644--- a/packages/db/src/sql-builder.ts+++ b/packages/db/src/sql-builder.ts@@ -8,6 +8,7 @@ export interface SqlBuilderObject {orderBy: Record<string, string>;from: string;joins: Record<string, string>;+ ctes: Record<string, string>;limit: number | undefined;offset: number | undefined;fill: string | undefined;@@ -25,6 +26,7 @@ export function createSqlBuilder() {orderBy: {},having: {},joins: {},+ ctes: {},limit: undefined,offset: undefined,fill: undefined,@@ -46,6 +48,14 @@ export function createSqlBuilder() {const getJoins = () =>Object.keys(sb.joins).length ? join(sb.joins, ' ') : '';const getFill = () => (sb.fill ? `WITH FILL ${sb.fill}` : '');+ const getWith = () => {+ const cteEntries = Object.entries(sb.ctes);+ if (cteEntries.length === 0) return '';+ const cteClauses = cteEntries.map(+ ([name, query]) => `${name} AS (${query})`,+ );+ return `WITH ${cteClauses.join(', ')} `;+ };return {sb,@@ -58,8 +68,13 @@ export function createSqlBuilder() {getHaving,getJoins,getFill,+ getWith,+ with: (name: string, query: string) => {+ sb.ctes[name] = query;+ },getSql: () => {const sql = [+ getWith(),getSelect(),getFrom(),getJoins(),diff --git a/packages/trpc/src/routers/chart.ts b/packages/trpc/src/routers/chart.tsindex 9c23c1b81..0debd31fe 100644--- a/packages/trpc/src/routers/chart.ts+++ b/packages/trpc/src/routers/chart.ts@@ -580,37 +580,35 @@ export const chartRouter = createTRPCRouter({sb.where.eventName = `name = ${sqlstring.escape(serie.name)}`;}- console.log('> breakdowns', input.breakdowns);+ // Collect profile fields from filters and breakdowns+ const profileFields = [+ ...serie.filters+ .filter((f) => f.name.startsWith('profile.'))+ .map((f) => f.name.replace('profile.', '')),+ ...(input.breakdowns+ ? Object.keys(input.breakdowns)+ .filter((key) => key.startsWith('profile.'))+ .map((key) => key.replace('profile.', ''))+ : []),+ ];++ if (profileFields.length > 0) {+ // Extract top-level field names and select only what's needed+ const fieldsToSelect = uniq(+ profileFields.map((f) => f.split('.')[0]),+ ).join(', ');+ sb.joins.profiles = `LEFT ANY JOIN (SELECT id, ${fieldsToSelect} FROM ${TABLE_NAMES.profiles} FINAL WHERE project_id = ${sqlstring.escape(projectId)}) as profile on profile.id = profile_id`;+ }+if (input.breakdowns) {Object.entries(input.breakdowns).forEach(([key, value]) => {- sb.where[`breakdown_${key}`] = `${key} = ${sqlstring.escape(value)}`;+ // Transform property keys (e.g., properties.method -> properties['method'])+ const propertyKey = getSelectPropertyKey(key);+ sb.where[`breakdown_${key}`] =+ `${propertyKey} = ${sqlstring.escape(value)}`;});}- // // Handle breakdowns if provided- // const anyBreakdownOnProfile = breakdowns.some((breakdown) =>- // breakdown.name.startsWith('profile.'),- // );- // const anyFilterOnProfile = [...event.filters, ...filters].some((filter) =>- // filter.name.startsWith('profile.'),- // );-- // if (anyFilterOnProfile || anyBreakdownOnProfile) {- // sb.joins.profiles = `LEFT ANY JOIN (SELECT- // id as "profile.id",- // email as "profile.email",- // first_name as "profile.first_name",- // last_name as "profile.last_name",- // properties as "profile.properties"- // FROM ${TABLE_NAMES.profiles} FINAL WHERE project_id = ${sqlstring.escape(projectId)}) as profile on profile.id = profile_id`;- // }-- // Apply breakdown filters if provided- // breakdowns.forEach((breakdown) => {- // // This is simplified - in reality we'd need to match the breakdown value- // // For now, we'll just get all profiles for the time bucket- // });-// Get unique profile IDsconst profileIds = await chQuery<{ profile_id: string }>(getSql());if (profileIds.length === 0) {