OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.
fix: sql issues for top level columns
Carl-Gerhard Lindesvärd4c8da2d87874
diff --git a/packages/db/src/services/chart-field-resolution.test.ts b/packages/db/src/services/chart-field-resolution.test.tsnew file mode 100644index 000000000..dfd98e274--- /dev/null+++ b/packages/db/src/services/chart-field-resolution.test.ts@@ -0,0 +1,104 @@+/**+ * Unit tests for the event-field resolution helpers.+ *+ * Background — these helpers exist because the chart, funnel, and conversion+ * services used to inline whatever field name the dashboard sent (saved+ * report, autocomplete picker, raw API call) directly into SQL. That worked+ * for the common case but produced UNKNOWN_IDENTIFIER errors when:+ *+ * - older clients sent camelCase names (`referrerName`) that don't match+ * the snake_case ClickHouse schema;+ * - users picked utm_* in the property filter UI without realising those+ * live in the `properties` map on the events table;+ * - a saved report referenced a custom property (`temple_name`) as if it+ * were a top-level column.+ *+ * All five errors logged in HyperDX 2026-05-14 → 2026-05-17 are covered here.+ */+import { describe, expect, it } from 'vitest';+import {+ isKnownEventField,+ normalizeEventField,+} from './chart.service';++describe('normalizeEventField', () => {+ it('rewrites camelCase aliases to their snake_case columns', () => {+ expect(normalizeEventField('referrerName')).toBe('referrer_name');+ expect(normalizeEventField('referrerType')).toBe('referrer_type');+ expect(normalizeEventField('sessionId')).toBe('session_id');+ expect(normalizeEventField('deviceId')).toBe('device_id');+ expect(normalizeEventField('osVersion')).toBe('os_version');+ expect(normalizeEventField('browserVersion')).toBe('browser_version');+ });++ it('routes bare utm_* names into the properties.__query.* form', () => {+ expect(normalizeEventField('utm_source')).toBe(+ 'properties.__query.utm_source',+ );+ expect(normalizeEventField('utm_medium')).toBe(+ 'properties.__query.utm_medium',+ );+ expect(normalizeEventField('utm_campaign')).toBe(+ 'properties.__query.utm_campaign',+ );+ });++ it('passes through canonical names unchanged', () => {+ expect(normalizeEventField('referrer_name')).toBe('referrer_name');+ expect(normalizeEventField('country')).toBe('country');+ expect(normalizeEventField('path')).toBe('path');+ expect(normalizeEventField('properties.foo')).toBe('properties.foo');+ expect(normalizeEventField('profile.email')).toBe('profile.email');+ expect(normalizeEventField('group.plan')).toBe('group.plan');+ expect(normalizeEventField('has_profile')).toBe('has_profile');+ });++ it('returns unknown names unchanged so callers can detect them', () => {+ expect(normalizeEventField('temple_name')).toBe('temple_name');+ expect(normalizeEventField('totally_made_up')).toBe('totally_made_up');+ });+});++describe('isKnownEventField', () => {+ it('accepts top-level events columns', () => {+ expect(isKnownEventField('country')).toBe(true);+ expect(isKnownEventField('path')).toBe(true);+ expect(isKnownEventField('referrer_name')).toBe(true);+ expect(isKnownEventField('os')).toBe(true);+ expect(isKnownEventField('revenue')).toBe(true);+ });++ it('accepts camelCase aliases via normalization', () => {+ expect(isKnownEventField('referrerName')).toBe(true);+ expect(isKnownEventField('sessionId')).toBe(true);+ });++ it('accepts bare utm_* (routed through properties)', () => {+ expect(isKnownEventField('utm_source')).toBe(true);+ expect(isKnownEventField('utm_campaign')).toBe(true);+ });++ it('accepts properties / profile / group / has_profile paths', () => {+ expect(isKnownEventField('properties.foo')).toBe(true);+ expect(isKnownEventField('profile.email')).toBe(true);+ expect(isKnownEventField('group.plan')).toBe(true);+ expect(isKnownEventField('has_profile')).toBe(true);+ });++ it('accepts cohort breakdowns (all-cohorts and single-cohort)', () => {+ expect(isKnownEventField('cohort')).toBe(true);+ expect(isKnownEventField('cohort:abc-123')).toBe(true);+ });++ it('rejects unknown identifiers — these used to leak into SQL', () => {+ // HyperDX regression: psycalc-app funnel emitted `cohort as b_0` even+ // though the funnel doesn't support all-cohorts. The funnel-side filter+ // is what blocks it now; this asserts the breakdown-name check itself+ // still distinguishes properties (`properties.foo`) from raw identifiers+ // by accepting both — the funnel/chart drop logic relies on the cohort+ // case being valid at this layer.+ expect(isKnownEventField('temple_name')).toBe(false);+ expect(isKnownEventField('totally_made_up_column')).toBe(false);+ expect(isKnownEventField('')).toBe(false);+ });+});diff --git a/packages/db/src/services/chart-sql.test.ts b/packages/db/src/services/chart-sql.test.tsindex 26541bc17..dd0063fe8 100644--- a/packages/db/src/services/chart-sql.test.ts+++ b/packages/db/src/services/chart-sql.test.ts@@ -233,6 +233,91 @@ describe('chart.service / getChartSql', () => {});await explain(sql);});++ // Regressions from HyperDX 2026-05-14 → 2026-05-17 ClickHouse error log.+ // Saved reports / older clients send field names that don't match the events+ // schema; the chart service used to inline them verbatim, crashing parse.+ itCH(+ 'normalizes camelCase filter alias (referrerName → referrer_name)',+ async () => {+ const sql = await getChartSql({+ event: event({+ filters: [+ { name: 'referrerName', operator: 'is', value: ['email'] },+ ],+ }),+ breakdowns: [],+ interval: 'day',+ startDate: START,+ endDate: END,+ projectId: PROJECT_ID,+ timezone: 'UTC',+ });+ expect(sql).toContain('referrer_name');+ expect(sql).not.toMatch(/(?<![._\w])referrerName/);+ await explain(sql);+ },+ );++ itCH(+ 'routes bare utm_source filter through properties map',+ async () => {+ const sql = await getChartSql({+ event: event({+ filters: [{ name: 'utm_source', operator: 'is', value: ['awn'] }],+ }),+ breakdowns: [],+ interval: 'day',+ startDate: START,+ endDate: END,+ projectId: PROJECT_ID,+ timezone: 'UTC',+ });+ expect(sql).toContain("properties['__query.utm_source']");+ // The unqualified `utm_source = …` form would fail with UNKNOWN_IDENTIFIER.+ expect(sql).not.toMatch(/(?<![._\w])utm_source\s*=/);+ await explain(sql);+ },+ );++ itCH(+ 'drops unknown breakdown rather than emitting invalid identifier',+ async () => {+ // `temple_name` is a custom property (lives in properties map) but+ // got saved as a top-level breakdown. The old code path emitted+ // `SELECT temple_name as _uc_label_1 FROM events`, failing parse.+ const sql = await getChartSql({+ event: event(),+ breakdowns: [breakdown('temple_name')],+ interval: 'day',+ startDate: START,+ endDate: END,+ projectId: PROJECT_ID,+ timezone: 'UTC',+ });+ expect(sql).not.toMatch(/(?<![._\w])temple_name/);+ expect(sql).not.toContain('_uc_label_1');+ await explain(sql);+ },+ );++ itCH('drops unknown filter rather than emitting invalid identifier', async () => {+ const sql = await getChartSql({+ event: event({+ filters: [+ { name: 'totally_made_up_column', operator: 'is', value: ['x'] },+ ],+ }),+ breakdowns: [],+ interval: 'day',+ startDate: START,+ endDate: END,+ projectId: PROJECT_ID,+ timezone: 'UTC',+ });+ expect(sql).not.toMatch(/totally_made_up_column/);+ await explain(sql);+ });});describe('chart.service / getAggregateChartSql', () => {diff --git a/packages/db/src/services/chart.service.ts b/packages/db/src/services/chart.service.tsindex 7bd851cdd..8cefdeb03 100644--- a/packages/db/src/services/chart.service.ts+++ b/packages/db/src/services/chart.service.ts@@ -13,6 +13,103 @@ import { formatClickhouseDate, TABLE_NAMES } from '../clickhouse/client';import { db } from '../prisma-client';import { createSqlBuilder } from '../sql-builder';+// Top-level columns on the events table. Derived from the migration in+// packages/db/code-migrations/3-init-ch.ts (+ revenue added in 6-add-revenue-+// column.ts). Used by `resolveEventColumn` to distinguish real columns from+// property keys and reject unknown identifiers before they reach ClickHouse.+const EVENT_TOP_LEVEL_COLUMNS = new Set<string>([+ 'id',+ 'name',+ 'sdk_name',+ 'sdk_version',+ 'device_id',+ 'profile_id',+ 'project_id',+ 'session_id',+ 'path',+ 'origin',+ 'referrer',+ 'referrer_name',+ 'referrer_type',+ 'duration',+ 'revenue',+ 'created_at',+ 'country',+ 'city',+ 'region',+ 'longitude',+ 'latitude',+ 'os',+ 'os_version',+ 'browser',+ 'browser_version',+ 'device',+ 'brand',+ 'model',+ 'imported_at',+]);++// Older clients / saved reports send some field names in camelCase. Map them+// to the canonical snake_case ClickHouse column so they don't fall through as+// unknown identifiers. The bare values (`utm_source` etc.) actually live in+// the `properties` map — `normalizeEventField` rewrites those into the+// `properties.__query.utm_*` form.+const EVENT_FIELD_ALIASES: Record<string, string> = {+ referrerName: 'referrer_name',+ referrerType: 'referrer_type',+ sessionId: 'session_id',+ deviceId: 'device_id',+ profileId: 'profile_id',+ projectId: 'project_id',+ osVersion: 'os_version',+ browserVersion: 'browser_version',+ sdkName: 'sdk_name',+ sdkVersion: 'sdk_version',+ createdAt: 'created_at',+ importedAt: 'imported_at',+};++const EVENT_UTM_BARE_COLUMNS = new Set<string>([+ 'utm_source',+ 'utm_medium',+ 'utm_campaign',+ 'utm_term',+ 'utm_content',+]);++// Normalize an incoming field name into its canonical form. Returns a string+// suitable for `getSelectPropertyKey` / `getEventFiltersWhereClause` — i.e.+// either a top-level column name (`referrer_name`), a `properties.foo` /+// `profile.foo` / `group.foo` path, or `has_profile`. Unknown names are+// returned unchanged; callers must guard with `isKnownEventField` before+// inlining into SQL.+export function normalizeEventField(name: string): string {+ if (EVENT_FIELD_ALIASES[name]) {+ return EVENT_FIELD_ALIASES[name]!;+ }+ if (EVENT_UTM_BARE_COLUMNS.has(name)) {+ return `properties.__query.${name}`;+ }+ return name;+}++// Returns true if `name` resolves to something we can inline into SQL safely:+// a known top-level events column, a properties / profile / group path, a+// cohort breakdown, or `has_profile`. Used to drop unknown filters/breakdowns+// instead of emitting invalid `SELECT cohort` / `SELECT temple_name` queries.+export function isKnownEventField(name: string): boolean {+ if (name === 'has_profile') return true;+ if (isAllCohortsBreakdown(name)) return true;+ if (extractCohortId(name)) return true;+ if (name.startsWith('properties.')) return true;+ if (name.startsWith('profile.')) return true;+ if (name.startsWith('group.')) return true;+ const normalized = normalizeEventField(name);+ if (normalized.startsWith('properties.')) return true;+ if (EVENT_TOP_LEVEL_COLUMNS.has(normalized)) return true;+ return false;+}+export type CohortMetadata = {id: string;name: string;@@ -221,7 +318,7 @@ export function getProfilePropertySelect(property: string): string {}export function getSelectPropertyKey(- property: string,+ rawProperty: string,projectId?: string,cohortId?: string,cohortName?: string,@@ -233,6 +330,13 @@ export function getSelectPropertyKey(*/eventsAlias?: string,) {+ // Map camelCase aliases (`referrerName` → `referrer_name`) and bare UTM+ // names (`utm_source` → `properties.__query.utm_source`) into their+ // canonical form before doing any pattern matching. The fallback at the+ // bottom of this function returns `property` verbatim, so without this+ // normalization an alias would leak into the generated SQL and fail with+ // UNKNOWN_IDENTIFIER.+ const property = normalizeEventField(rawProperty);const extractedCohortId = cohortId || extractCohortId(property);if (extractedCohortId && projectId) {@@ -302,7 +406,12 @@ export async function getChartSql({with: addCte,} = createSqlBuilder();- let breakdowns = initialBreakdowns;+ // Drop breakdowns whose field name doesn't resolve to a known events+ // column, properties path, profile path, group path, or cohort. The chart+ // service used to inline whatever the dashboard sent — saved reports with+ // fields like `temple_name` (a property, not a column) reached the _uc CTE+ // as `SELECT temple_name as _uc_label_1 FROM events`, failing parse.+ let breakdowns = initialBreakdowns.filter((b) => isKnownEventField(b.name));const requestedAllCohortsBreakdown = breakdowns.some((b) =>isAllCohortsBreakdown(b.name),);@@ -718,7 +827,12 @@ export async function getAggregateChartSql({}) {const { sb, join, getJoins, with: addCte, getSql } = createSqlBuilder();- let breakdowns = initialBreakdowns;+ // Drop breakdowns whose field name doesn't resolve to a known events+ // column, properties path, profile path, group path, or cohort. The chart+ // service used to inline whatever the dashboard sent — saved reports with+ // fields like `temple_name` (a property, not a column) reached the _uc CTE+ // as `SELECT temple_name as _uc_label_1 FROM events`, failing parse.+ let breakdowns = initialBreakdowns.filter((b) => isKnownEventField(b.name));const requestedAllCohortsBreakdown = breakdowns.some((b) =>isAllCohortsBreakdown(b.name),);@@ -1012,11 +1126,30 @@ export function getEventFiltersWhereClause(* `e.properties[...]` instead of the ambiguous `properties[...]`.*/eventsAlias?: string,+ /**+ * Which physical table the WHERE clause is being built for. Affects which+ * names count as "top-level columns" and whether bare `utm_*` gets routed+ * into the events-specific `properties.__query.utm_*` map. Defaults to+ * 'events' because that's where the vast majority of callers (chart,+ * funnel, conversion, sankey, event services) target — OverviewService+ * sets it to 'sessions' when querying the sessions table.+ */+ tableScope: 'events' | 'sessions' = 'events',) {const where: Record<string, string> = {};filters.forEach((filter, index) => {const id = `f${index}`;- const { name, value, operator } = filter;+ const { value, operator } = filter;+ // Normalize camelCase aliases (`referrerName` → `referrer_name`) on both+ // tables — both events and sessions schemas use snake_case. The bare-+ // utm rewrite only applies to events because sessions stores utm_* as+ // real top-level columns; doing it for sessions would emit+ // `properties['__query.utm_source']` against a table that has no+ // `properties` column.+ const name =+ tableScope === 'sessions'+ ? EVENT_FIELD_ALIASES[filter.name] ?? filter.name+ : normalizeEventField(filter.name);if ((operator === 'inCohort' || operator === 'notInCohort') &&@@ -1345,6 +1478,15 @@ export function getEventFiltersWhereClause(}}} else {+ // Bare-column branch. For events queries: enforce that `name` is one+ // of the known top-level columns (anything else would crash parse with+ // UNKNOWN_IDENTIFIER). For sessions queries: skip the guard because+ // the sessions table has its own column set (utm_*, entry_path, etc.)+ // that OverviewService.getRawWhereClause already vets via its+ // WHITELISTED_FILTERS pre-pass.+ if (tableScope === 'events' && !EVENT_TOP_LEVEL_COLUMNS.has(name)) {+ return;+ }switch (operator) {case 'is': {if (value.length === 1) {diff --git a/packages/db/src/services/conversion.service.ts b/packages/db/src/services/conversion.service.tsindex ba1ce8e89..5d3724ac0 100644--- a/packages/db/src/services/conversion.service.ts+++ b/packages/db/src/services/conversion.service.ts@@ -11,6 +11,7 @@ import {fetchCohortsMetadata,getEventFiltersWhereClause,getSelectPropertyKey,+ isKnownEventField,} from './chart.service';import { onlyReportEvents } from './reports.service';@@ -35,6 +36,10 @@ export class ConversionService {const funnelWindow = funnelOptions?.funnelWindow ?? 24;const group = funnelGroup === 'profile_id' ? 'profile_id' : 'session_id';+ // Same guard as FunnelService / getChartSql — drop breakdowns whose name+ // can't be resolved against the events schema.+ breakdowns = breakdowns.filter((b) => isKnownEventField(b.name));+const allFilters = series.flatMap((s) =>s.type === 'event' ? s.filters ?? [] : [],);diff --git a/packages/db/src/services/funnel.service.ts b/packages/db/src/services/funnel.service.tsindex f3a939ffd..233f6f5f7 100644--- a/packages/db/src/services/funnel.service.ts+++ b/packages/db/src/services/funnel.service.ts@@ -13,6 +13,7 @@ import {fetchCohortsMetadata,getEventFiltersWhereClause,getSelectPropertyKey,+ isKnownEventField,} from './chart.service';import { onlyReportEvents } from './reports.service';@@ -226,6 +227,14 @@ export class FunnelService {const funnelWindow = funnelOptions?.funnelWindow ?? 24;const funnelGroup = funnelOptions?.funnelGroup;+ // Drop breakdowns that don't resolve to a known events column, properties+ // path, profile path, group path, or cohort. The funnel CTE inlines each+ // breakdown's name directly via getSelectPropertyKey — a saved report with+ // breakdown `cohort` (intended for the chart's all-cohorts feature; the+ // funnel doesn't support it) used to produce `cohort as b_0 FROM events+ // GROUP BY b_0`, failing parse.+ breakdowns = breakdowns.filter((b) => isKnownEventField(b.name));+const eventSeries = onlyReportEvents(series);if (eventSeries.length === 0) {diff --git a/packages/db/src/services/overview.service.ts b/packages/db/src/services/overview.service.tsindex 78e40826b..7e3677827 100644--- a/packages/db/src/services/overview.service.ts+++ b/packages/db/src/services/overview.service.ts@@ -661,7 +661,10 @@ export class OverviewService {return [{ ...item, name: `properties.__query.${item.name}` }];}return [item];- })+ }),+ undefined,+ undefined,+ type,);return Object.values(where).join(' AND ');diff --git a/packages/trpc/src/routers/chart.ts b/packages/trpc/src/routers/chart.tsindex 0b1553f77..8a4c23e04 100644--- a/packages/trpc/src/routers/chart.ts+++ b/packages/trpc/src/routers/chart.ts@@ -20,6 +20,8 @@ import {getSelectPropertyKey,getSettingsForProject,type IServiceProfile,+ isKnownEventField,+ normalizeEventField,onlyReportEvents,sankeyService,TABLE_NAMES,@@ -51,17 +53,6 @@ import {publicProcedure,} from '../trpc';-// utm_* are surfaced as filterable "columns" in the UI but only exist on the-// sessions table — on events they live inside the properties map under the-// __query.utm_* keys.-const UTM_COLUMNS = [- 'utm_source',- 'utm_medium',- 'utm_campaign',- 'utm_term',- 'utm_content',-];-function utc(date: string | Date) {if (typeof date === 'string') {return date.replace('T', ' ').slice(0, 19);@@ -396,12 +387,15 @@ export const chartRouter = createTRPCRouter({const res = await query.execute();values.push(...res.map((r) => String(r.values)).filter(Boolean));} else {- // utm_* don't exist as columns on the events table — they live in- // the properties map. Route them through the properties code path- // so the SELECT becomes `properties['__query.utm_source']`.- const resolvedProperty = UTM_COLUMNS.includes(property)- ? `properties.__query.${property}`- : property;+ // Normalize bare utm_* names to `properties.__query.utm_*` and rewrite+ // camelCase aliases (`referrerName`) to their snake_case columns.+ // Unknown identifiers (saved-report typos like `temple_name`, or+ // misnamed columns from older clients) get an empty value list rather+ // than crashing the autocomplete query with UNKNOWN_IDENTIFIER.+ const resolvedProperty = normalizeEventField(property);+ if (!isKnownEventField(resolvedProperty)) {+ return { values: [] };+ }const query = clix(ch).select<{ values: string[] }>([`distinct ${getSelectPropertyKey(resolvedProperty)} as values`,