Skip to content

OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.

fix: sql issues for top level columns

Carl-Gerhard Lindesvärd4c8da2d87874
diff --git a/packages/db/src/services/chart-field-resolution.test.ts b/packages/db/src/services/chart-field-resolution.test.ts
new file mode 100644
index 000000000..dfd98e274
--- /dev/null
+++ b/packages/db/src/services/chart-field-resolution.test.ts
@@ -0,0 +1,104 @@
+/**
+ * Unit tests for the event-field resolution helpers.
+ *
+ * Background — these helpers exist because the chart, funnel, and conversion
+ * services used to inline whatever field name the dashboard sent (saved
+ * report, autocomplete picker, raw API call) directly into SQL. That worked
+ * for the common case but produced UNKNOWN_IDENTIFIER errors when:
+ *
+ * - older clients sent camelCase names (`referrerName`) that don't match
+ * the snake_case ClickHouse schema;
+ * - users picked utm_* in the property filter UI without realising those
+ * live in the `properties` map on the events table;
+ * - a saved report referenced a custom property (`temple_name`) as if it
+ * were a top-level column.
+ *
+ * All five errors logged in HyperDX 2026-05-14 → 2026-05-17 are covered here.
+ */
+import { describe, expect, it } from 'vitest';
+import {
+ isKnownEventField,
+ normalizeEventField,
+} from './chart.service';
+
+describe('normalizeEventField', () => {
+ it('rewrites camelCase aliases to their snake_case columns', () => {
+ expect(normalizeEventField('referrerName')).toBe('referrer_name');
+ expect(normalizeEventField('referrerType')).toBe('referrer_type');
+ expect(normalizeEventField('sessionId')).toBe('session_id');
+ expect(normalizeEventField('deviceId')).toBe('device_id');
+ expect(normalizeEventField('osVersion')).toBe('os_version');
+ expect(normalizeEventField('browserVersion')).toBe('browser_version');
+ });
+
+ it('routes bare utm_* names into the properties.__query.* form', () => {
+ expect(normalizeEventField('utm_source')).toBe(
+ 'properties.__query.utm_source',
+ );
+ expect(normalizeEventField('utm_medium')).toBe(
+ 'properties.__query.utm_medium',
+ );
+ expect(normalizeEventField('utm_campaign')).toBe(
+ 'properties.__query.utm_campaign',
+ );
+ });
+
+ it('passes through canonical names unchanged', () => {
+ expect(normalizeEventField('referrer_name')).toBe('referrer_name');
+ expect(normalizeEventField('country')).toBe('country');
+ expect(normalizeEventField('path')).toBe('path');
+ expect(normalizeEventField('properties.foo')).toBe('properties.foo');
+ expect(normalizeEventField('profile.email')).toBe('profile.email');
+ expect(normalizeEventField('group.plan')).toBe('group.plan');
+ expect(normalizeEventField('has_profile')).toBe('has_profile');
+ });
+
+ it('returns unknown names unchanged so callers can detect them', () => {
+ expect(normalizeEventField('temple_name')).toBe('temple_name');
+ expect(normalizeEventField('totally_made_up')).toBe('totally_made_up');
+ });
+});
+
+describe('isKnownEventField', () => {
+ it('accepts top-level events columns', () => {
+ expect(isKnownEventField('country')).toBe(true);
+ expect(isKnownEventField('path')).toBe(true);
+ expect(isKnownEventField('referrer_name')).toBe(true);
+ expect(isKnownEventField('os')).toBe(true);
+ expect(isKnownEventField('revenue')).toBe(true);
+ });
+
+ it('accepts camelCase aliases via normalization', () => {
+ expect(isKnownEventField('referrerName')).toBe(true);
+ expect(isKnownEventField('sessionId')).toBe(true);
+ });
+
+ it('accepts bare utm_* (routed through properties)', () => {
+ expect(isKnownEventField('utm_source')).toBe(true);
+ expect(isKnownEventField('utm_campaign')).toBe(true);
+ });
+
+ it('accepts properties / profile / group / has_profile paths', () => {
+ expect(isKnownEventField('properties.foo')).toBe(true);
+ expect(isKnownEventField('profile.email')).toBe(true);
+ expect(isKnownEventField('group.plan')).toBe(true);
+ expect(isKnownEventField('has_profile')).toBe(true);
+ });
+
+ it('accepts cohort breakdowns (all-cohorts and single-cohort)', () => {
+ expect(isKnownEventField('cohort')).toBe(true);
+ expect(isKnownEventField('cohort:abc-123')).toBe(true);
+ });
+
+ it('rejects unknown identifiers — these used to leak into SQL', () => {
+ // HyperDX regression: psycalc-app funnel emitted `cohort as b_0` even
+ // though the funnel doesn't support all-cohorts. The funnel-side filter
+ // is what blocks it now; this asserts the breakdown-name check itself
+ // still distinguishes properties (`properties.foo`) from raw identifiers
+ // by accepting both — the funnel/chart drop logic relies on the cohort
+ // case being valid at this layer.
+ expect(isKnownEventField('temple_name')).toBe(false);
+ expect(isKnownEventField('totally_made_up_column')).toBe(false);
+ expect(isKnownEventField('')).toBe(false);
+ });
+});
diff --git a/packages/db/src/services/chart-sql.test.ts b/packages/db/src/services/chart-sql.test.ts
index 26541bc17..dd0063fe8 100644
--- a/packages/db/src/services/chart-sql.test.ts
+++ b/packages/db/src/services/chart-sql.test.ts
@@ -233,6 +233,91 @@ describe('chart.service / getChartSql', () => {
});
await explain(sql);
});
+
+ // Regressions from HyperDX 2026-05-14 → 2026-05-17 ClickHouse error log.
+ // Saved reports / older clients send field names that don't match the events
+ // schema; the chart service used to inline them verbatim, crashing parse.
+ itCH(
+ 'normalizes camelCase filter alias (referrerName → referrer_name)',
+ async () => {
+ const sql = await getChartSql({
+ event: event({
+ filters: [
+ { name: 'referrerName', operator: 'is', value: ['email'] },
+ ],
+ }),
+ breakdowns: [],
+ interval: 'day',
+ startDate: START,
+ endDate: END,
+ projectId: PROJECT_ID,
+ timezone: 'UTC',
+ });
+ expect(sql).toContain('referrer_name');
+ expect(sql).not.toMatch(/(?<![._\w])referrerName/);
+ await explain(sql);
+ },
+ );
+
+ itCH(
+ 'routes bare utm_source filter through properties map',
+ async () => {
+ const sql = await getChartSql({
+ event: event({
+ filters: [{ name: 'utm_source', operator: 'is', value: ['awn'] }],
+ }),
+ breakdowns: [],
+ interval: 'day',
+ startDate: START,
+ endDate: END,
+ projectId: PROJECT_ID,
+ timezone: 'UTC',
+ });
+ expect(sql).toContain("properties['__query.utm_source']");
+ // The unqualified `utm_source = …` form would fail with UNKNOWN_IDENTIFIER.
+ expect(sql).not.toMatch(/(?<![._\w])utm_source\s*=/);
+ await explain(sql);
+ },
+ );
+
+ itCH(
+ 'drops unknown breakdown rather than emitting invalid identifier',
+ async () => {
+ // `temple_name` is a custom property (lives in properties map) but
+ // got saved as a top-level breakdown. The old code path emitted
+ // `SELECT temple_name as _uc_label_1 FROM events`, failing parse.
+ const sql = await getChartSql({
+ event: event(),
+ breakdowns: [breakdown('temple_name')],
+ interval: 'day',
+ startDate: START,
+ endDate: END,
+ projectId: PROJECT_ID,
+ timezone: 'UTC',
+ });
+ expect(sql).not.toMatch(/(?<![._\w])temple_name/);
+ expect(sql).not.toContain('_uc_label_1');
+ await explain(sql);
+ },
+ );
+
+ itCH('drops unknown filter rather than emitting invalid identifier', async () => {
+ const sql = await getChartSql({
+ event: event({
+ filters: [
+ { name: 'totally_made_up_column', operator: 'is', value: ['x'] },
+ ],
+ }),
+ breakdowns: [],
+ interval: 'day',
+ startDate: START,
+ endDate: END,
+ projectId: PROJECT_ID,
+ timezone: 'UTC',
+ });
+ expect(sql).not.toMatch(/totally_made_up_column/);
+ await explain(sql);
+ });
});
describe('chart.service / getAggregateChartSql', () => {
diff --git a/packages/db/src/services/chart.service.ts b/packages/db/src/services/chart.service.ts
index 7bd851cdd..8cefdeb03 100644
--- a/packages/db/src/services/chart.service.ts
+++ b/packages/db/src/services/chart.service.ts
@@ -13,6 +13,103 @@ import { formatClickhouseDate, TABLE_NAMES } from '../clickhouse/client';
import { db } from '../prisma-client';
import { createSqlBuilder } from '../sql-builder';
+// Top-level columns on the events table. Derived from the migration in
+// packages/db/code-migrations/3-init-ch.ts (+ revenue added in 6-add-revenue-
+// column.ts). Used by `resolveEventColumn` to distinguish real columns from
+// property keys and reject unknown identifiers before they reach ClickHouse.
+const EVENT_TOP_LEVEL_COLUMNS = new Set<string>([
+ 'id',
+ 'name',
+ 'sdk_name',
+ 'sdk_version',
+ 'device_id',
+ 'profile_id',
+ 'project_id',
+ 'session_id',
+ 'path',
+ 'origin',
+ 'referrer',
+ 'referrer_name',
+ 'referrer_type',
+ 'duration',
+ 'revenue',
+ 'created_at',
+ 'country',
+ 'city',
+ 'region',
+ 'longitude',
+ 'latitude',
+ 'os',
+ 'os_version',
+ 'browser',
+ 'browser_version',
+ 'device',
+ 'brand',
+ 'model',
+ 'imported_at',
+]);
+
+// Older clients / saved reports send some field names in camelCase. Map them
+// to the canonical snake_case ClickHouse column so they don't fall through as
+// unknown identifiers. The bare values (`utm_source` etc.) actually live in
+// the `properties` map — `normalizeEventField` rewrites those into the
+// `properties.__query.utm_*` form.
+const EVENT_FIELD_ALIASES: Record<string, string> = {
+ referrerName: 'referrer_name',
+ referrerType: 'referrer_type',
+ sessionId: 'session_id',
+ deviceId: 'device_id',
+ profileId: 'profile_id',
+ projectId: 'project_id',
+ osVersion: 'os_version',
+ browserVersion: 'browser_version',
+ sdkName: 'sdk_name',
+ sdkVersion: 'sdk_version',
+ createdAt: 'created_at',
+ importedAt: 'imported_at',
+};
+
+const EVENT_UTM_BARE_COLUMNS = new Set<string>([
+ 'utm_source',
+ 'utm_medium',
+ 'utm_campaign',
+ 'utm_term',
+ 'utm_content',
+]);
+
+// Normalize an incoming field name into its canonical form. Returns a string
+// suitable for `getSelectPropertyKey` / `getEventFiltersWhereClause` — i.e.
+// either a top-level column name (`referrer_name`), a `properties.foo` /
+// `profile.foo` / `group.foo` path, or `has_profile`. Unknown names are
+// returned unchanged; callers must guard with `isKnownEventField` before
+// inlining into SQL.
+export function normalizeEventField(name: string): string {
+ if (EVENT_FIELD_ALIASES[name]) {
+ return EVENT_FIELD_ALIASES[name]!;
+ }
+ if (EVENT_UTM_BARE_COLUMNS.has(name)) {
+ return `properties.__query.${name}`;
+ }
+ return name;
+}
+
+// Returns true if `name` resolves to something we can inline into SQL safely:
+// a known top-level events column, a properties / profile / group path, a
+// cohort breakdown, or `has_profile`. Used to drop unknown filters/breakdowns
+// instead of emitting invalid `SELECT cohort` / `SELECT temple_name` queries.
+export function isKnownEventField(name: string): boolean {
+ if (name === 'has_profile') return true;
+ if (isAllCohortsBreakdown(name)) return true;
+ if (extractCohortId(name)) return true;
+ if (name.startsWith('properties.')) return true;
+ if (name.startsWith('profile.')) return true;
+ if (name.startsWith('group.')) return true;
+ const normalized = normalizeEventField(name);
+ if (normalized.startsWith('properties.')) return true;
+ if (EVENT_TOP_LEVEL_COLUMNS.has(normalized)) return true;
+ return false;
+}
+
export type CohortMetadata = {
id: string;
name: string;
@@ -221,7 +318,7 @@ export function getProfilePropertySelect(property: string): string {
}
export function getSelectPropertyKey(
- property: string,
+ rawProperty: string,
projectId?: string,
cohortId?: string,
cohortName?: string,
@@ -233,6 +330,13 @@ export function getSelectPropertyKey(
*/
eventsAlias?: string,
) {
+ // Map camelCase aliases (`referrerName` → `referrer_name`) and bare UTM
+ // names (`utm_source` → `properties.__query.utm_source`) into their
+ // canonical form before doing any pattern matching. The fallback at the
+ // bottom of this function returns `property` verbatim, so without this
+ // normalization an alias would leak into the generated SQL and fail with
+ // UNKNOWN_IDENTIFIER.
+ const property = normalizeEventField(rawProperty);
const extractedCohortId = cohortId || extractCohortId(property);
if (extractedCohortId && projectId) {
@@ -302,7 +406,12 @@ export async function getChartSql({
with: addCte,
} = createSqlBuilder();
- let breakdowns = initialBreakdowns;
+ // Drop breakdowns whose field name doesn't resolve to a known events
+ // column, properties path, profile path, group path, or cohort. The chart
+ // service used to inline whatever the dashboard sent — saved reports with
+ // fields like `temple_name` (a property, not a column) reached the _uc CTE
+ // as `SELECT temple_name as _uc_label_1 FROM events`, failing parse.
+ let breakdowns = initialBreakdowns.filter((b) => isKnownEventField(b.name));
const requestedAllCohortsBreakdown = breakdowns.some((b) =>
isAllCohortsBreakdown(b.name),
);
@@ -718,7 +827,12 @@ export async function getAggregateChartSql({
}) {
const { sb, join, getJoins, with: addCte, getSql } = createSqlBuilder();
- let breakdowns = initialBreakdowns;
+ // Drop breakdowns whose field name doesn't resolve to a known events
+ // column, properties path, profile path, group path, or cohort. The chart
+ // service used to inline whatever the dashboard sent — saved reports with
+ // fields like `temple_name` (a property, not a column) reached the _uc CTE
+ // as `SELECT temple_name as _uc_label_1 FROM events`, failing parse.
+ let breakdowns = initialBreakdowns.filter((b) => isKnownEventField(b.name));
const requestedAllCohortsBreakdown = breakdowns.some((b) =>
isAllCohortsBreakdown(b.name),
);
@@ -1012,11 +1126,30 @@ export function getEventFiltersWhereClause(
* `e.properties[...]` instead of the ambiguous `properties[...]`.
*/
eventsAlias?: string,
+ /**
+ * Which physical table the WHERE clause is being built for. Affects which
+ * names count as "top-level columns" and whether bare `utm_*` gets routed
+ * into the events-specific `properties.__query.utm_*` map. Defaults to
+ * 'events' because that's where the vast majority of callers (chart,
+ * funnel, conversion, sankey, event services) target — OverviewService
+ * sets it to 'sessions' when querying the sessions table.
+ */
+ tableScope: 'events' | 'sessions' = 'events',
) {
const where: Record<string, string> = {};
filters.forEach((filter, index) => {
const id = `f${index}`;
- const { name, value, operator } = filter;
+ const { value, operator } = filter;
+ // Normalize camelCase aliases (`referrerName` → `referrer_name`) on both
+ // tables — both events and sessions schemas use snake_case. The bare-
+ // utm rewrite only applies to events because sessions stores utm_* as
+ // real top-level columns; doing it for sessions would emit
+ // `properties['__query.utm_source']` against a table that has no
+ // `properties` column.
+ const name =
+ tableScope === 'sessions'
+ ? EVENT_FIELD_ALIASES[filter.name] ?? filter.name
+ : normalizeEventField(filter.name);
if (
(operator === 'inCohort' || operator === 'notInCohort') &&
@@ -1345,6 +1478,15 @@ export function getEventFiltersWhereClause(
}
}
} else {
+ // Bare-column branch. For events queries: enforce that `name` is one
+ // of the known top-level columns (anything else would crash parse with
+ // UNKNOWN_IDENTIFIER). For sessions queries: skip the guard because
+ // the sessions table has its own column set (utm_*, entry_path, etc.)
+ // that OverviewService.getRawWhereClause already vets via its
+ // WHITELISTED_FILTERS pre-pass.
+ if (tableScope === 'events' && !EVENT_TOP_LEVEL_COLUMNS.has(name)) {
+ return;
+ }
switch (operator) {
case 'is': {
if (value.length === 1) {
diff --git a/packages/db/src/services/conversion.service.ts b/packages/db/src/services/conversion.service.ts
index ba1ce8e89..5d3724ac0 100644
--- a/packages/db/src/services/conversion.service.ts
+++ b/packages/db/src/services/conversion.service.ts
@@ -11,6 +11,7 @@ import {
fetchCohortsMetadata,
getEventFiltersWhereClause,
getSelectPropertyKey,
+ isKnownEventField,
} from './chart.service';
import { onlyReportEvents } from './reports.service';
@@ -35,6 +36,10 @@ export class ConversionService {
const funnelWindow = funnelOptions?.funnelWindow ?? 24;
const group = funnelGroup === 'profile_id' ? 'profile_id' : 'session_id';
+ // Same guard as FunnelService / getChartSql — drop breakdowns whose name
+ // can't be resolved against the events schema.
+ breakdowns = breakdowns.filter((b) => isKnownEventField(b.name));
+
const allFilters = series.flatMap((s) =>
s.type === 'event' ? s.filters ?? [] : [],
);
diff --git a/packages/db/src/services/funnel.service.ts b/packages/db/src/services/funnel.service.ts
index f3a939ffd..233f6f5f7 100644
--- a/packages/db/src/services/funnel.service.ts
+++ b/packages/db/src/services/funnel.service.ts
@@ -13,6 +13,7 @@ import {
fetchCohortsMetadata,
getEventFiltersWhereClause,
getSelectPropertyKey,
+ isKnownEventField,
} from './chart.service';
import { onlyReportEvents } from './reports.service';
@@ -226,6 +227,14 @@ export class FunnelService {
const funnelWindow = funnelOptions?.funnelWindow ?? 24;
const funnelGroup = funnelOptions?.funnelGroup;
+ // Drop breakdowns that don't resolve to a known events column, properties
+ // path, profile path, group path, or cohort. The funnel CTE inlines each
+ // breakdown's name directly via getSelectPropertyKey — a saved report with
+ // breakdown `cohort` (intended for the chart's all-cohorts feature; the
+ // funnel doesn't support it) used to produce `cohort as b_0 FROM events
+ // GROUP BY b_0`, failing parse.
+ breakdowns = breakdowns.filter((b) => isKnownEventField(b.name));
+
const eventSeries = onlyReportEvents(series);
if (eventSeries.length === 0) {
diff --git a/packages/db/src/services/overview.service.ts b/packages/db/src/services/overview.service.ts
index 78e40826b..7e3677827 100644
--- a/packages/db/src/services/overview.service.ts
+++ b/packages/db/src/services/overview.service.ts
@@ -661,7 +661,10 @@ export class OverviewService {
return [{ ...item, name: `properties.__query.${item.name}` }];
}
return [item];
- })
+ }),
+ undefined,
+ undefined,
+ type,
);
return Object.values(where).join(' AND ');
diff --git a/packages/trpc/src/routers/chart.ts b/packages/trpc/src/routers/chart.ts
index 0b1553f77..8a4c23e04 100644
--- a/packages/trpc/src/routers/chart.ts
+++ b/packages/trpc/src/routers/chart.ts
@@ -20,6 +20,8 @@ import {
getSelectPropertyKey,
getSettingsForProject,
type IServiceProfile,
+ isKnownEventField,
+ normalizeEventField,
onlyReportEvents,
sankeyService,
TABLE_NAMES,
@@ -51,17 +53,6 @@ import {
publicProcedure,
} from '../trpc';
-// utm_* are surfaced as filterable "columns" in the UI but only exist on the
-// sessions table — on events they live inside the properties map under the
-// __query.utm_* keys.
-const UTM_COLUMNS = [
- 'utm_source',
- 'utm_medium',
- 'utm_campaign',
- 'utm_term',
- 'utm_content',
-];
-
function utc(date: string | Date) {
if (typeof date === 'string') {
return date.replace('T', ' ').slice(0, 19);
@@ -396,12 +387,15 @@ export const chartRouter = createTRPCRouter({
const res = await query.execute();
values.push(...res.map((r) => String(r.values)).filter(Boolean));
} else {
- // utm_* don't exist as columns on the events table — they live in
- // the properties map. Route them through the properties code path
- // so the SELECT becomes `properties['__query.utm_source']`.
- const resolvedProperty = UTM_COLUMNS.includes(property)
- ? `properties.__query.${property}`
- : property;
+ // Normalize bare utm_* names to `properties.__query.utm_*` and rewrite
+ // camelCase aliases (`referrerName`) to their snake_case columns.
+ // Unknown identifiers (saved-report typos like `temple_name`, or
+ // misnamed columns from older clients) get an empty value list rather
+ // than crashing the autocomplete query with UNKNOWN_IDENTIFIER.
+ const resolvedProperty = normalizeEventField(property);
+ if (!isKnownEventField(resolvedProperty)) {
+ return { values: [] };
+ }
const query = clix(ch)
.select<{ values: string[] }>([
`distinct ${getSelectPropertyKey(resolvedProperty)} as values`,