OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.
fix(api): overview UTM and page filters give correct numbers
Carl-Gerhard Lindesvärd1fa053c86cb4
diff --git a/packages/core/src/modules/overview/overview.service.ts b/packages/core/src/modules/overview/overview.service.tsindex 97f93bdce..00c413ca6 100644--- a/packages/core/src/modules/overview/overview.service.ts+++ b/packages/core/src/modules/overview/overview.service.ts@@ -84,6 +84,19 @@ const UTM_COLUMNS = ['utm_term','utm_content',];+const UTM_PROPERTY_PREFIX = 'properties.__query.';++// A session has no `path`/`origin` of its own; its entry page stands in.+const SESSION_COLUMN_FOR_FILTER: Record<string, string> = {+ path: 'entry_path',+ origin: 'entry_origin',+};++// These compare against nothing, so they apply without a value.+const VALUELESS_OPERATORS: ReadonlySet<string> = new Set([+ 'isNull',+ 'isNotNull',+]);interface MetricsRow {bounce_rate: number;@@ -329,8 +342,17 @@ function sessionsFilterMode(params: {};}+/** A filter still being edited in the UI has no value yet and filters nothing. */+function isFilterApplicable(filter: IChartEventFilter): boolean {+ return (+ (filter.value?.length ?? 0) > 0 || VALUELESS_OPERATORS.has(filter.operator)+ );+}+export function isPageFilter(filters: IChartEventFilter[]) {- return filters.some((filter) => filter.name === 'path' && filter.value);+ return filters.some(+ (filter) => filter.name === 'path' && isFilterApplicable(filter)+ );}export async function getMetrics(@@ -531,45 +553,36 @@ async function getMetricsWithPageFilter(};}+/**+ * The sources widget names a UTM filter by its events spelling+ * (`properties.__query.utm_source`), saved filters may use the bare column;+ * both resolve to the bare name before the whitelist sees them.+ */+function toOverviewFilterName(name: string): string {+ const bareName = name.startsWith(UTM_PROPERTY_PREFIX)+ ? name.slice(UTM_PROPERTY_PREFIX.length)+ : name;+ return UTM_COLUMNS.includes(bareName) ? bareName : name;+}++function toTableFilterName(type: 'events' | 'sessions', name: string): string {+ if (type === 'events') {+ return UTM_COLUMNS.includes(name) ? `${UTM_PROPERTY_PREFIX}${name}` : name;+ }+ return SESSION_COLUMN_FOR_FILTER[name] ?? name;+}+export function getRawWhereClause(type: 'events' | 'sessions',filters: IChartEventFilter[]): SqlFragment | null {const where = getEventFiltersWhereClause(filters.flatMap((item) => {- if (!WHITELISTED_FILTERS.includes(item.name)) {+ const name = toOverviewFilterName(item.name);+ if (!WHITELISTED_FILTERS.includes(name)) {return [];}- if (type === 'sessions') {- if (item.name === 'path') {- return [{ ...item, name: 'entry_path' }];- }- if (item.name === 'origin') {- return [{ ...item, name: 'entry_origin' }];- }- if (item.name.startsWith('properties.__query.utm_')) {- return [- {- ...item,- name: item.name.replace('properties.__query.utm_', 'utm_'),- },- ];- }- // sessions table has no `properties` map for arbitrary keys —- // drop them instead of generating an invalid WHERE clause.- if (item.name.startsWith('properties.')) {- return [];- }- return [item];- }- // events table has no top-level utm_* columns — those live in the- // properties map under the __query.utm_* keys. Route them through- // getEventFiltersWhereClause's properties.* path so we emit- // `properties['__query.utm_source']` instead of the bare column.- if (UTM_COLUMNS.includes(item.name)) {- return [{ ...item, name: `properties.__query.${item.name}` }];- }- return [item];+ return [{ ...item, name: toTableFilterName(type, name) }];}),undefined,undefined,diff --git a/packages/core/src/modules/overview/src/metrics-page-filter.test.ts b/packages/core/src/modules/overview/src/metrics-page-filter.test.tsindex 5447ea617..b5934573a 100644--- a/packages/core/src/modules/overview/src/metrics-page-filter.test.ts+++ b/packages/core/src/modules/overview/src/metrics-page-filter.test.ts@@ -21,6 +21,9 @@ const SESSION_DAY = '2026-01-05';const EVENT_DAY = '2026-01-08';const WINDOW_START = '2026-01-01 00:00:00';const WINDOW_END = '2026-01-15 00:00:00';+const DURATION_PATH = '/checkout';+const DURATION_SESSION_ID = 'session-duration';+const SECONDS_ON_DURATION_PATH = 60;/** One bounced and one non-bounced session, both on `SESSION_DAY`. */const EXPECTED_OVERALL_BOUNCE_RATE = 50;@@ -89,6 +92,21 @@ const screenView = (id: string, index: number) => ({path: FILTERED_PATH,});+/** `/home` → `DURATION_PATH` → `/home`, so the filtered view is followed only by another page. */+const durationSessionViews = [+ ['00000000-0000-0000-0000-0000000000c1', '/home', '10:00:00'],+ ['00000000-0000-0000-0000-0000000000c2', DURATION_PATH, '10:00:30'],+ ['00000000-0000-0000-0000-0000000000c3', '/home', '10:01:30'],+].map(([id, path, time]) => ({+ id,+ project_id: PROJECT_ID,+ name: 'screen_view',+ created_at: `${SESSION_DAY} ${time}`,+ profile_id: 'profile-duration',+ session_id: DURATION_SESSION_ID,+ path,+}));+beforeAll(async () => {({ ch } = await import('@openpanel/db/src/clickhouse/client'));OV = await import('./overview.sql');@@ -118,6 +136,7 @@ beforeAll(async () => {values: [screenView('00000000-0000-0000-0000-0000000000b1', 1),screenView('00000000-0000-0000-0000-0000000000b2', 2),+ ...durationSessionViews,],format: 'JSONEachRow',});@@ -158,3 +177,21 @@ describe('metricsWithPageFilterQuery — window-wide totals', () => {}});});++describe('metricsWithPageFilterQuery — session duration', () => {+ it('measures a filtered view up to the next view of any page in its session', async () => {+ const rows = await metricsFor(DURATION_PATH);+ const day = rows.find((row) => row.total_screen_views > 0);++ expect(day?.total_screen_views).toBe(1);+ expect(day?.avg_session_duration).toBe(SECONDS_ON_DURATION_PATH);+ });++ it('reports 0 rather than null for a bucket whose views have no measurable duration', async () => {+ const rows = await metricsFor(FILTERED_PATH);++ for (const row of rows) {+ expect(row.avg_session_duration).not.toBeNull();+ }+ });+});diff --git a/packages/core/src/modules/overview/src/overview-raw-where.sql.test.ts b/packages/core/src/modules/overview/src/overview-raw-where.sql.test.tsindex 7cb2ef095..3056a288e 100644--- a/packages/core/src/modules/overview/src/overview-raw-where.sql.test.ts+++ b/packages/core/src/modules/overview/src/overview-raw-where.sql.test.ts@@ -22,7 +22,7 @@import { afterAll, beforeAll, describe, expect, it, spyOn } from 'bun:test';import { ch } from '@openpanel/db/src/clickhouse/client';import type { SqlFragment } from '@openpanel/db/src/clickhouse/sql';-import { getRawWhereClause } from '../overview.service';+import { getRawWhereClause, isPageFilter } from '../overview.service';const PROJECT_ID = 'test-sql-validation';@@ -97,6 +97,52 @@ describe('overview.service / getRawWhereClause (UTM remapping)', () => {expect(query_params).toEqual({ p1: 'awn' });});+ const UTM_FIELDS = [+ 'utm_source',+ 'utm_medium',+ 'utm_campaign',+ 'utm_term',+ 'utm_content',+ ];++ it('accepts the properties.__query.utm_* spelling the sources widget sets', () => {+ for (const field of UTM_FIELDS) {+ for (const operator of ['is', 'isNot', 'contains'] as const) {+ const filters = [+ { name: `properties.__query.${field}`, operator, value: ['google'] },+ ];+ const bare = [{ name: field, operator, value: ['google'] }];+ expect(rendered(getRawWhereClause('sessions', filters))).toEqual(+ rendered(getRawWhereClause('sessions', bare))+ );+ expect(rendered(getRawWhereClause('events', filters))).toEqual(+ rendered(getRawWhereClause('events', bare))+ );+ expect(rendered(getRawWhereClause('sessions', filters)).query).toMatch(+ new RegExp(`(?<![._\\w])${field}\\b`)+ );+ }+ }+ });++ it('drops a filter that has no value yet', () => {+ expect(+ getRawWhereClause('events', [{ name: 'path', operator: 'is', value: [] }])+ ).toBeNull();+ });++ it('does not treat a path filter without a value as a page filter', () => {+ expect(isPageFilter([{ name: 'path', operator: 'is', value: [] }])).toBe(+ false+ );+ expect(+ isPageFilter([{ name: 'path', operator: 'is', value: ['/cart'] }])+ ).toBe(true);+ expect(+ isPageFilter([{ name: 'path', operator: 'isNotNull', value: [] }])+ ).toBe(true);+ });+it('drops non-whitelisted filters', () => {const where = getRawWhereClause('events', [{ name: 'malicious_column', operator: 'is', value: ['x'] },@@ -120,6 +166,28 @@ describe('overview.service / getRawWhereClause (UTM remapping)', () => {});+ itCH(+ 'properties.__query.utm_* filters parse against both tables',+ async () => {+ for (const table of ['events', 'sessions'] as const) {+ const { query, query_params } = rendered(+ getRawWhereClause(table, [+ {+ name: 'properties.__query.utm_campaign',+ operator: 'isNot',+ value: ['spring'],+ },+ ])+ );+ // An empty clause leaves a dangling `AND`, which EXPLAIN rejects.+ await explain(+ `SELECT count() FROM ${table} WHERE project_id = '${PROJECT_ID}' AND ${query}`,+ query_params+ );+ }+ }+ );+itCH('sessions utm_source filter parses against real sessions table',async () => {diff --git a/packages/core/src/modules/overview/src/overview.sql.ts b/packages/core/src/modules/overview/src/overview.sql.tsindex 0a7c415bc..1e342ad83 100644--- a/packages/core/src/modules/overview/src/overview.sql.ts+++ b/packages/core/src/modules/overview/src/overview.sql.ts@@ -207,17 +207,25 @@ export function metricsWithPageFilterQuery(// re-executed once per reference, so each source is read exactly once here// and the window-wide totals are lifted off its own `WITH ROLLUP` row rather// than from a second scan.+ //+ // The filter is evaluated as a column and applied only after `lead`: a view's+ // duration runs to the session's next view of any page, not the next view+ // of a page that also matches the filter.const filteredScreenViews = sql`- SELECT- ${dateBucket} AS date,- profile_id,- session_id,- dateDiff('millisecond', created_at, lead(created_at, 1, created_at) OVER (PARTITION BY session_id ORDER BY created_at)) AS duration- FROM events- WHERE project_id = ${sql.string(input.projectId)}- AND name = 'screen_view'- AND ${dateRangeWhere('created_at', input.startDate, input.endDate)}- ${rawWhere(input.rawEventFilterWhere)}+ SELECT date, profile_id, session_id, duration+ FROM (+ SELECT+ ${dateBucket} AS date,+ profile_id,+ session_id,+ dateDiff('millisecond', created_at, lead(created_at, 1, created_at) OVER (PARTITION BY session_id ORDER BY created_at)) AS duration,+ ${input.rawEventFilterWhere ?? sql`true`} AS matches_filter+ FROM events+ WHERE project_id = ${sql.string(input.projectId)}+ AND name = 'screen_view'+ AND ${dateRangeWhere('created_at', input.startDate, input.endDate)}+ )+ WHERE matches_filter`;const eventAgg = sql`@@ -227,7 +235,7 @@ export function metricsWithPageFilterQuery(uniq(session_id) AS total_sessions,count(*) AS total_screen_views,round((count(*) * 1.) / uniq(session_id), 2) AS views_per_session,- round(avgIf(duration, duration > 0), 2) / 1000 AS avg_session_duration+ ifNotFinite(round(avgIf(duration, duration > 0), 2) / 1000, 0) AS avg_session_durationFROM filtered_screen_viewsGROUP BY dateWITH ROLLUP