Skip to content

OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.

fix(api): overview UTM and page filters give correct numbers

Carl-Gerhard Lindesvärd1fa053c86cb4
diff --git a/packages/core/src/modules/overview/overview.service.ts b/packages/core/src/modules/overview/overview.service.ts
index 97f93bdce..00c413ca6 100644
--- a/packages/core/src/modules/overview/overview.service.ts
+++ b/packages/core/src/modules/overview/overview.service.ts
@@ -84,6 +84,19 @@ const UTM_COLUMNS = [
'utm_term',
'utm_content',
];
+const UTM_PROPERTY_PREFIX = 'properties.__query.';
+
+// A session has no `path`/`origin` of its own; its entry page stands in.
+const SESSION_COLUMN_FOR_FILTER: Record<string, string> = {
+ path: 'entry_path',
+ origin: 'entry_origin',
+};
+
+// These compare against nothing, so they apply without a value.
+const VALUELESS_OPERATORS: ReadonlySet<string> = new Set([
+ 'isNull',
+ 'isNotNull',
+]);
interface MetricsRow {
bounce_rate: number;
@@ -329,8 +342,17 @@ function sessionsFilterMode(params: {
};
}
+/** A filter still being edited in the UI has no value yet and filters nothing. */
+function isFilterApplicable(filter: IChartEventFilter): boolean {
+ return (
+ (filter.value?.length ?? 0) > 0 || VALUELESS_OPERATORS.has(filter.operator)
+ );
+}
+
export function isPageFilter(filters: IChartEventFilter[]) {
- return filters.some((filter) => filter.name === 'path' && filter.value);
+ return filters.some(
+ (filter) => filter.name === 'path' && isFilterApplicable(filter)
+ );
}
export async function getMetrics(
@@ -531,45 +553,36 @@ async function getMetricsWithPageFilter(
};
}
+/**
+ * The sources widget names a UTM filter by its events spelling
+ * (`properties.__query.utm_source`), saved filters may use the bare column;
+ * both resolve to the bare name before the whitelist sees them.
+ */
+function toOverviewFilterName(name: string): string {
+ const bareName = name.startsWith(UTM_PROPERTY_PREFIX)
+ ? name.slice(UTM_PROPERTY_PREFIX.length)
+ : name;
+ return UTM_COLUMNS.includes(bareName) ? bareName : name;
+}
+
+function toTableFilterName(type: 'events' | 'sessions', name: string): string {
+ if (type === 'events') {
+ return UTM_COLUMNS.includes(name) ? `${UTM_PROPERTY_PREFIX}${name}` : name;
+ }
+ return SESSION_COLUMN_FOR_FILTER[name] ?? name;
+}
+
export function getRawWhereClause(
type: 'events' | 'sessions',
filters: IChartEventFilter[]
): SqlFragment | null {
const where = getEventFiltersWhereClause(
filters.flatMap((item) => {
- if (!WHITELISTED_FILTERS.includes(item.name)) {
+ const name = toOverviewFilterName(item.name);
+ if (!WHITELISTED_FILTERS.includes(name)) {
return [];
}
- if (type === 'sessions') {
- if (item.name === 'path') {
- return [{ ...item, name: 'entry_path' }];
- }
- if (item.name === 'origin') {
- return [{ ...item, name: 'entry_origin' }];
- }
- if (item.name.startsWith('properties.__query.utm_')) {
- return [
- {
- ...item,
- name: item.name.replace('properties.__query.utm_', 'utm_'),
- },
- ];
- }
- // sessions table has no `properties` map for arbitrary keys —
- // drop them instead of generating an invalid WHERE clause.
- if (item.name.startsWith('properties.')) {
- return [];
- }
- return [item];
- }
- // events table has no top-level utm_* columns — those live in the
- // properties map under the __query.utm_* keys. Route them through
- // getEventFiltersWhereClause's properties.* path so we emit
- // `properties['__query.utm_source']` instead of the bare column.
- if (UTM_COLUMNS.includes(item.name)) {
- return [{ ...item, name: `properties.__query.${item.name}` }];
- }
- return [item];
+ return [{ ...item, name: toTableFilterName(type, name) }];
}),
undefined,
undefined,
diff --git a/packages/core/src/modules/overview/src/metrics-page-filter.test.ts b/packages/core/src/modules/overview/src/metrics-page-filter.test.ts
index 5447ea617..b5934573a 100644
--- a/packages/core/src/modules/overview/src/metrics-page-filter.test.ts
+++ b/packages/core/src/modules/overview/src/metrics-page-filter.test.ts
@@ -21,6 +21,9 @@ const SESSION_DAY = '2026-01-05';
const EVENT_DAY = '2026-01-08';
const WINDOW_START = '2026-01-01 00:00:00';
const WINDOW_END = '2026-01-15 00:00:00';
+const DURATION_PATH = '/checkout';
+const DURATION_SESSION_ID = 'session-duration';
+const SECONDS_ON_DURATION_PATH = 60;
/** One bounced and one non-bounced session, both on `SESSION_DAY`. */
const EXPECTED_OVERALL_BOUNCE_RATE = 50;
@@ -89,6 +92,21 @@ const screenView = (id: string, index: number) => ({
path: FILTERED_PATH,
});
+/** `/home` → `DURATION_PATH` → `/home`, so the filtered view is followed only by another page. */
+const durationSessionViews = [
+ ['00000000-0000-0000-0000-0000000000c1', '/home', '10:00:00'],
+ ['00000000-0000-0000-0000-0000000000c2', DURATION_PATH, '10:00:30'],
+ ['00000000-0000-0000-0000-0000000000c3', '/home', '10:01:30'],
+].map(([id, path, time]) => ({
+ id,
+ project_id: PROJECT_ID,
+ name: 'screen_view',
+ created_at: `${SESSION_DAY} ${time}`,
+ profile_id: 'profile-duration',
+ session_id: DURATION_SESSION_ID,
+ path,
+}));
+
beforeAll(async () => {
({ ch } = await import('@openpanel/db/src/clickhouse/client'));
OV = await import('./overview.sql');
@@ -118,6 +136,7 @@ beforeAll(async () => {
values: [
screenView('00000000-0000-0000-0000-0000000000b1', 1),
screenView('00000000-0000-0000-0000-0000000000b2', 2),
+ ...durationSessionViews,
],
format: 'JSONEachRow',
});
@@ -158,3 +177,21 @@ describe('metricsWithPageFilterQuery — window-wide totals', () => {
}
});
});
+
+describe('metricsWithPageFilterQuery — session duration', () => {
+ it('measures a filtered view up to the next view of any page in its session', async () => {
+ const rows = await metricsFor(DURATION_PATH);
+ const day = rows.find((row) => row.total_screen_views > 0);
+
+ expect(day?.total_screen_views).toBe(1);
+ expect(day?.avg_session_duration).toBe(SECONDS_ON_DURATION_PATH);
+ });
+
+ it('reports 0 rather than null for a bucket whose views have no measurable duration', async () => {
+ const rows = await metricsFor(FILTERED_PATH);
+
+ for (const row of rows) {
+ expect(row.avg_session_duration).not.toBeNull();
+ }
+ });
+});
diff --git a/packages/core/src/modules/overview/src/overview-raw-where.sql.test.ts b/packages/core/src/modules/overview/src/overview-raw-where.sql.test.ts
index 7cb2ef095..3056a288e 100644
--- a/packages/core/src/modules/overview/src/overview-raw-where.sql.test.ts
+++ b/packages/core/src/modules/overview/src/overview-raw-where.sql.test.ts
@@ -22,7 +22,7 @@
import { afterAll, beforeAll, describe, expect, it, spyOn } from 'bun:test';
import { ch } from '@openpanel/db/src/clickhouse/client';
import type { SqlFragment } from '@openpanel/db/src/clickhouse/sql';
-import { getRawWhereClause } from '../overview.service';
+import { getRawWhereClause, isPageFilter } from '../overview.service';
const PROJECT_ID = 'test-sql-validation';
@@ -97,6 +97,52 @@ describe('overview.service / getRawWhereClause (UTM remapping)', () => {
expect(query_params).toEqual({ p1: 'awn' });
});
+ const UTM_FIELDS = [
+ 'utm_source',
+ 'utm_medium',
+ 'utm_campaign',
+ 'utm_term',
+ 'utm_content',
+ ];
+
+ it('accepts the properties.__query.utm_* spelling the sources widget sets', () => {
+ for (const field of UTM_FIELDS) {
+ for (const operator of ['is', 'isNot', 'contains'] as const) {
+ const filters = [
+ { name: `properties.__query.${field}`, operator, value: ['google'] },
+ ];
+ const bare = [{ name: field, operator, value: ['google'] }];
+ expect(rendered(getRawWhereClause('sessions', filters))).toEqual(
+ rendered(getRawWhereClause('sessions', bare))
+ );
+ expect(rendered(getRawWhereClause('events', filters))).toEqual(
+ rendered(getRawWhereClause('events', bare))
+ );
+ expect(rendered(getRawWhereClause('sessions', filters)).query).toMatch(
+ new RegExp(`(?<![._\\w])${field}\\b`)
+ );
+ }
+ }
+ });
+
+ it('drops a filter that has no value yet', () => {
+ expect(
+ getRawWhereClause('events', [{ name: 'path', operator: 'is', value: [] }])
+ ).toBeNull();
+ });
+
+ it('does not treat a path filter without a value as a page filter', () => {
+ expect(isPageFilter([{ name: 'path', operator: 'is', value: [] }])).toBe(
+ false
+ );
+ expect(
+ isPageFilter([{ name: 'path', operator: 'is', value: ['/cart'] }])
+ ).toBe(true);
+ expect(
+ isPageFilter([{ name: 'path', operator: 'isNotNull', value: [] }])
+ ).toBe(true);
+ });
+
it('drops non-whitelisted filters', () => {
const where = getRawWhereClause('events', [
{ name: 'malicious_column', operator: 'is', value: ['x'] },
@@ -120,6 +166,28 @@ describe('overview.service / getRawWhereClause (UTM remapping)', () => {
}
);
+ itCH(
+ 'properties.__query.utm_* filters parse against both tables',
+ async () => {
+ for (const table of ['events', 'sessions'] as const) {
+ const { query, query_params } = rendered(
+ getRawWhereClause(table, [
+ {
+ name: 'properties.__query.utm_campaign',
+ operator: 'isNot',
+ value: ['spring'],
+ },
+ ])
+ );
+ // An empty clause leaves a dangling `AND`, which EXPLAIN rejects.
+ await explain(
+ `SELECT count() FROM ${table} WHERE project_id = '${PROJECT_ID}' AND ${query}`,
+ query_params
+ );
+ }
+ }
+ );
+
itCH(
'sessions utm_source filter parses against real sessions table',
async () => {
diff --git a/packages/core/src/modules/overview/src/overview.sql.ts b/packages/core/src/modules/overview/src/overview.sql.ts
index 0a7c415bc..1e342ad83 100644
--- a/packages/core/src/modules/overview/src/overview.sql.ts
+++ b/packages/core/src/modules/overview/src/overview.sql.ts
@@ -207,17 +207,25 @@ export function metricsWithPageFilterQuery(
// re-executed once per reference, so each source is read exactly once here
// and the window-wide totals are lifted off its own `WITH ROLLUP` row rather
// than from a second scan.
+ //
+ // The filter is evaluated as a column and applied only after `lead`: a view's
+ // duration runs to the session's next view of any page, not the next view
+ // of a page that also matches the filter.
const filteredScreenViews = sql`
- SELECT
- ${dateBucket} AS date,
- profile_id,
- session_id,
- dateDiff('millisecond', created_at, lead(created_at, 1, created_at) OVER (PARTITION BY session_id ORDER BY created_at)) AS duration
- FROM events
- WHERE project_id = ${sql.string(input.projectId)}
- AND name = 'screen_view'
- AND ${dateRangeWhere('created_at', input.startDate, input.endDate)}
- ${rawWhere(input.rawEventFilterWhere)}
+ SELECT date, profile_id, session_id, duration
+ FROM (
+ SELECT
+ ${dateBucket} AS date,
+ profile_id,
+ session_id,
+ dateDiff('millisecond', created_at, lead(created_at, 1, created_at) OVER (PARTITION BY session_id ORDER BY created_at)) AS duration,
+ ${input.rawEventFilterWhere ?? sql`true`} AS matches_filter
+ FROM events
+ WHERE project_id = ${sql.string(input.projectId)}
+ AND name = 'screen_view'
+ AND ${dateRangeWhere('created_at', input.startDate, input.endDate)}
+ )
+ WHERE matches_filter
`;
const eventAgg = sql`
@@ -227,7 +235,7 @@ export function metricsWithPageFilterQuery(
uniq(session_id) AS total_sessions,
count(*) AS total_screen_views,
round((count(*) * 1.) / uniq(session_id), 2) AS views_per_session,
- round(avgIf(duration, duration > 0), 2) / 1000 AS avg_session_duration
+ ifNotFinite(round(avgIf(duration, duration > 0), 2) / 1000, 0) AS avg_session_duration
FROM filtered_screen_views
GROUP BY date
WITH ROLLUP