Umami is a privacy-first web and product analytics platform. Traffic, campaigns, behavior, conversions, and revenue in one place. No cookies, no surveillance, self-hosted or in the cloud.
TypeScript analyticsaudience-segmentationchartscohort-analysisgoogle-analyticsproduct-analyticsstatisticsuser-journeyweb-analytics
Fix excludeBounce query, simplify bounce definition for SPA websites, query routing for performance, add tests
Francis Caob3aec54bb3d5
diff --git a/docker-compose.yml b/docker-compose.ymlindex bc8d8da08..348c294ca 100644--- a/docker-compose.yml+++ b/docker-compose.yml@@ -7,7 +7,6 @@ services:environment:DATABASE_URL: postgresql://umami:umami@db:5432/umamiAPP_SECRET: replace-me-with-a-random-string- # BOUNCE_THRESHOLD: 2 # OPTIONAL: set the minimum amount custom events to trigger a bouncedepends_on:db:condition: service_healthydiff --git a/src/lib/clickhouse.ts b/src/lib/clickhouse.tsindex bfce9ba62..7d3331dd7 100644--- a/src/lib/clickhouse.ts+++ b/src/lib/clickhouse.ts@@ -177,13 +177,17 @@ function getExcludeBounceQuery(filters: Record<string, any>) {}return `join- (select distinct session_id as exclude_session_id, visit_id as exclude_visit_id+ (select session_id as exclude_session_id, visit_id as exclude_visit_idfrom website_event_stats_hourlywhere website_id = {websiteId:UUID}and created_at between {startDate:DateTime64} and {endDate:DateTime64}- and event_type = 1+ and event_type != 5group by session_id, visit_idhaving sum(views) > 1+ or (+ sum(views) = 1+ and sumIf(length(event_name), event_type = 2) > 0+ )) excludeBounceon excludeBounce.exclude_session_id = website_event.session_idand excludeBounce.exclude_visit_id = website_event.visit_iddiff --git a/src/lib/constants.ts b/src/lib/constants.tsindex 5420bc6cc..89e250e73 100644--- a/src/lib/constants.ts+++ b/src/lib/constants.ts@@ -1,5 +1,4 @@export const CURRENT_VERSION = process.env.currentVersion;-export const BOUNCE_THRESHOLD = Math.max(1, Number.parseInt(process.env.BOUNCE_THRESHOLD, 10) || 1);export const AUTH_TOKEN = 'umami.auth';export const LOCALE_CONFIG = 'umami.locale';export const TIMEZONE_CONFIG = 'umami.timezone';diff --git a/src/lib/prisma.ts b/src/lib/prisma.tsindex 18b53a1cb..5c76aef4a 100644--- a/src/lib/prisma.ts+++ b/src/lib/prisma.ts@@ -2,13 +2,7 @@ import { PrismaPg } from '@prisma/adapter-pg';import { readReplicas } from '@prisma/extension-read-replicas';import debug from 'debug';import { PrismaClient } from '@/generated/prisma/client';-import {- DATA_TYPE,- DEFAULT_PAGE_SIZE,- FILTER_COLUMNS,- OPERATORS,- SESSION_COLUMNS,-} from './constants';+import { DATA_TYPE, DEFAULT_PAGE_SIZE, FILTER_COLUMNS, OPERATORS, SESSION_COLUMNS } from './constants';import { filtersObjectToArray } from './params';import type { Operator, PropertyFilter, QueryFilters, QueryOptions } from './types';@@ -249,13 +243,17 @@ function getExcludeBounceQuery(filters: Record<string, any>) {}return `join- (select distinct session_id, visit_id+ (select session_id, visit_idfrom website_eventwhere website_id = {{websiteId}}and created_at between {{startDate}} and {{endDate}}- and event_type = 1+ and event_type != 5group by session_id, visit_id- having count(*) > 1+ having sum(case when event_type NOT IN (2, 5) then 1 else 0 end) > 1+ or (+ sum(case when event_type NOT IN (2, 5) then 1 else 0 end) = 1+ and sum(case when event_type = 2 then 1 else 0 end) > 0+ )) excludeBounceon excludeBounce.session_id = website_event.session_idand excludeBounce.visit_id = website_event.visit_iddiff --git a/src/permissions/website.test.ts b/src/permissions/website.test.tsindex b9c00a103..13679f3b6 100644--- a/src/permissions/website.test.ts+++ b/src/permissions/website.test.ts@@ -1,6 +1,5 @@import { beforeEach, describe, expect, test, vi } from 'vitest';import { getEntity } from '@/lib/entity';-import prisma from '@/lib/prisma';import { getTeamUser, getWebsite } from '@/queries/prisma';import {canCreateWebsite,@@ -13,6 +12,11 @@ import {canViewWebsite,} from './website';+const { websiteFindManyMock, teamUserFindManyMock } = vi.hoisted(() => ({+ websiteFindManyMock: vi.fn(),+ teamUserFindManyMock: vi.fn(),+}));+vi.mock('@/lib/entity', () => ({getEntity: vi.fn(),}));@@ -26,10 +30,10 @@ vi.mock('@/lib/prisma', () => ({default: {client: {website: {- findMany: vi.fn(),+ findMany: websiteFindManyMock,},teamUser: {- findMany: vi.fn(),+ findMany: teamUserFindManyMock,},},},@@ -54,8 +58,8 @@ beforeEach(() => {vi.mocked(getEntity).mockReset();vi.mocked(getWebsite).mockReset();vi.mocked(getTeamUser).mockReset();- vi.mocked(prisma.client.website.findMany).mockReset();- vi.mocked(prisma.client.teamUser.findMany).mockReset();+ websiteFindManyMock.mockReset();+ teamUserFindManyMock.mockReset();});describe('canViewWebsite', () => {@@ -139,12 +143,12 @@ describe('canViewBatchWebsites', () => {});test('returns owned, team, and share allowed ids for a user', async () => {- vi.mocked(prisma.client.website.findMany).mockResolvedValue([+ websiteFindManyMock.mockResolvedValue([{ id: 'owned', userId: 'user-1', teamId: null },{ id: 'team', userId: null, teamId: 'team-1' },{ id: 'foreign', userId: 'other', teamId: null },] as any);- vi.mocked(prisma.client.teamUser.findMany).mockResolvedValue([{ teamId: 'team-1' }] as any);+ teamUserFindManyMock.mockResolvedValue([{ teamId: 'team-1' }] as any);await expect(canViewBatchWebsites({ user: normalUser, shareToken: { websiteId: 'shared' } as any }, [@@ -157,10 +161,10 @@ describe('canViewBatchWebsites', () => {});test('excludes team websites when the user is not a team member', async () => {- vi.mocked(prisma.client.website.findMany).mockResolvedValue([+ websiteFindManyMock.mockResolvedValue([{ id: 'team', userId: null, teamId: 'team-1' },] as any);- vi.mocked(prisma.client.teamUser.findMany).mockResolvedValue([] as any);+ teamUserFindManyMock.mockResolvedValue([] as any);await expect(canViewBatchWebsites({ user: normalUser }, ['team'])).resolves.toEqual([]);});diff --git a/src/queries/sql/getChannelExpandedMetrics.test.ts b/src/queries/sql/getChannelExpandedMetrics.test.tsindex 452071e86..8edf8cc17 100644--- a/src/queries/sql/getChannelExpandedMetrics.test.ts+++ b/src/queries/sql/getChannelExpandedMetrics.test.ts@@ -83,7 +83,9 @@ describe('getChannelExpandedMetrics postgres branch', () => {expect(query).toContain('sum(visit_stats.c) as "pageviews"');expect(query).toContain('count(distinct visit_stats.session_id) as "visitors"');expect(query).toContain('count(distinct visit_stats.visit_id) as "visits"');- expect(query).toContain('sum(case when visit_stats.c = 1 then 1 else 0 end) as "bounces"');+ expect(query).toContain('sum(case when visit_stats.c = 1 and coalesce(visit_events.has_custom_event, 0) = 0 then 1 else 0 end) as "bounces"');+ expect(query).toContain('visit_events as');+ expect(query).toContain('left join visit_events');// getTimestampDiffSQL output is injected into the totaltime columnexpect(query).toContain('sum(ts_diff(min_time, max_time)) as "totaltime"');expect(getTimestampDiffSQL).toHaveBeenCalledWith(@@ -151,7 +153,8 @@ describe('getChannelExpandedMetrics clickhouse branch', () => {expect(query).toContain('uniq(t.session_id) as "visitors"');expect(query).toContain('uniq(t.visit_id) as "visits"');- expect(query).toContain('sum(if(t.c = 1, 1, 0)) as "bounces"');+ expect(query).toContain('sumIf(1, t.c = 1 and ifNull(e.has_custom_event, 0) = 0) as "bounces"');+ expect(query).toContain('left join (');expect(query).toContain('sum(max_time-min_time) as "totaltime"');expect(query).toContain('where website_id = {websiteId:UUID}');expect(query).toContain(diff --git a/src/queries/sql/getChannelExpandedMetrics.ts b/src/queries/sql/getChannelExpandedMetrics.tsindex 6d9eee301..fa3c1dc23 100644--- a/src/queries/sql/getChannelExpandedMetrics.ts+++ b/src/queries/sql/getChannelExpandedMetrics.ts@@ -1,6 +1,5 @@import clickhouse from '@/lib/clickhouse';import {- BOUNCE_THRESHOLD,EMAIL_DOMAINS,EVENT_TYPE,LLM_DOMAINS,@@ -49,6 +48,29 @@ async function relationalQuery(...filters,websiteId,});+ const needsBounceEvents = filters.excludeBounce !== true;+ const bounceQuery = needsBounceEvents+ ? `sum(case when visit_stats.c = 1 and coalesce(visit_events.has_custom_event, 0) = 0 then 1 else 0 end) as "bounces",`+ : '0 as "bounces",';+ const visitEventsQuery = needsBounceEvents+ ? `,++ visit_events as (+ select+ session_id,+ visit_id,+ 1 as has_custom_event+ from website_event+ where website_id = {{websiteId::uuid}}+ and created_at between {{startDate}} and {{endDate}}+ and event_type = ${EVENT_TYPE.customEvent}+ group by session_id, visit_id)`+ : '';+ const visitEventsJoin = needsBounceEvents+ ? `left join visit_events+ on visit_events.session_id = visit_stats.session_id+ and visit_events.visit_id = visit_stats.visit_id`+ : '';return rawQuery(`@@ -122,33 +144,21 @@ async function relationalQuery(min(created_at) as min_time,max(created_at) as max_timefrom prefix- group by session_id, visit_id),-- visit_events as (- select- session_id,- visit_id,- count(*) as events_count- from website_event- where website_id = {{websiteId::uuid}}- and created_at between {{startDate}} and {{endDate}}- and event_type = ${EVENT_TYPE.customEvent}group by session_id, visit_id)+ ${visitEventsQuery}selectvisit_channels.name,sum(visit_stats.c) as "pageviews",count(distinct visit_stats.session_id) as "visitors",count(distinct visit_stats.visit_id) as "visits",- sum(case when visit_stats.c = 1 and coalesce(visit_events.events_count, 0) < ${BOUNCE_THRESHOLD} then 1 else 0 end) as "bounces",+ ${bounceQuery}sum(${getTimestampDiffSQL('visit_stats.min_time', 'visit_stats.max_time')}) as "totaltime"from visit_statsjoin visit_channelson visit_channels.session_id = visit_stats.session_idand visit_channels.visit_id = visit_stats.visit_id- left join visit_events- on visit_events.session_id = visit_stats.session_id- and visit_events.visit_id = visit_stats.visit_id+ ${visitEventsJoin}group by visit_channels.nameorder by visitors desc, visits desc`,@@ -166,6 +176,20 @@ async function clickhouseQuery(...filters,websiteId,});+ const needsBounceEvents = filters.excludeBounce !== true;+ const bounceQuery = needsBounceEvents+ ? `sumIf(1, t.c = 1 and ifNull(e.has_custom_event, 0) = 0) as "bounces",`+ : '0 as "bounces",';+ const visitEventsJoin = needsBounceEvents+ ? `left join (+ select session_id, visit_id, toUInt8(1) as has_custom_event+ from website_event+ where website_id = {websiteId:UUID}+ and created_at between {startDate:DateTime64} and {endDate:DateTime64}+ and event_type = ${EVENT_TYPE.customEvent}+ group by session_id, visit_id+ ) as e using (session_id, visit_id)`+ : '';return rawQuery(`@@ -174,7 +198,7 @@ async function clickhouseQuery(sum(t.c) as "pageviews",uniq(t.session_id) as "visitors",uniq(t.visit_id) as "visits",- sumIf(1, t.c = 1 and ifNull(e.events_count, 0) < ${BOUNCE_THRESHOLD}) as "bounces",+ ${bounceQuery}sum(max_time-min_time) as "totaltime"from (select@@ -229,14 +253,7 @@ async function clickhouseQuery()group by session_id, visit_id) as t- left join (- select session_id, visit_id, toUInt32(count()) as events_count- from website_event- where website_id = {websiteId:UUID}- and created_at between {startDate:DateTime64} and {endDate:DateTime64}- and event_type = ${EVENT_TYPE.customEvent}- group by session_id, visit_id- ) as e using (session_id, visit_id)+ ${visitEventsJoin}group by nameorder by visitors desc, visits desc;`,diff --git a/src/queries/sql/getWebsiteStats.test.ts b/src/queries/sql/getWebsiteStats.test.tsnew file mode 100644index 000000000..c604683db--- /dev/null+++ b/src/queries/sql/getWebsiteStats.test.ts@@ -0,0 +1,135 @@+import { afterEach, describe, expect, test, vi } from 'vitest';++const parseFiltersResult = {+ queryParams: { websiteId: 'website-1' },+ filterQuery: 'and website_event.url_path = {{path}}',+ joinSessionQuery: 'join session on session.session_id = website_event.session_id',+ cohortQuery: 'join cohort on cohort.session_id = website_event.session_id',+ excludeBounceQuery: 'join excludeBounce on excludeBounce.visit_id = website_event.visit_id',+};++async function loadModule({+ mode,+}: {+ mode: 'prisma' | 'clickhouse';+}) {+ vi.resetModules();++ const state = { mode };+ const prismaRawQuery = vi.fn().mockResolvedValue([{}]);+ const prismaParseFilters = vi.fn().mockReturnValue(parseFiltersResult);+ const getTimestampDiffSQL = vi.fn().mockReturnValue('ts_diff(t.min_time, t.max_time)');+ const clickhouseRawQuery = vi.fn().mockResolvedValue([{}]);+ const clickhouseParseFilters = vi.fn().mockReturnValue(parseFiltersResult);++ vi.doMock('@/lib/db', () => ({+ CLICKHOUSE: 'clickhouse',+ PRISMA: 'prisma',+ runQuery: vi.fn((queries: Record<string, () => unknown>) => queries[state.mode]()),+ }));++ vi.doMock('@/lib/prisma', () => ({+ default: {+ rawQuery: prismaRawQuery,+ parseFilters: prismaParseFilters,+ getTimestampDiffSQL,+ },+ }));++ vi.doMock('@/lib/clickhouse', () => ({+ default: {+ rawQuery: clickhouseRawQuery,+ parseFilters: clickhouseParseFilters,+ },+ }));++ const mod = await import('./getWebsiteStats');++ return {+ getWebsiteStats: mod.getWebsiteStats,+ prismaRawQuery,+ clickhouseRawQuery,+ };+}++afterEach(() => {+ vi.resetModules();+ vi.clearAllMocks();+});++describe('getWebsiteStats default bounce path', () => {+ test('postgres uses a single grouped pass for the default path', async () => {+ const { getWebsiteStats, prismaRawQuery } = await loadModule({ mode: 'prisma' });++ await getWebsiteStats('website-1', {});++ const [query] = prismaRawQuery.mock.calls[0];++ expect(query).toContain('t.c = 1 and t.has_custom_event = 0');+ expect(query).toContain('max(case when website_event.event_type = 2 then 1 else 0 end) as "has_custom_event"');+ expect(query).not.toContain('left join (');+ });++ test('clickhouse uses the hourly aggregate view without the extra join', async () => {+ const { getWebsiteStats, clickhouseRawQuery } = await loadModule({ mode: 'clickhouse' });++ await getWebsiteStats('website-1', {});++ const [query] = clickhouseRawQuery.mock.calls[0];++ expect(query).toContain('from website_event_stats_hourly "website_event"');+ expect(query).toContain('sumIf(1, t.c = 1 and t.has_custom_event = 0) as "bounces"');+ expect(query).toContain('has_custom_event');+ expect(query).not.toContain('left join (');+ });+});++describe('getWebsiteStats filtered bounce path', () => {+ test('postgres switches to the visit-events join when event filters are present', async () => {+ const { getWebsiteStats, prismaRawQuery } = await loadModule({ mode: 'prisma' });++ await getWebsiteStats('website-1', { path: '/pricing' } as any);++ const [query] = prismaRawQuery.mock.calls[0];++ expect(query).toContain('left join (');+ expect(query).toContain('coalesce(e.has_custom_event, 0) = 0');+ expect(query).toContain('event_type = 2');+ });++ test('clickhouse switches to the raw-event join when event filters are present', async () => {+ const { getWebsiteStats, clickhouseRawQuery } = await loadModule({ mode: 'clickhouse' });++ await getWebsiteStats('website-1', { path: '/pricing' } as any);++ const [query] = clickhouseRawQuery.mock.calls[0];++ expect(query).toContain('from website_event');+ expect(query).toContain('left join (');+ expect(query).toContain('ifNull(e.has_custom_event, 0) = 0');+ });+});++describe('getWebsiteStats excludeBounce path', () => {+ test('clickhouse skips the extra join when excludeBounce is enabled', async () => {+ const { getWebsiteStats, clickhouseRawQuery } = await loadModule({ mode: 'clickhouse' });++ await getWebsiteStats('website-1', { excludeBounce: true });++ const [query] = clickhouseRawQuery.mock.calls[0];++ expect(query).toContain('0 as "bounces"');+ expect(query).not.toContain('left join (');+ });++ test('postgres skips the extra join when excludeBounce is enabled on the filtered path', async () => {+ const { getWebsiteStats, prismaRawQuery } = await loadModule({ mode: 'prisma' });++ await getWebsiteStats('website-1', { excludeBounce: true, path: '/pricing' } as any);++ const [query] = prismaRawQuery.mock.calls[0];++ expect(query).toContain('0 as "bounces"');+ expect(query).not.toContain('left join (');+ });+});diff --git a/src/queries/sql/getWebsiteStats.ts b/src/queries/sql/getWebsiteStats.tsindex 6ca085db4..681c41abe 100644--- a/src/queries/sql/getWebsiteStats.ts+++ b/src/queries/sql/getWebsiteStats.ts@@ -1,5 +1,5 @@import clickhouse from '@/lib/clickhouse';-import { BOUNCE_THRESHOLD, EVENT_COLUMNS, EVENT_TYPE } from '@/lib/constants';+import { EVENT_COLUMNS, EVENT_TYPE } from '@/lib/constants';import { CLICKHOUSE, PRISMA, runQuery } from '@/lib/db';import prisma from '@/lib/prisma';import type { QueryFilters } from '@/lib/types';@@ -35,9 +35,60 @@ async function relationalQuery(});const { excludeBounce } = filters;+ const hasEventFilters =+ EVENT_COLUMNS.some(item => Object.keys(filters).includes(item)) ||+ !!filters.eventPropertyFilters?.length;++ if (!hasEventFilters) {+ return rawQuery(+ `+ select+ cast(coalesce(sum(t.c), 0) as bigint) as "pageviews",+ count(distinct t.session_id) as "visitors",+ count(distinct t.visit_id) as "visits",+ ${excludeBounce ? '0' : 'coalesce(sum(case when t.c = 1 and t.has_custom_event = 0 then 1 else 0 end), 0)'} as "bounces",+ cast(coalesce(sum(${getTimestampDiffSQL('t.min_time', 't.max_time')}), 0) as bigint) as "totaltime"+ from (+ select+ website_event.session_id,+ website_event.visit_id,+ sum(case when website_event.event_type NOT IN (2, 5) then 1 else 0 end) as "c",+ min(case when website_event.event_type NOT IN (2, 5) then website_event.created_at end) as "min_time",+ max(case when website_event.event_type NOT IN (2, 5) then website_event.created_at end) as "max_time",+ max(case when website_event.event_type = ${EVENT_TYPE.customEvent} then 1 else 0 end) as "has_custom_event"+ from website_event+ ${cohortQuery}+ ${excludeBounceQuery}+ ${joinSessionQuery}+ where website_event.website_id = {{websiteId::uuid}}+ and website_event.created_at between {{startDate}} and {{endDate}}+ and website_event.event_type != ${EVENT_TYPE.performance}+ ${filterQuery}+ group by 1, 2+ having sum(case when website_event.event_type NOT IN (2, 5) then 1 else 0 end) > 0+ ) as t+ `,+ queryParams,+ FUNCTION_NAME,+ ).then(result => result?.[0]);+ }+const bounceQuery = excludeBounce? '0'- : `coalesce(sum(case when t.c = 1 and t.events_count < ${BOUNCE_THRESHOLD} then 1 else 0 end), 0)`;+ : 'coalesce(sum(case when t.c = 1 and coalesce(e.has_custom_event, 0) = 0 then 1 else 0 end), 0)';+ const visitEventsJoin = excludeBounce+ ? ''+ : `+ left join (+ select session_id, visit_id, 1 as "has_custom_event"+ from website_event+ where website_id = {{websiteId::uuid}}+ and created_at between {{startDate}} and {{endDate}}+ and event_type = ${EVENT_TYPE.customEvent}+ group by 1, 2+ ) as e+ on e.session_id = t.session_id+ and e.visit_id = t.visit_id`;return rawQuery(`@@ -53,16 +104,7 @@ async function relationalQuery(website_event.visit_id,count(*) as "c",min(website_event.created_at) as "min_time",- max(website_event.created_at) as "max_time",- max((- select count(*)- from website_event we2- where we2.website_id = website_event.website_id- and we2.session_id = website_event.session_id- and we2.visit_id = website_event.visit_id- and we2.created_at between {{startDate}} and {{endDate}}- and we2.event_type = ${EVENT_TYPE.customEvent}- )) as "events_count"+ max(website_event.created_at) as "max_time"from website_event${cohortQuery}${excludeBounceQuery}@@ -72,8 +114,8 @@ async function relationalQuery(and website_event.event_type NOT IN (2, 5)${filterQuery}group by 1, 2-) as t+ ${visitEventsJoin}`,queryParams,FUNCTION_NAME,@@ -92,17 +134,20 @@ async function clickhouseQuery(let sql = '';const { excludeBounce } = filters;+ const hasEventFilters =+ EVENT_COLUMNS.some(item => Object.keys(filters).includes(item)) ||+ !!filters.eventPropertyFilters?.length;const bounceQuery = excludeBounce? '0'- : `sumIf(1, t.c = 1 and ifNull(e.events_count, 0) < ${BOUNCE_THRESHOLD})`;+ : 'sumIf(1, t.c = 1 and t.has_custom_event = 0)';- if (EVENT_COLUMNS.some(item => Object.keys(filters).includes(item))) {+ if (hasEventFilters) {sql = `selectsum(t.c) as "pageviews",uniq(t.session_id) as "visitors",uniq(t.visit_id) as "visits",- ${bounceQuery} as "bounces",+ ${excludeBounce ? '0' : 'sumIf(1, t.c = 1 and ifNull(e.has_custom_event, 0) = 0)'} as "bounces",sum(max_time-min_time) as "totaltime"from (select@@ -120,14 +165,14 @@ async function clickhouseQuery(${filterQuery}group by session_id, visit_id) as t- left join (- select session_id, visit_id, toUInt32(count()) as events_count+ ${excludeBounce ? '' : `left join (+ select session_id, visit_id, toUInt8(1) as has_custom_eventfrom website_eventwhere website_id = {websiteId:UUID}and created_at between {startDate:DateTime64} and {endDate:DateTime64}and event_type = ${EVENT_TYPE.customEvent}group by session_id, visit_id- ) as e using (session_id, visit_id);+ ) as e using (session_id, visit_id)`};`;} else {sql = `@@ -137,28 +182,24 @@ async function clickhouseQuery(uniq(visit_id) as "visits",${bounceQuery} as "bounces",sum(max_time-min_time) as "totaltime"- from (select- session_id,- visit_id,- sum(views) c,- min(min_time) min_time,- max(max_time) max_time+ from (+ select+ session_id,+ visit_id,+ sum(views) c,+ minIf(min_time, event_type NOT IN (2, 5)) min_time,+ maxIf(max_time, event_type NOT IN (2, 5)) max_time,+ max(if(event_type = ${EVENT_TYPE.customEvent} and length(event_name) > 0, 1, 0)) has_custom_eventfrom website_event_stats_hourly "website_event"${cohortQuery}${excludeBounceQuery}- where website_id = {websiteId:UUID}- and created_at between {startDate:DateTime64} and {endDate:DateTime64}- and event_type NOT IN (2, 5)- ${filterQuery}- group by session_id, visit_id- ) as t- left join (- select session_id, visit_id, toUInt32(sum(length(event_name))) as events_count- from website_event_stats_hourlywhere website_id = {websiteId:UUID}and created_at between {startDate:DateTime64} and {endDate:DateTime64}+ and event_type != ${EVENT_TYPE.performance}+ ${filterQuery}group by session_id, visit_id- ) as e using (session_id, visit_id);+ having c > 0+ ) as t`;}diff --git a/src/queries/sql/pageviews/getPageviewExpandedMetrics.ts b/src/queries/sql/pageviews/getPageviewExpandedMetrics.tsindex 153086584..d66b912f0 100644--- a/src/queries/sql/pageviews/getPageviewExpandedMetrics.ts+++ b/src/queries/sql/pageviews/getPageviewExpandedMetrics.ts@@ -1,5 +1,10 @@import clickhouse from '@/lib/clickhouse';-import { BOUNCE_THRESHOLD, EVENT_TYPE, FILTER_COLUMNS, GROUPED_DOMAINS, SESSION_COLUMNS } from '@/lib/constants';+import {+ EVENT_TYPE,+ FILTER_COLUMNS,+ GROUPED_DOMAINS,+ SESSION_COLUMNS,+} from '@/lib/constants';import { CLICKHOUSE, PRISMA, runQuery } from '@/lib/db';import prisma from '@/lib/prisma';import type { QueryFilters } from '@/lib/types';@@ -46,6 +51,7 @@ async function relationalQuery(},{ joinSession: SESSION_COLUMNS.includes(type) },);+ const needsBounceEvents = filters.excludeBounce !== true;const fullPathSearchQuery =type === 'fullPath' && filters.search? `and (case when website_event.url_query != '' then website_event.url_path || '?' || website_event.url_query else website_event.url_path end) ilike {{fullPathSearch}}`@@ -88,6 +94,21 @@ async function relationalQuery(type === 'fullPath'? `case when website_event.url_query != '' then website_event.url_path || '?' || website_event.url_query else website_event.url_path end`: column;+ const bounceQuery = needsBounceEvents+ ? `sum(case when t.c = 1 and coalesce(e.has_custom_event, 0) = 0 then 1 else 0 end) as "bounces",`+ : '0 as "bounces",';+ const visitEventsJoin = needsBounceEvents+ ? `left join (+ select session_id, visit_id, 1 as "has_custom_event"+ from website_event+ where website_id = {{websiteId::uuid}}+ and created_at between {{startDate}} and {{endDate}}+ and event_type = ${EVENT_TYPE.customEvent}+ group by 1, 2+ ) as e+ on e.session_id = t.session_id+ and e.visit_id = t.visit_id`+ : '';return rawQuery(`@@ -96,7 +117,7 @@ async function relationalQuery(sum(t.c) as "pageviews",count(distinct t.session_id) as "visitors",count(distinct t.visit_id) as "visits",- sum(case when t.c = 1 and t.events_count < ${BOUNCE_THRESHOLD} then 1 else 0 end) as "bounces",+ ${bounceQuery}sum(${getTimestampDiffSQL('t.min_time', 't.max_time')}) as "totaltime"from (select@@ -105,16 +126,7 @@ async function relationalQuery(website_event.visit_id,count(*) as "c",min(website_event.created_at) as "min_time",- max(website_event.created_at) as "max_time",- max((- select count(*)- from website_event we2- where we2.website_id = website_event.website_id- and we2.session_id = website_event.session_id- and we2.visit_id = website_event.visit_id- and we2.created_at between {{startDate}} and {{endDate}}- and we2.event_type = ${EVENT_TYPE.customEvent}- )) as "events_count"+ max(website_event.created_at) as "max_time"from website_event${cohortQuery}${excludeBounceQuery}@@ -128,6 +140,7 @@ async function relationalQuery(${filterQuery}group by ${groupByColumn}, website_event.session_id, website_event.visit_id) as t+ ${visitEventsJoin}where name != ''group by nameorder by visitors desc, visits desc@@ -154,6 +167,7 @@ async function clickhouseQuery(...filters,websiteId,});+ const needsBounceEvents = filters.excludeBounce !== true;const fullPathSearchQuery =type === 'fullPath' && filters.search? `and positionCaseInsensitive(if(url_query != '', concat(url_path, '?', url_query), url_path), {fullPathSearch:String}) > 0`@@ -189,6 +203,19 @@ async function clickhouseQuery(} else if (type === 'fullPath') {selectColumn = `if(url_query != '', concat(url_path, '?', url_query), url_path)`;}+ const bounceQuery = needsBounceEvents+ ? `sumIf(1, t.c = 1 and ifNull(e.has_custom_event, 0) = 0) as "bounces",`+ : '0 as "bounces",';+ const visitEventsJoin = needsBounceEvents+ ? `left join (+ select session_id, visit_id, toUInt8(1) as has_custom_event+ from website_event+ where website_id = {websiteId:UUID}+ and created_at between {startDate:DateTime64} and {endDate:DateTime64}+ and event_type = ${EVENT_TYPE.customEvent}+ group by session_id, visit_id+ ) as e using (session_id, visit_id)`+ : '';return rawQuery(`@@ -197,7 +224,7 @@ async function clickhouseQuery(sum(t.c) as "pageviews",uniq(t.session_id) as "visitors",uniq(t.visit_id) as "visits",- sumIf(1, t.c = 1 and ifNull(e.events_count, 0) < ${BOUNCE_THRESHOLD}) as "bounces",+ ${bounceQuery}sum(max_time-min_time) as "totaltime"from (select@@ -220,14 +247,7 @@ async function clickhouseQuery(${filterQuery}group by name, session_id, visit_id) as t- left join (- select session_id, visit_id, toUInt32(count()) as events_count- from website_event- where website_id = {websiteId:UUID}- and created_at between {startDate:DateTime64} and {endDate:DateTime64}- and event_type = ${EVENT_TYPE.customEvent}- group by session_id, visit_id- ) as e using (session_id, visit_id)+ ${visitEventsJoin}group by nameorder by visitors desc, visits desclimit ${limit}diff --git a/src/queries/sql/reports/getBreakdown.ts b/src/queries/sql/reports/getBreakdown.tsindex 13dcccf17..57258013b 100644--- a/src/queries/sql/reports/getBreakdown.ts+++ b/src/queries/sql/reports/getBreakdown.ts@@ -1,5 +1,9 @@import clickhouse from '@/lib/clickhouse';-import { BOUNCE_THRESHOLD, EVENT_TYPE, FILTER_COLUMNS, SESSION_COLUMNS } from '@/lib/constants';+import {+ EVENT_TYPE,+ FILTER_COLUMNS,+ SESSION_COLUMNS,+} from '@/lib/constants';import { CLICKHOUSE, PRISMA, runQuery } from '@/lib/db';import prisma from '@/lib/prisma';import type { QueryFilters } from '@/lib/types';@@ -31,7 +35,7 @@ async function relationalQuery(): Promise<BreakdownData[]> {const { getTimestampDiffSQL, parseFilters, rawQuery } = prisma;const { startDate, endDate, fields } = parameters;- const { filterQuery, joinSessionQuery, cohortQuery, queryParams } = parseFilters(+ const { filterQuery, joinSessionQuery, cohortQuery, excludeBounceQuery, queryParams } = parseFilters({...filters,websiteId,@@ -43,6 +47,22 @@ async function relationalQuery(joinSession: !!fields.find((name: string) => SESSION_COLUMNS.includes(name)),},);+ const needsBounceEvents = filters.excludeBounce !== true;+ const bounceQuery = needsBounceEvents+ ? `sum(case when t.c = 1 and coalesce(e.has_custom_event, 0) = 0 then 1 else 0 end) as "bounces",`+ : '0 as "bounces",';+ const visitEventsJoin = needsBounceEvents+ ? `left join (+ select session_id, visit_id, 1 as "has_custom_event"+ from website_event+ where website_id = {{websiteId::uuid}}+ and created_at between {{startDate}} and {{endDate}}+ and event_type = ${EVENT_TYPE.customEvent}+ group by 1, 2+ ) as e+ on e.session_id = t.session_id+ and e.visit_id = t.visit_id`+ : '';return rawQuery(`@@ -50,7 +70,7 @@ async function relationalQuery(sum(t.c) as "views",count(distinct t.session_id) as "visitors",count(distinct t.visit_id) as "visits",- sum(case when t.c = 1 and t.events_count < ${BOUNCE_THRESHOLD} then 1 else 0 end) as "bounces",+ ${bounceQuery}sum(${getTimestampDiffSQL('t.min_time', 't.max_time')}) as "totaltime",${parseFieldsByName(fields)}from (@@ -60,18 +80,10 @@ async function relationalQuery(website_event.visit_id,count(*) as "c",min(website_event.created_at) as "min_time",- max(website_event.created_at) as "max_time",- max((- select count(*)- from website_event we2- where we2.website_id = website_event.website_id- and we2.session_id = website_event.session_id- and we2.visit_id = website_event.visit_id- and we2.created_at between {{startDate}} and {{endDate}}- and we2.event_type = ${EVENT_TYPE.customEvent}- )) as "events_count"+ max(website_event.created_at) as "max_time"from website_event${cohortQuery}+ ${excludeBounceQuery}${joinSessionQuery}where website_event.website_id = {{websiteId::uuid}}and website_event.created_at between {{startDate}} and {{endDate}}@@ -79,6 +91,7 @@ async function relationalQuery(group by ${parseFieldsByName(fields)},website_event.session_id, website_event.visit_id) as t+ ${visitEventsJoin}group by ${parseFieldsByName(fields)}order by 2 desc, 1 desclimit 500@@ -94,13 +107,27 @@ async function clickhouseQuery(): Promise<BreakdownData[]> {const { parseFilters, rawQuery } = clickhouse;const { startDate, endDate, fields } = parameters;- const { filterQuery, cohortQuery, queryParams } = parseFilters({+ const { filterQuery, cohortQuery, excludeBounceQuery, queryParams } = parseFilters({...filters,websiteId,startDate,endDate,eventType: EVENT_TYPE.pageView,});+ const needsBounceEvents = filters.excludeBounce !== true;+ const bounceQuery = needsBounceEvents+ ? `sumIf(1, t.c = 1 and ifNull(e.has_custom_event, 0) = 0) as "bounces",`+ : '0 as "bounces",';+ const visitEventsJoin = needsBounceEvents+ ? `left join (+ select session_id, visit_id, toUInt8(1) as has_custom_event+ from website_event+ where website_id = {websiteId:UUID}+ and created_at between {startDate:DateTime64} and {endDate:DateTime64}+ and event_type = ${EVENT_TYPE.customEvent}+ group by session_id, visit_id+ ) as e using (session_id, visit_id)`+ : '';return rawQuery(`@@ -108,7 +135,7 @@ async function clickhouseQuery(sum(t.c) as "views",count(distinct t.session_id) as "visitors",count(distinct t.visit_id) as "visits",- sumIf(1, t.c = 1 and ifNull(e.events_count, 0) < ${BOUNCE_THRESHOLD}) as "bounces",+ ${bounceQuery}sum(max_time-min_time) as "totaltime",${parseFieldsByName(fields)}from (@@ -121,20 +148,14 @@ async function clickhouseQuery(max(created_at) max_timefrom website_event${cohortQuery}+ ${excludeBounceQuery}where website_id = {websiteId:UUID}and created_at between {startDate:DateTime64} and {endDate:DateTime64}${filterQuery}group by ${parseFieldsByName(fields)},session_id, visit_id) as t- left join (- select session_id, visit_id, toUInt32(count()) as events_count- from website_event- where website_id = {websiteId:UUID}- and created_at between {startDate:DateTime64} and {endDate:DateTime64}- and event_type = ${EVENT_TYPE.customEvent}- group by session_id, visit_id- ) as e using (session_id, visit_id)+ ${visitEventsJoin}group by ${parseFieldsByName(fields)}order by 2 desc, 1 desclimit 500diff --git a/src/queries/sql/sessions/getSessionExpandedMetrics.ts b/src/queries/sql/sessions/getSessionExpandedMetrics.tsindex c917a40f9..1127469f8 100644--- a/src/queries/sql/sessions/getSessionExpandedMetrics.ts+++ b/src/queries/sql/sessions/getSessionExpandedMetrics.ts@@ -1,5 +1,9 @@import clickhouse from '@/lib/clickhouse';-import { BOUNCE_THRESHOLD, EVENT_TYPE, FILTER_COLUMNS, SESSION_COLUMNS } from '@/lib/constants';+import {+ EVENT_TYPE,+ FILTER_COLUMNS,+ SESSION_COLUMNS,+} from '@/lib/constants';import { CLICKHOUSE, PRISMA, runQuery } from '@/lib/db';import prisma from '@/lib/prisma';import type { QueryFilters } from '@/lib/types';@@ -48,11 +52,27 @@ async function relationalQuery(joinSession: SESSION_COLUMNS.includes(type),},);+ const needsBounceEvents = filters.excludeBounce !== true;const includeCountry = column === 'city' || column === 'region';if (type === 'language') {column = `lower(left(${type}, 2))`;}+ const bounceQuery = needsBounceEvents+ ? `sum(case when t.c = 1 and coalesce(e.has_custom_event, 0) = 0 then 1 else 0 end) as "bounces",`+ : '0 as "bounces",';+ const visitEventsJoin = needsBounceEvents+ ? `left join (+ select session_id, visit_id, 1 as "has_custom_event"+ from website_event+ where website_id = {{websiteId::uuid}}+ and created_at between {{startDate}} and {{endDate}}+ and event_type = ${EVENT_TYPE.customEvent}+ group by 1, 2+ ) as e+ on e.session_id = t.session_id+ and e.visit_id = t.visit_id`+ : '';return rawQuery(`@@ -62,7 +82,7 @@ async function relationalQuery(sum(t.c) as "pageviews",count(distinct t.session_id) as "visitors",count(distinct t.visit_id) as "visits",- sum(case when t.c = 1 and t.events_count < ${BOUNCE_THRESHOLD} then 1 else 0 end) as "bounces",+ ${bounceQuery}sum(${getTimestampDiffSQL('t.min_time', 't.max_time')}) as "totaltime"from (select@@ -72,16 +92,7 @@ async function relationalQuery(website_event.visit_id,count(*) as "c",min(website_event.created_at) as "min_time",- max(website_event.created_at) as "max_time",- max((- select count(*)- from website_event we2- where we2.website_id = website_event.website_id- and we2.session_id = website_event.session_id- and we2.visit_id = website_event.visit_id- and we2.created_at between {{startDate}} and {{endDate}}- and we2.event_type = ${EVENT_TYPE.customEvent}- )) as "events_count"+ max(website_event.created_at) as "max_time"from website_event${cohortQuery}${excludeBounceQuery}@@ -93,6 +104,7 @@ async function relationalQuery(group by name, website_event.session_id, website_event.visit_id${includeCountry ? ', country' : ''}) as t+ ${visitEventsJoin}where name != ''group by name${includeCountry ? ', country' : ''}@@ -117,11 +129,25 @@ async function clickhouseQuery(...filters,websiteId,});+ const needsBounceEvents = filters.excludeBounce !== true;const includeCountry = column === 'city' || column === 'region';if (type === 'language') {column = `lower(left(${type}, 2))`;}+ const bounceQuery = needsBounceEvents+ ? `sumIf(1, t.c = 1 and ifNull(e.has_custom_event, 0) = 0) as "bounces",`+ : '0 as "bounces",';+ const visitEventsJoin = needsBounceEvents+ ? `left join (+ select session_id, visit_id, toUInt8(1) as has_custom_event+ from website_event+ where website_id = {websiteId:UUID}+ and created_at between {startDate:DateTime64} and {endDate:DateTime64}+ and event_type = ${EVENT_TYPE.customEvent}+ group by session_id, visit_id+ ) as e using (session_id, visit_id)`+ : '';return rawQuery(`@@ -131,7 +157,7 @@ async function clickhouseQuery(sum(t.c) as "pageviews",uniq(t.session_id) as "visitors",uniq(t.visit_id) as "visits",- sumIf(1, t.c = 1 and ifNull(e.events_count, 0) < ${BOUNCE_THRESHOLD}) as "bounces",+ ${bounceQuery}sum(max_time-min_time) as "totaltime"from (select@@ -153,14 +179,7 @@ async function clickhouseQuery(group by name, session_id, visit_id${includeCountry ? ', country' : ''}) as t- left join (- select session_id, visit_id, toUInt32(count()) as events_count- from website_event- where website_id = {websiteId:UUID}- and created_at between {startDate:DateTime64} and {endDate:DateTime64}- and event_type = ${EVENT_TYPE.customEvent}- group by session_id, visit_id- ) as e using (session_id, visit_id)+ ${visitEventsJoin}group by name${includeCountry ? ', country' : ''}order by visitors desc, visits desc