Skip to content

armgit / umami

Public Imported from GitHub
0

Umami is a privacy-first web and product analytics platform. Traffic, campaigns, behavior, conversions, and revenue in one place. No cookies, no surveillance, self-hosted or in the cloud.

Fix excludeBounce query, simplify bounce definition for SPA websites, query routing for performance, add tests

Francis Caob3aec54bb3d5
diff --git a/docker-compose.yml b/docker-compose.yml
index bc8d8da08..348c294ca 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -7,7 +7,6 @@ services:
environment:
DATABASE_URL: postgresql://umami:umami@db:5432/umami
APP_SECRET: replace-me-with-a-random-string
- # BOUNCE_THRESHOLD: 2 # OPTIONAL: set the minimum amount custom events to trigger a bounce
depends_on:
db:
condition: service_healthy
diff --git a/src/lib/clickhouse.ts b/src/lib/clickhouse.ts
index bfce9ba62..7d3331dd7 100644
--- a/src/lib/clickhouse.ts
+++ b/src/lib/clickhouse.ts
@@ -177,13 +177,17 @@ function getExcludeBounceQuery(filters: Record<string, any>) {
}
return `join
- (select distinct session_id as exclude_session_id, visit_id as exclude_visit_id
+ (select session_id as exclude_session_id, visit_id as exclude_visit_id
from website_event_stats_hourly
where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime64} and {endDate:DateTime64}
- and event_type = 1
+ and event_type != 5
group by session_id, visit_id
having sum(views) > 1
+ or (
+ sum(views) = 1
+ and sumIf(length(event_name), event_type = 2) > 0
+ )
) excludeBounce
on excludeBounce.exclude_session_id = website_event.session_id
and excludeBounce.exclude_visit_id = website_event.visit_id
diff --git a/src/lib/constants.ts b/src/lib/constants.ts
index 5420bc6cc..89e250e73 100644
--- a/src/lib/constants.ts
+++ b/src/lib/constants.ts
@@ -1,5 +1,4 @@
export const CURRENT_VERSION = process.env.currentVersion;
-export const BOUNCE_THRESHOLD = Math.max(1, Number.parseInt(process.env.BOUNCE_THRESHOLD, 10) || 1);
export const AUTH_TOKEN = 'umami.auth';
export const LOCALE_CONFIG = 'umami.locale';
export const TIMEZONE_CONFIG = 'umami.timezone';
diff --git a/src/lib/prisma.ts b/src/lib/prisma.ts
index 18b53a1cb..5c76aef4a 100644
--- a/src/lib/prisma.ts
+++ b/src/lib/prisma.ts
@@ -2,13 +2,7 @@ import { PrismaPg } from '@prisma/adapter-pg';
import { readReplicas } from '@prisma/extension-read-replicas';
import debug from 'debug';
import { PrismaClient } from '@/generated/prisma/client';
-import {
- DATA_TYPE,
- DEFAULT_PAGE_SIZE,
- FILTER_COLUMNS,
- OPERATORS,
- SESSION_COLUMNS,
-} from './constants';
+import { DATA_TYPE, DEFAULT_PAGE_SIZE, FILTER_COLUMNS, OPERATORS, SESSION_COLUMNS } from './constants';
import { filtersObjectToArray } from './params';
import type { Operator, PropertyFilter, QueryFilters, QueryOptions } from './types';
@@ -249,13 +243,17 @@ function getExcludeBounceQuery(filters: Record<string, any>) {
}
return `join
- (select distinct session_id, visit_id
+ (select session_id, visit_id
from website_event
where website_id = {{websiteId}}
and created_at between {{startDate}} and {{endDate}}
- and event_type = 1
+ and event_type != 5
group by session_id, visit_id
- having count(*) > 1
+ having sum(case when event_type NOT IN (2, 5) then 1 else 0 end) > 1
+ or (
+ sum(case when event_type NOT IN (2, 5) then 1 else 0 end) = 1
+ and sum(case when event_type = 2 then 1 else 0 end) > 0
+ )
) excludeBounce
on excludeBounce.session_id = website_event.session_id
and excludeBounce.visit_id = website_event.visit_id
diff --git a/src/permissions/website.test.ts b/src/permissions/website.test.ts
index b9c00a103..13679f3b6 100644
--- a/src/permissions/website.test.ts
+++ b/src/permissions/website.test.ts
@@ -1,6 +1,5 @@
import { beforeEach, describe, expect, test, vi } from 'vitest';
import { getEntity } from '@/lib/entity';
-import prisma from '@/lib/prisma';
import { getTeamUser, getWebsite } from '@/queries/prisma';
import {
canCreateWebsite,
@@ -13,6 +12,11 @@ import {
canViewWebsite,
} from './website';
+const { websiteFindManyMock, teamUserFindManyMock } = vi.hoisted(() => ({
+ websiteFindManyMock: vi.fn(),
+ teamUserFindManyMock: vi.fn(),
+}));
+
vi.mock('@/lib/entity', () => ({
getEntity: vi.fn(),
}));
@@ -26,10 +30,10 @@ vi.mock('@/lib/prisma', () => ({
default: {
client: {
website: {
- findMany: vi.fn(),
+ findMany: websiteFindManyMock,
},
teamUser: {
- findMany: vi.fn(),
+ findMany: teamUserFindManyMock,
},
},
},
@@ -54,8 +58,8 @@ beforeEach(() => {
vi.mocked(getEntity).mockReset();
vi.mocked(getWebsite).mockReset();
vi.mocked(getTeamUser).mockReset();
- vi.mocked(prisma.client.website.findMany).mockReset();
- vi.mocked(prisma.client.teamUser.findMany).mockReset();
+ websiteFindManyMock.mockReset();
+ teamUserFindManyMock.mockReset();
});
describe('canViewWebsite', () => {
@@ -139,12 +143,12 @@ describe('canViewBatchWebsites', () => {
});
test('returns owned, team, and share allowed ids for a user', async () => {
- vi.mocked(prisma.client.website.findMany).mockResolvedValue([
+ websiteFindManyMock.mockResolvedValue([
{ id: 'owned', userId: 'user-1', teamId: null },
{ id: 'team', userId: null, teamId: 'team-1' },
{ id: 'foreign', userId: 'other', teamId: null },
] as any);
- vi.mocked(prisma.client.teamUser.findMany).mockResolvedValue([{ teamId: 'team-1' }] as any);
+ teamUserFindManyMock.mockResolvedValue([{ teamId: 'team-1' }] as any);
await expect(
canViewBatchWebsites({ user: normalUser, shareToken: { websiteId: 'shared' } as any }, [
@@ -157,10 +161,10 @@ describe('canViewBatchWebsites', () => {
});
test('excludes team websites when the user is not a team member', async () => {
- vi.mocked(prisma.client.website.findMany).mockResolvedValue([
+ websiteFindManyMock.mockResolvedValue([
{ id: 'team', userId: null, teamId: 'team-1' },
] as any);
- vi.mocked(prisma.client.teamUser.findMany).mockResolvedValue([] as any);
+ teamUserFindManyMock.mockResolvedValue([] as any);
await expect(canViewBatchWebsites({ user: normalUser }, ['team'])).resolves.toEqual([]);
});
diff --git a/src/queries/sql/getChannelExpandedMetrics.test.ts b/src/queries/sql/getChannelExpandedMetrics.test.ts
index 452071e86..8edf8cc17 100644
--- a/src/queries/sql/getChannelExpandedMetrics.test.ts
+++ b/src/queries/sql/getChannelExpandedMetrics.test.ts
@@ -83,7 +83,9 @@ describe('getChannelExpandedMetrics postgres branch', () => {
expect(query).toContain('sum(visit_stats.c) as "pageviews"');
expect(query).toContain('count(distinct visit_stats.session_id) as "visitors"');
expect(query).toContain('count(distinct visit_stats.visit_id) as "visits"');
- expect(query).toContain('sum(case when visit_stats.c = 1 then 1 else 0 end) as "bounces"');
+ expect(query).toContain('sum(case when visit_stats.c = 1 and coalesce(visit_events.has_custom_event, 0) = 0 then 1 else 0 end) as "bounces"');
+ expect(query).toContain('visit_events as');
+ expect(query).toContain('left join visit_events');
// getTimestampDiffSQL output is injected into the totaltime column
expect(query).toContain('sum(ts_diff(min_time, max_time)) as "totaltime"');
expect(getTimestampDiffSQL).toHaveBeenCalledWith(
@@ -151,7 +153,8 @@ describe('getChannelExpandedMetrics clickhouse branch', () => {
expect(query).toContain('uniq(t.session_id) as "visitors"');
expect(query).toContain('uniq(t.visit_id) as "visits"');
- expect(query).toContain('sum(if(t.c = 1, 1, 0)) as "bounces"');
+ expect(query).toContain('sumIf(1, t.c = 1 and ifNull(e.has_custom_event, 0) = 0) as "bounces"');
+ expect(query).toContain('left join (');
expect(query).toContain('sum(max_time-min_time) as "totaltime"');
expect(query).toContain('where website_id = {websiteId:UUID}');
expect(query).toContain(
diff --git a/src/queries/sql/getChannelExpandedMetrics.ts b/src/queries/sql/getChannelExpandedMetrics.ts
index 6d9eee301..fa3c1dc23 100644
--- a/src/queries/sql/getChannelExpandedMetrics.ts
+++ b/src/queries/sql/getChannelExpandedMetrics.ts
@@ -1,6 +1,5 @@
import clickhouse from '@/lib/clickhouse';
import {
- BOUNCE_THRESHOLD,
EMAIL_DOMAINS,
EVENT_TYPE,
LLM_DOMAINS,
@@ -49,6 +48,29 @@ async function relationalQuery(
...filters,
websiteId,
});
+ const needsBounceEvents = filters.excludeBounce !== true;
+ const bounceQuery = needsBounceEvents
+ ? `sum(case when visit_stats.c = 1 and coalesce(visit_events.has_custom_event, 0) = 0 then 1 else 0 end) as "bounces",`
+ : '0 as "bounces",';
+ const visitEventsQuery = needsBounceEvents
+ ? `,
+
+ visit_events as (
+ select
+ session_id,
+ visit_id,
+ 1 as has_custom_event
+ from website_event
+ where website_id = {{websiteId::uuid}}
+ and created_at between {{startDate}} and {{endDate}}
+ and event_type = ${EVENT_TYPE.customEvent}
+ group by session_id, visit_id)`
+ : '';
+ const visitEventsJoin = needsBounceEvents
+ ? `left join visit_events
+ on visit_events.session_id = visit_stats.session_id
+ and visit_events.visit_id = visit_stats.visit_id`
+ : '';
return rawQuery(
`
@@ -122,33 +144,21 @@ async function relationalQuery(
min(created_at) as min_time,
max(created_at) as max_time
from prefix
- group by session_id, visit_id),
-
- visit_events as (
- select
- session_id,
- visit_id,
- count(*) as events_count
- from website_event
- where website_id = {{websiteId::uuid}}
- and created_at between {{startDate}} and {{endDate}}
- and event_type = ${EVENT_TYPE.customEvent}
group by session_id, visit_id)
+ ${visitEventsQuery}
select
visit_channels.name,
sum(visit_stats.c) as "pageviews",
count(distinct visit_stats.session_id) as "visitors",
count(distinct visit_stats.visit_id) as "visits",
- sum(case when visit_stats.c = 1 and coalesce(visit_events.events_count, 0) < ${BOUNCE_THRESHOLD} then 1 else 0 end) as "bounces",
+ ${bounceQuery}
sum(${getTimestampDiffSQL('visit_stats.min_time', 'visit_stats.max_time')}) as "totaltime"
from visit_stats
join visit_channels
on visit_channels.session_id = visit_stats.session_id
and visit_channels.visit_id = visit_stats.visit_id
- left join visit_events
- on visit_events.session_id = visit_stats.session_id
- and visit_events.visit_id = visit_stats.visit_id
+ ${visitEventsJoin}
group by visit_channels.name
order by visitors desc, visits desc
`,
@@ -166,6 +176,20 @@ async function clickhouseQuery(
...filters,
websiteId,
});
+ const needsBounceEvents = filters.excludeBounce !== true;
+ const bounceQuery = needsBounceEvents
+ ? `sumIf(1, t.c = 1 and ifNull(e.has_custom_event, 0) = 0) as "bounces",`
+ : '0 as "bounces",';
+ const visitEventsJoin = needsBounceEvents
+ ? `left join (
+ select session_id, visit_id, toUInt8(1) as has_custom_event
+ from website_event
+ where website_id = {websiteId:UUID}
+ and created_at between {startDate:DateTime64} and {endDate:DateTime64}
+ and event_type = ${EVENT_TYPE.customEvent}
+ group by session_id, visit_id
+ ) as e using (session_id, visit_id)`
+ : '';
return rawQuery(
`
@@ -174,7 +198,7 @@ async function clickhouseQuery(
sum(t.c) as "pageviews",
uniq(t.session_id) as "visitors",
uniq(t.visit_id) as "visits",
- sumIf(1, t.c = 1 and ifNull(e.events_count, 0) < ${BOUNCE_THRESHOLD}) as "bounces",
+ ${bounceQuery}
sum(max_time-min_time) as "totaltime"
from (
select
@@ -229,14 +253,7 @@ async function clickhouseQuery(
)
group by session_id, visit_id
) as t
- left join (
- select session_id, visit_id, toUInt32(count()) as events_count
- from website_event
- where website_id = {websiteId:UUID}
- and created_at between {startDate:DateTime64} and {endDate:DateTime64}
- and event_type = ${EVENT_TYPE.customEvent}
- group by session_id, visit_id
- ) as e using (session_id, visit_id)
+ ${visitEventsJoin}
group by name
order by visitors desc, visits desc;
`,
diff --git a/src/queries/sql/getWebsiteStats.test.ts b/src/queries/sql/getWebsiteStats.test.ts
new file mode 100644
index 000000000..c604683db
--- /dev/null
+++ b/src/queries/sql/getWebsiteStats.test.ts
@@ -0,0 +1,135 @@
+import { afterEach, describe, expect, test, vi } from 'vitest';
+
+const parseFiltersResult = {
+ queryParams: { websiteId: 'website-1' },
+ filterQuery: 'and website_event.url_path = {{path}}',
+ joinSessionQuery: 'join session on session.session_id = website_event.session_id',
+ cohortQuery: 'join cohort on cohort.session_id = website_event.session_id',
+ excludeBounceQuery: 'join excludeBounce on excludeBounce.visit_id = website_event.visit_id',
+};
+
+async function loadModule({
+ mode,
+}: {
+ mode: 'prisma' | 'clickhouse';
+}) {
+ vi.resetModules();
+
+ const state = { mode };
+ const prismaRawQuery = vi.fn().mockResolvedValue([{}]);
+ const prismaParseFilters = vi.fn().mockReturnValue(parseFiltersResult);
+ const getTimestampDiffSQL = vi.fn().mockReturnValue('ts_diff(t.min_time, t.max_time)');
+ const clickhouseRawQuery = vi.fn().mockResolvedValue([{}]);
+ const clickhouseParseFilters = vi.fn().mockReturnValue(parseFiltersResult);
+
+ vi.doMock('@/lib/db', () => ({
+ CLICKHOUSE: 'clickhouse',
+ PRISMA: 'prisma',
+ runQuery: vi.fn((queries: Record<string, () => unknown>) => queries[state.mode]()),
+ }));
+
+ vi.doMock('@/lib/prisma', () => ({
+ default: {
+ rawQuery: prismaRawQuery,
+ parseFilters: prismaParseFilters,
+ getTimestampDiffSQL,
+ },
+ }));
+
+ vi.doMock('@/lib/clickhouse', () => ({
+ default: {
+ rawQuery: clickhouseRawQuery,
+ parseFilters: clickhouseParseFilters,
+ },
+ }));
+
+ const mod = await import('./getWebsiteStats');
+
+ return {
+ getWebsiteStats: mod.getWebsiteStats,
+ prismaRawQuery,
+ clickhouseRawQuery,
+ };
+}
+
+afterEach(() => {
+ vi.resetModules();
+ vi.clearAllMocks();
+});
+
+describe('getWebsiteStats default bounce path', () => {
+ test('postgres uses a single grouped pass for the default path', async () => {
+ const { getWebsiteStats, prismaRawQuery } = await loadModule({ mode: 'prisma' });
+
+ await getWebsiteStats('website-1', {});
+
+ const [query] = prismaRawQuery.mock.calls[0];
+
+ expect(query).toContain('t.c = 1 and t.has_custom_event = 0');
+ expect(query).toContain('max(case when website_event.event_type = 2 then 1 else 0 end) as "has_custom_event"');
+ expect(query).not.toContain('left join (');
+ });
+
+ test('clickhouse uses the hourly aggregate view without the extra join', async () => {
+ const { getWebsiteStats, clickhouseRawQuery } = await loadModule({ mode: 'clickhouse' });
+
+ await getWebsiteStats('website-1', {});
+
+ const [query] = clickhouseRawQuery.mock.calls[0];
+
+ expect(query).toContain('from website_event_stats_hourly "website_event"');
+ expect(query).toContain('sumIf(1, t.c = 1 and t.has_custom_event = 0) as "bounces"');
+ expect(query).toContain('has_custom_event');
+ expect(query).not.toContain('left join (');
+ });
+});
+
+describe('getWebsiteStats filtered bounce path', () => {
+ test('postgres switches to the visit-events join when event filters are present', async () => {
+ const { getWebsiteStats, prismaRawQuery } = await loadModule({ mode: 'prisma' });
+
+ await getWebsiteStats('website-1', { path: '/pricing' } as any);
+
+ const [query] = prismaRawQuery.mock.calls[0];
+
+ expect(query).toContain('left join (');
+ expect(query).toContain('coalesce(e.has_custom_event, 0) = 0');
+ expect(query).toContain('event_type = 2');
+ });
+
+ test('clickhouse switches to the raw-event join when event filters are present', async () => {
+ const { getWebsiteStats, clickhouseRawQuery } = await loadModule({ mode: 'clickhouse' });
+
+ await getWebsiteStats('website-1', { path: '/pricing' } as any);
+
+ const [query] = clickhouseRawQuery.mock.calls[0];
+
+ expect(query).toContain('from website_event');
+ expect(query).toContain('left join (');
+ expect(query).toContain('ifNull(e.has_custom_event, 0) = 0');
+ });
+});
+
+describe('getWebsiteStats excludeBounce path', () => {
+ test('clickhouse skips the extra join when excludeBounce is enabled', async () => {
+ const { getWebsiteStats, clickhouseRawQuery } = await loadModule({ mode: 'clickhouse' });
+
+ await getWebsiteStats('website-1', { excludeBounce: true });
+
+ const [query] = clickhouseRawQuery.mock.calls[0];
+
+ expect(query).toContain('0 as "bounces"');
+ expect(query).not.toContain('left join (');
+ });
+
+ test('postgres skips the extra join when excludeBounce is enabled on the filtered path', async () => {
+ const { getWebsiteStats, prismaRawQuery } = await loadModule({ mode: 'prisma' });
+
+ await getWebsiteStats('website-1', { excludeBounce: true, path: '/pricing' } as any);
+
+ const [query] = prismaRawQuery.mock.calls[0];
+
+ expect(query).toContain('0 as "bounces"');
+ expect(query).not.toContain('left join (');
+ });
+});
diff --git a/src/queries/sql/getWebsiteStats.ts b/src/queries/sql/getWebsiteStats.ts
index 6ca085db4..681c41abe 100644
--- a/src/queries/sql/getWebsiteStats.ts
+++ b/src/queries/sql/getWebsiteStats.ts
@@ -1,5 +1,5 @@
import clickhouse from '@/lib/clickhouse';
-import { BOUNCE_THRESHOLD, EVENT_COLUMNS, EVENT_TYPE } from '@/lib/constants';
+import { EVENT_COLUMNS, EVENT_TYPE } from '@/lib/constants';
import { CLICKHOUSE, PRISMA, runQuery } from '@/lib/db';
import prisma from '@/lib/prisma';
import type { QueryFilters } from '@/lib/types';
@@ -35,9 +35,60 @@ async function relationalQuery(
});
const { excludeBounce } = filters;
+ const hasEventFilters =
+ EVENT_COLUMNS.some(item => Object.keys(filters).includes(item)) ||
+ !!filters.eventPropertyFilters?.length;
+
+ if (!hasEventFilters) {
+ return rawQuery(
+ `
+ select
+ cast(coalesce(sum(t.c), 0) as bigint) as "pageviews",
+ count(distinct t.session_id) as "visitors",
+ count(distinct t.visit_id) as "visits",
+ ${excludeBounce ? '0' : 'coalesce(sum(case when t.c = 1 and t.has_custom_event = 0 then 1 else 0 end), 0)'} as "bounces",
+ cast(coalesce(sum(${getTimestampDiffSQL('t.min_time', 't.max_time')}), 0) as bigint) as "totaltime"
+ from (
+ select
+ website_event.session_id,
+ website_event.visit_id,
+ sum(case when website_event.event_type NOT IN (2, 5) then 1 else 0 end) as "c",
+ min(case when website_event.event_type NOT IN (2, 5) then website_event.created_at end) as "min_time",
+ max(case when website_event.event_type NOT IN (2, 5) then website_event.created_at end) as "max_time",
+ max(case when website_event.event_type = ${EVENT_TYPE.customEvent} then 1 else 0 end) as "has_custom_event"
+ from website_event
+ ${cohortQuery}
+ ${excludeBounceQuery}
+ ${joinSessionQuery}
+ where website_event.website_id = {{websiteId::uuid}}
+ and website_event.created_at between {{startDate}} and {{endDate}}
+ and website_event.event_type != ${EVENT_TYPE.performance}
+ ${filterQuery}
+ group by 1, 2
+ having sum(case when website_event.event_type NOT IN (2, 5) then 1 else 0 end) > 0
+ ) as t
+ `,
+ queryParams,
+ FUNCTION_NAME,
+ ).then(result => result?.[0]);
+ }
+
const bounceQuery = excludeBounce
? '0'
- : `coalesce(sum(case when t.c = 1 and t.events_count < ${BOUNCE_THRESHOLD} then 1 else 0 end), 0)`;
+ : 'coalesce(sum(case when t.c = 1 and coalesce(e.has_custom_event, 0) = 0 then 1 else 0 end), 0)';
+ const visitEventsJoin = excludeBounce
+ ? ''
+ : `
+ left join (
+ select session_id, visit_id, 1 as "has_custom_event"
+ from website_event
+ where website_id = {{websiteId::uuid}}
+ and created_at between {{startDate}} and {{endDate}}
+ and event_type = ${EVENT_TYPE.customEvent}
+ group by 1, 2
+ ) as e
+ on e.session_id = t.session_id
+ and e.visit_id = t.visit_id`;
return rawQuery(
`
@@ -53,16 +104,7 @@ async function relationalQuery(
website_event.visit_id,
count(*) as "c",
min(website_event.created_at) as "min_time",
- max(website_event.created_at) as "max_time",
- max((
- select count(*)
- from website_event we2
- where we2.website_id = website_event.website_id
- and we2.session_id = website_event.session_id
- and we2.visit_id = website_event.visit_id
- and we2.created_at between {{startDate}} and {{endDate}}
- and we2.event_type = ${EVENT_TYPE.customEvent}
- )) as "events_count"
+ max(website_event.created_at) as "max_time"
from website_event
${cohortQuery}
${excludeBounceQuery}
@@ -72,8 +114,8 @@ async function relationalQuery(
and website_event.event_type NOT IN (2, 5)
${filterQuery}
group by 1, 2
-
) as t
+ ${visitEventsJoin}
`,
queryParams,
FUNCTION_NAME,
@@ -92,17 +134,20 @@ async function clickhouseQuery(
let sql = '';
const { excludeBounce } = filters;
+ const hasEventFilters =
+ EVENT_COLUMNS.some(item => Object.keys(filters).includes(item)) ||
+ !!filters.eventPropertyFilters?.length;
const bounceQuery = excludeBounce
? '0'
- : `sumIf(1, t.c = 1 and ifNull(e.events_count, 0) < ${BOUNCE_THRESHOLD})`;
+ : 'sumIf(1, t.c = 1 and t.has_custom_event = 0)';
- if (EVENT_COLUMNS.some(item => Object.keys(filters).includes(item))) {
+ if (hasEventFilters) {
sql = `
select
sum(t.c) as "pageviews",
uniq(t.session_id) as "visitors",
uniq(t.visit_id) as "visits",
- ${bounceQuery} as "bounces",
+ ${excludeBounce ? '0' : 'sumIf(1, t.c = 1 and ifNull(e.has_custom_event, 0) = 0)'} as "bounces",
sum(max_time-min_time) as "totaltime"
from (
select
@@ -120,14 +165,14 @@ async function clickhouseQuery(
${filterQuery}
group by session_id, visit_id
) as t
- left join (
- select session_id, visit_id, toUInt32(count()) as events_count
+ ${excludeBounce ? '' : `left join (
+ select session_id, visit_id, toUInt8(1) as has_custom_event
from website_event
where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime64} and {endDate:DateTime64}
and event_type = ${EVENT_TYPE.customEvent}
group by session_id, visit_id
- ) as e using (session_id, visit_id);
+ ) as e using (session_id, visit_id)`};
`;
} else {
sql = `
@@ -137,28 +182,24 @@ async function clickhouseQuery(
uniq(visit_id) as "visits",
${bounceQuery} as "bounces",
sum(max_time-min_time) as "totaltime"
- from (select
- session_id,
- visit_id,
- sum(views) c,
- min(min_time) min_time,
- max(max_time) max_time
+ from (
+ select
+ session_id,
+ visit_id,
+ sum(views) c,
+ minIf(min_time, event_type NOT IN (2, 5)) min_time,
+ maxIf(max_time, event_type NOT IN (2, 5)) max_time,
+ max(if(event_type = ${EVENT_TYPE.customEvent} and length(event_name) > 0, 1, 0)) has_custom_event
from website_event_stats_hourly "website_event"
${cohortQuery}
${excludeBounceQuery}
- where website_id = {websiteId:UUID}
- and created_at between {startDate:DateTime64} and {endDate:DateTime64}
- and event_type NOT IN (2, 5)
- ${filterQuery}
- group by session_id, visit_id
- ) as t
- left join (
- select session_id, visit_id, toUInt32(sum(length(event_name))) as events_count
- from website_event_stats_hourly
where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime64} and {endDate:DateTime64}
+ and event_type != ${EVENT_TYPE.performance}
+ ${filterQuery}
group by session_id, visit_id
- ) as e using (session_id, visit_id);
+ having c > 0
+ ) as t
`;
}
diff --git a/src/queries/sql/pageviews/getPageviewExpandedMetrics.ts b/src/queries/sql/pageviews/getPageviewExpandedMetrics.ts
index 153086584..d66b912f0 100644
--- a/src/queries/sql/pageviews/getPageviewExpandedMetrics.ts
+++ b/src/queries/sql/pageviews/getPageviewExpandedMetrics.ts
@@ -1,5 +1,10 @@
import clickhouse from '@/lib/clickhouse';
-import { BOUNCE_THRESHOLD, EVENT_TYPE, FILTER_COLUMNS, GROUPED_DOMAINS, SESSION_COLUMNS } from '@/lib/constants';
+import {
+ EVENT_TYPE,
+ FILTER_COLUMNS,
+ GROUPED_DOMAINS,
+ SESSION_COLUMNS,
+} from '@/lib/constants';
import { CLICKHOUSE, PRISMA, runQuery } from '@/lib/db';
import prisma from '@/lib/prisma';
import type { QueryFilters } from '@/lib/types';
@@ -46,6 +51,7 @@ async function relationalQuery(
},
{ joinSession: SESSION_COLUMNS.includes(type) },
);
+ const needsBounceEvents = filters.excludeBounce !== true;
const fullPathSearchQuery =
type === 'fullPath' && filters.search
? `and (case when website_event.url_query != '' then website_event.url_path || '?' || website_event.url_query else website_event.url_path end) ilike {{fullPathSearch}}`
@@ -88,6 +94,21 @@ async function relationalQuery(
type === 'fullPath'
? `case when website_event.url_query != '' then website_event.url_path || '?' || website_event.url_query else website_event.url_path end`
: column;
+ const bounceQuery = needsBounceEvents
+ ? `sum(case when t.c = 1 and coalesce(e.has_custom_event, 0) = 0 then 1 else 0 end) as "bounces",`
+ : '0 as "bounces",';
+ const visitEventsJoin = needsBounceEvents
+ ? `left join (
+ select session_id, visit_id, 1 as "has_custom_event"
+ from website_event
+ where website_id = {{websiteId::uuid}}
+ and created_at between {{startDate}} and {{endDate}}
+ and event_type = ${EVENT_TYPE.customEvent}
+ group by 1, 2
+ ) as e
+ on e.session_id = t.session_id
+ and e.visit_id = t.visit_id`
+ : '';
return rawQuery(
`
@@ -96,7 +117,7 @@ async function relationalQuery(
sum(t.c) as "pageviews",
count(distinct t.session_id) as "visitors",
count(distinct t.visit_id) as "visits",
- sum(case when t.c = 1 and t.events_count < ${BOUNCE_THRESHOLD} then 1 else 0 end) as "bounces",
+ ${bounceQuery}
sum(${getTimestampDiffSQL('t.min_time', 't.max_time')}) as "totaltime"
from (
select
@@ -105,16 +126,7 @@ async function relationalQuery(
website_event.visit_id,
count(*) as "c",
min(website_event.created_at) as "min_time",
- max(website_event.created_at) as "max_time",
- max((
- select count(*)
- from website_event we2
- where we2.website_id = website_event.website_id
- and we2.session_id = website_event.session_id
- and we2.visit_id = website_event.visit_id
- and we2.created_at between {{startDate}} and {{endDate}}
- and we2.event_type = ${EVENT_TYPE.customEvent}
- )) as "events_count"
+ max(website_event.created_at) as "max_time"
from website_event
${cohortQuery}
${excludeBounceQuery}
@@ -128,6 +140,7 @@ async function relationalQuery(
${filterQuery}
group by ${groupByColumn}, website_event.session_id, website_event.visit_id
) as t
+ ${visitEventsJoin}
where name != ''
group by name
order by visitors desc, visits desc
@@ -154,6 +167,7 @@ async function clickhouseQuery(
...filters,
websiteId,
});
+ const needsBounceEvents = filters.excludeBounce !== true;
const fullPathSearchQuery =
type === 'fullPath' && filters.search
? `and positionCaseInsensitive(if(url_query != '', concat(url_path, '?', url_query), url_path), {fullPathSearch:String}) > 0`
@@ -189,6 +203,19 @@ async function clickhouseQuery(
} else if (type === 'fullPath') {
selectColumn = `if(url_query != '', concat(url_path, '?', url_query), url_path)`;
}
+ const bounceQuery = needsBounceEvents
+ ? `sumIf(1, t.c = 1 and ifNull(e.has_custom_event, 0) = 0) as "bounces",`
+ : '0 as "bounces",';
+ const visitEventsJoin = needsBounceEvents
+ ? `left join (
+ select session_id, visit_id, toUInt8(1) as has_custom_event
+ from website_event
+ where website_id = {websiteId:UUID}
+ and created_at between {startDate:DateTime64} and {endDate:DateTime64}
+ and event_type = ${EVENT_TYPE.customEvent}
+ group by session_id, visit_id
+ ) as e using (session_id, visit_id)`
+ : '';
return rawQuery(
`
@@ -197,7 +224,7 @@ async function clickhouseQuery(
sum(t.c) as "pageviews",
uniq(t.session_id) as "visitors",
uniq(t.visit_id) as "visits",
- sumIf(1, t.c = 1 and ifNull(e.events_count, 0) < ${BOUNCE_THRESHOLD}) as "bounces",
+ ${bounceQuery}
sum(max_time-min_time) as "totaltime"
from (
select
@@ -220,14 +247,7 @@ async function clickhouseQuery(
${filterQuery}
group by name, session_id, visit_id
) as t
- left join (
- select session_id, visit_id, toUInt32(count()) as events_count
- from website_event
- where website_id = {websiteId:UUID}
- and created_at between {startDate:DateTime64} and {endDate:DateTime64}
- and event_type = ${EVENT_TYPE.customEvent}
- group by session_id, visit_id
- ) as e using (session_id, visit_id)
+ ${visitEventsJoin}
group by name
order by visitors desc, visits desc
limit ${limit}
diff --git a/src/queries/sql/reports/getBreakdown.ts b/src/queries/sql/reports/getBreakdown.ts
index 13dcccf17..57258013b 100644
--- a/src/queries/sql/reports/getBreakdown.ts
+++ b/src/queries/sql/reports/getBreakdown.ts
@@ -1,5 +1,9 @@
import clickhouse from '@/lib/clickhouse';
-import { BOUNCE_THRESHOLD, EVENT_TYPE, FILTER_COLUMNS, SESSION_COLUMNS } from '@/lib/constants';
+import {
+ EVENT_TYPE,
+ FILTER_COLUMNS,
+ SESSION_COLUMNS,
+} from '@/lib/constants';
import { CLICKHOUSE, PRISMA, runQuery } from '@/lib/db';
import prisma from '@/lib/prisma';
import type { QueryFilters } from '@/lib/types';
@@ -31,7 +35,7 @@ async function relationalQuery(
): Promise<BreakdownData[]> {
const { getTimestampDiffSQL, parseFilters, rawQuery } = prisma;
const { startDate, endDate, fields } = parameters;
- const { filterQuery, joinSessionQuery, cohortQuery, queryParams } = parseFilters(
+ const { filterQuery, joinSessionQuery, cohortQuery, excludeBounceQuery, queryParams } = parseFilters(
{
...filters,
websiteId,
@@ -43,6 +47,22 @@ async function relationalQuery(
joinSession: !!fields.find((name: string) => SESSION_COLUMNS.includes(name)),
},
);
+ const needsBounceEvents = filters.excludeBounce !== true;
+ const bounceQuery = needsBounceEvents
+ ? `sum(case when t.c = 1 and coalesce(e.has_custom_event, 0) = 0 then 1 else 0 end) as "bounces",`
+ : '0 as "bounces",';
+ const visitEventsJoin = needsBounceEvents
+ ? `left join (
+ select session_id, visit_id, 1 as "has_custom_event"
+ from website_event
+ where website_id = {{websiteId::uuid}}
+ and created_at between {{startDate}} and {{endDate}}
+ and event_type = ${EVENT_TYPE.customEvent}
+ group by 1, 2
+ ) as e
+ on e.session_id = t.session_id
+ and e.visit_id = t.visit_id`
+ : '';
return rawQuery(
`
@@ -50,7 +70,7 @@ async function relationalQuery(
sum(t.c) as "views",
count(distinct t.session_id) as "visitors",
count(distinct t.visit_id) as "visits",
- sum(case when t.c = 1 and t.events_count < ${BOUNCE_THRESHOLD} then 1 else 0 end) as "bounces",
+ ${bounceQuery}
sum(${getTimestampDiffSQL('t.min_time', 't.max_time')}) as "totaltime",
${parseFieldsByName(fields)}
from (
@@ -60,18 +80,10 @@ async function relationalQuery(
website_event.visit_id,
count(*) as "c",
min(website_event.created_at) as "min_time",
- max(website_event.created_at) as "max_time",
- max((
- select count(*)
- from website_event we2
- where we2.website_id = website_event.website_id
- and we2.session_id = website_event.session_id
- and we2.visit_id = website_event.visit_id
- and we2.created_at between {{startDate}} and {{endDate}}
- and we2.event_type = ${EVENT_TYPE.customEvent}
- )) as "events_count"
+ max(website_event.created_at) as "max_time"
from website_event
${cohortQuery}
+ ${excludeBounceQuery}
${joinSessionQuery}
where website_event.website_id = {{websiteId::uuid}}
and website_event.created_at between {{startDate}} and {{endDate}}
@@ -79,6 +91,7 @@ async function relationalQuery(
group by ${parseFieldsByName(fields)},
website_event.session_id, website_event.visit_id
) as t
+ ${visitEventsJoin}
group by ${parseFieldsByName(fields)}
order by 2 desc, 1 desc
limit 500
@@ -94,13 +107,27 @@ async function clickhouseQuery(
): Promise<BreakdownData[]> {
const { parseFilters, rawQuery } = clickhouse;
const { startDate, endDate, fields } = parameters;
- const { filterQuery, cohortQuery, queryParams } = parseFilters({
+ const { filterQuery, cohortQuery, excludeBounceQuery, queryParams } = parseFilters({
...filters,
websiteId,
startDate,
endDate,
eventType: EVENT_TYPE.pageView,
});
+ const needsBounceEvents = filters.excludeBounce !== true;
+ const bounceQuery = needsBounceEvents
+ ? `sumIf(1, t.c = 1 and ifNull(e.has_custom_event, 0) = 0) as "bounces",`
+ : '0 as "bounces",';
+ const visitEventsJoin = needsBounceEvents
+ ? `left join (
+ select session_id, visit_id, toUInt8(1) as has_custom_event
+ from website_event
+ where website_id = {websiteId:UUID}
+ and created_at between {startDate:DateTime64} and {endDate:DateTime64}
+ and event_type = ${EVENT_TYPE.customEvent}
+ group by session_id, visit_id
+ ) as e using (session_id, visit_id)`
+ : '';
return rawQuery(
`
@@ -108,7 +135,7 @@ async function clickhouseQuery(
sum(t.c) as "views",
count(distinct t.session_id) as "visitors",
count(distinct t.visit_id) as "visits",
- sumIf(1, t.c = 1 and ifNull(e.events_count, 0) < ${BOUNCE_THRESHOLD}) as "bounces",
+ ${bounceQuery}
sum(max_time-min_time) as "totaltime",
${parseFieldsByName(fields)}
from (
@@ -121,20 +148,14 @@ async function clickhouseQuery(
max(created_at) max_time
from website_event
${cohortQuery}
+ ${excludeBounceQuery}
where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime64} and {endDate:DateTime64}
${filterQuery}
group by ${parseFieldsByName(fields)},
session_id, visit_id
) as t
- left join (
- select session_id, visit_id, toUInt32(count()) as events_count
- from website_event
- where website_id = {websiteId:UUID}
- and created_at between {startDate:DateTime64} and {endDate:DateTime64}
- and event_type = ${EVENT_TYPE.customEvent}
- group by session_id, visit_id
- ) as e using (session_id, visit_id)
+ ${visitEventsJoin}
group by ${parseFieldsByName(fields)}
order by 2 desc, 1 desc
limit 500
diff --git a/src/queries/sql/sessions/getSessionExpandedMetrics.ts b/src/queries/sql/sessions/getSessionExpandedMetrics.ts
index c917a40f9..1127469f8 100644
--- a/src/queries/sql/sessions/getSessionExpandedMetrics.ts
+++ b/src/queries/sql/sessions/getSessionExpandedMetrics.ts
@@ -1,5 +1,9 @@
import clickhouse from '@/lib/clickhouse';
-import { BOUNCE_THRESHOLD, EVENT_TYPE, FILTER_COLUMNS, SESSION_COLUMNS } from '@/lib/constants';
+import {
+ EVENT_TYPE,
+ FILTER_COLUMNS,
+ SESSION_COLUMNS,
+} from '@/lib/constants';
import { CLICKHOUSE, PRISMA, runQuery } from '@/lib/db';
import prisma from '@/lib/prisma';
import type { QueryFilters } from '@/lib/types';
@@ -48,11 +52,27 @@ async function relationalQuery(
joinSession: SESSION_COLUMNS.includes(type),
},
);
+ const needsBounceEvents = filters.excludeBounce !== true;
const includeCountry = column === 'city' || column === 'region';
if (type === 'language') {
column = `lower(left(${type}, 2))`;
}
+ const bounceQuery = needsBounceEvents
+ ? `sum(case when t.c = 1 and coalesce(e.has_custom_event, 0) = 0 then 1 else 0 end) as "bounces",`
+ : '0 as "bounces",';
+ const visitEventsJoin = needsBounceEvents
+ ? `left join (
+ select session_id, visit_id, 1 as "has_custom_event"
+ from website_event
+ where website_id = {{websiteId::uuid}}
+ and created_at between {{startDate}} and {{endDate}}
+ and event_type = ${EVENT_TYPE.customEvent}
+ group by 1, 2
+ ) as e
+ on e.session_id = t.session_id
+ and e.visit_id = t.visit_id`
+ : '';
return rawQuery(
`
@@ -62,7 +82,7 @@ async function relationalQuery(
sum(t.c) as "pageviews",
count(distinct t.session_id) as "visitors",
count(distinct t.visit_id) as "visits",
- sum(case when t.c = 1 and t.events_count < ${BOUNCE_THRESHOLD} then 1 else 0 end) as "bounces",
+ ${bounceQuery}
sum(${getTimestampDiffSQL('t.min_time', 't.max_time')}) as "totaltime"
from (
select
@@ -72,16 +92,7 @@ async function relationalQuery(
website_event.visit_id,
count(*) as "c",
min(website_event.created_at) as "min_time",
- max(website_event.created_at) as "max_time",
- max((
- select count(*)
- from website_event we2
- where we2.website_id = website_event.website_id
- and we2.session_id = website_event.session_id
- and we2.visit_id = website_event.visit_id
- and we2.created_at between {{startDate}} and {{endDate}}
- and we2.event_type = ${EVENT_TYPE.customEvent}
- )) as "events_count"
+ max(website_event.created_at) as "max_time"
from website_event
${cohortQuery}
${excludeBounceQuery}
@@ -93,6 +104,7 @@ async function relationalQuery(
group by name, website_event.session_id, website_event.visit_id
${includeCountry ? ', country' : ''}
) as t
+ ${visitEventsJoin}
where name != ''
group by name
${includeCountry ? ', country' : ''}
@@ -117,11 +129,25 @@ async function clickhouseQuery(
...filters,
websiteId,
});
+ const needsBounceEvents = filters.excludeBounce !== true;
const includeCountry = column === 'city' || column === 'region';
if (type === 'language') {
column = `lower(left(${type}, 2))`;
}
+ const bounceQuery = needsBounceEvents
+ ? `sumIf(1, t.c = 1 and ifNull(e.has_custom_event, 0) = 0) as "bounces",`
+ : '0 as "bounces",';
+ const visitEventsJoin = needsBounceEvents
+ ? `left join (
+ select session_id, visit_id, toUInt8(1) as has_custom_event
+ from website_event
+ where website_id = {websiteId:UUID}
+ and created_at between {startDate:DateTime64} and {endDate:DateTime64}
+ and event_type = ${EVENT_TYPE.customEvent}
+ group by session_id, visit_id
+ ) as e using (session_id, visit_id)`
+ : '';
return rawQuery(
`
@@ -131,7 +157,7 @@ async function clickhouseQuery(
sum(t.c) as "pageviews",
uniq(t.session_id) as "visitors",
uniq(t.visit_id) as "visits",
- sumIf(1, t.c = 1 and ifNull(e.events_count, 0) < ${BOUNCE_THRESHOLD}) as "bounces",
+ ${bounceQuery}
sum(max_time-min_time) as "totaltime"
from (
select
@@ -153,14 +179,7 @@ async function clickhouseQuery(
group by name, session_id, visit_id
${includeCountry ? ', country' : ''}
) as t
- left join (
- select session_id, visit_id, toUInt32(count()) as events_count
- from website_event
- where website_id = {websiteId:UUID}
- and created_at between {startDate:DateTime64} and {endDate:DateTime64}
- and event_type = ${EVENT_TYPE.customEvent}
- group by session_id, visit_id
- ) as e using (session_id, visit_id)
+ ${visitEventsJoin}
group by name
${includeCountry ? ', country' : ''}
order by visitors desc, visits desc