Skip to content

armgit / umami

Public Imported from GitHub
0

Umami is a privacy-first web and product analytics platform. Traffic, campaigns, behavior, conversions, and revenue in one place. No cookies, no surveillance, self-hosted or in the cloud.

change pagestable to visitors and update clickhouse hostname column to array

Francis Cao2dcb9e21bdf9
diff --git a/db/clickhouse/migrations/08_update_hostname_view.sql b/db/clickhouse/migrations/08_update_hostname_view.sql
new file mode 100644
index 000000000..061fa6285
--- /dev/null
+++ b/db/clickhouse/migrations/08_update_hostname_view.sql
@@ -0,0 +1,253 @@
+-- create new hourly table
+CREATE TABLE umami.website_event_stats_hourly_new
+(
+ website_id UUID,
+ session_id UUID,
+ visit_id UUID,
+ hostname SimpleAggregateFunction(groupArrayArray, Array(String)),
+ browser LowCardinality(String),
+ os LowCardinality(String),
+ device LowCardinality(String),
+ screen LowCardinality(String),
+ language LowCardinality(String),
+ country LowCardinality(String),
+ region LowCardinality(String),
+ city String,
+ entry_url AggregateFunction(argMin, String, DateTime('UTC')),
+ exit_url AggregateFunction(argMax, String, DateTime('UTC')),
+ url_path SimpleAggregateFunction(groupArrayArray, Array(String)),
+ url_query SimpleAggregateFunction(groupArrayArray, Array(String)),
+ utm_source SimpleAggregateFunction(groupArrayArray, Array(String)),
+ utm_medium SimpleAggregateFunction(groupArrayArray, Array(String)),
+ utm_campaign SimpleAggregateFunction(groupArrayArray, Array(String)),
+ utm_content SimpleAggregateFunction(groupArrayArray, Array(String)),
+ utm_term SimpleAggregateFunction(groupArrayArray, Array(String)),
+ referrer_domain SimpleAggregateFunction(groupArrayArray, Array(String)),
+ page_title SimpleAggregateFunction(groupArrayArray, Array(String)),
+ gclid SimpleAggregateFunction(groupArrayArray, Array(String)),
+ fbclid SimpleAggregateFunction(groupArrayArray, Array(String)),
+ msclkid SimpleAggregateFunction(groupArrayArray, Array(String)),
+ ttclid SimpleAggregateFunction(groupArrayArray, Array(String)),
+ li_fat_id SimpleAggregateFunction(groupArrayArray, Array(String)),
+ twclid SimpleAggregateFunction(groupArrayArray, Array(String)),
+ event_type UInt32,
+ event_name SimpleAggregateFunction(groupArrayArray, Array(String)),
+ views SimpleAggregateFunction(sum, UInt64),
+ min_time SimpleAggregateFunction(min, DateTime('UTC')),
+ max_time SimpleAggregateFunction(max, DateTime('UTC')),
+ tag SimpleAggregateFunction(groupArrayArray, Array(String)),
+ distinct_id String,
+ created_at Datetime('UTC')
+)
+ENGINE = AggregatingMergeTree
+ PARTITION BY toYYYYMM(created_at)
+ ORDER BY (
+ website_id,
+ event_type,
+ toStartOfHour(created_at),
+ cityHash64(visit_id),
+ visit_id
+ )
+ SAMPLE BY cityHash64(visit_id);
+
+-- create view
+CREATE MATERIALIZED VIEW umami.website_event_stats_hourly_mv_new
+TO umami.website_event_stats_hourly_new
+AS
+SELECT
+ website_id,
+ session_id,
+ visit_id,
+ hostnames as hostname,
+ browser,
+ os,
+ device,
+ screen,
+ language,
+ country,
+ region,
+ city,
+ entry_url,
+ exit_url,
+ url_paths as url_path,
+ url_query,
+ utm_source,
+ utm_medium,
+ utm_campaign,
+ utm_content,
+ utm_term,
+ referrer_domain,
+ page_title,
+ gclid,
+ fbclid,
+ msclkid,
+ ttclid,
+ li_fat_id,
+ twclid,
+ event_type,
+ event_name,
+ views,
+ min_time,
+ max_time,
+ tag,
+ distinct_id,
+ timestamp as created_at
+FROM (SELECT
+ website_id,
+ session_id,
+ visit_id,
+ arrayFilter(x -> x != '', groupArray(hostname)) hostnames,
+ browser,
+ os,
+ device,
+ screen,
+ language,
+ country,
+ region,
+ city,
+ argMinState(url_path, created_at) entry_url,
+ argMaxState(url_path, created_at) exit_url,
+ arrayFilter(x -> x != '', groupArray(url_path)) as url_paths,
+ arrayFilter(x -> x != '', groupArray(url_query)) url_query,
+ arrayFilter(x -> x != '', groupArray(utm_source)) utm_source,
+ arrayFilter(x -> x != '', groupArray(utm_medium)) utm_medium,
+ arrayFilter(x -> x != '', groupArray(utm_campaign)) utm_campaign,
+ arrayFilter(x -> x != '', groupArray(utm_content)) utm_content,
+ arrayFilter(x -> x != '', groupArray(utm_term)) utm_term,
+ arrayFilter(x -> x != '' and x != hostname, groupArray(referrer_domain)) referrer_domain,
+ arrayFilter(x -> x != '', groupArray(page_title)) page_title,
+ arrayFilter(x -> x != '', groupArray(gclid)) gclid,
+ arrayFilter(x -> x != '', groupArray(fbclid)) fbclid,
+ arrayFilter(x -> x != '', groupArray(msclkid)) msclkid,
+ arrayFilter(x -> x != '', groupArray(ttclid)) ttclid,
+ arrayFilter(x -> x != '', groupArray(li_fat_id)) li_fat_id,
+ arrayFilter(x -> x != '', groupArray(twclid)) twclid,
+ event_type,
+ if(event_type = 2, groupArray(event_name), []) event_name,
+ sumIf(1, event_type = 1) views,
+ min(created_at) min_time,
+ max(created_at) max_time,
+ arrayFilter(x -> x != '', groupArray(tag)) tag,
+ distinct_id,
+ toStartOfHour(created_at) timestamp
+FROM umami.website_event
+GROUP BY website_id,
+ session_id,
+ visit_id,
+ hostname,
+ browser,
+ os,
+ device,
+ screen,
+ language,
+ country,
+ region,
+ city,
+ event_type,
+ distinct_id,
+ timestamp);
+
+-- rename tables
+RENAME TABLE umami.website_event_stats_hourly TO umami.website_event_stats_hourly_old;
+RENAME TABLE umami.website_event_stats_hourly_new TO umami.website_event_stats_hourly;
+
+-- drop views
+DROP TABLE umami.website_event_stats_hourly_mv;
+DROP TABLE umami.website_event_stats_hourly_mv_new;
+
+-- recreate view
+CREATE MATERIALIZED VIEW umami.website_event_stats_hourly_mv
+TO umami.website_event_stats_hourly
+AS
+SELECT
+ website_id,
+ session_id,
+ visit_id,
+ hostnames as hostname,
+ browser,
+ os,
+ device,
+ screen,
+ language,
+ country,
+ region,
+ city,
+ entry_url,
+ exit_url,
+ url_paths as url_path,
+ url_query,
+ utm_source,
+ utm_medium,
+ utm_campaign,
+ utm_content,
+ utm_term,
+ referrer_domain,
+ page_title,
+ gclid,
+ fbclid,
+ msclkid,
+ ttclid,
+ li_fat_id,
+ twclid,
+ event_type,
+ event_name,
+ views,
+ min_time,
+ max_time,
+ tag,
+ distinct_id,
+ timestamp as created_at
+FROM (SELECT
+ website_id,
+ session_id,
+ visit_id,
+ arrayFilter(x -> x != '', groupArray(hostname)) hostnames,
+ browser,
+ os,
+ device,
+ screen,
+ language,
+ country,
+ region,
+ city,
+ argMinState(url_path, created_at) entry_url,
+ argMaxState(url_path, created_at) exit_url,
+ arrayFilter(x -> x != '', groupArray(url_path)) as url_paths,
+ arrayFilter(x -> x != '', groupArray(url_query)) url_query,
+ arrayFilter(x -> x != '', groupArray(utm_source)) utm_source,
+ arrayFilter(x -> x != '', groupArray(utm_medium)) utm_medium,
+ arrayFilter(x -> x != '', groupArray(utm_campaign)) utm_campaign,
+ arrayFilter(x -> x != '', groupArray(utm_content)) utm_content,
+ arrayFilter(x -> x != '', groupArray(utm_term)) utm_term,
+ arrayFilter(x -> x != '' and x != hostname, groupArray(referrer_domain)) referrer_domain,
+ arrayFilter(x -> x != '', groupArray(page_title)) page_title,
+ arrayFilter(x -> x != '', groupArray(gclid)) gclid,
+ arrayFilter(x -> x != '', groupArray(fbclid)) fbclid,
+ arrayFilter(x -> x != '', groupArray(msclkid)) msclkid,
+ arrayFilter(x -> x != '', groupArray(ttclid)) ttclid,
+ arrayFilter(x -> x != '', groupArray(li_fat_id)) li_fat_id,
+ arrayFilter(x -> x != '', groupArray(twclid)) twclid,
+ event_type,
+ if(event_type = 2, groupArray(event_name), []) event_name,
+ sumIf(1, event_type = 1) views,
+ min(created_at) min_time,
+ max(created_at) max_time,
+ arrayFilter(x -> x != '', groupArray(tag)) tag,
+ distinct_id,
+ toStartOfHour(created_at) timestamp
+FROM umami.website_event
+GROUP BY website_id,
+ session_id,
+ visit_id,
+ hostname,
+ browser,
+ os,
+ device,
+ screen,
+ language,
+ country,
+ region,
+ city,
+ event_type,
+ distinct_id,
+ timestamp);
diff --git a/db/clickhouse/schema.sql b/db/clickhouse/schema.sql
index d9831f57a..41b3a1b3e 100644
--- a/db/clickhouse/schema.sql
+++ b/db/clickhouse/schema.sql
@@ -90,7 +90,7 @@ CREATE TABLE umami.website_event_stats_hourly
website_id UUID,
session_id UUID,
visit_id UUID,
- hostname LowCardinality(String),
+ hostname SimpleAggregateFunction(groupArrayArray, Array(String)),
browser LowCardinality(String),
os LowCardinality(String),
device LowCardinality(String),
@@ -143,7 +143,7 @@ SELECT
website_id,
session_id,
visit_id,
- hostname,
+ hostnames as hostname,
browser,
os,
device,
@@ -181,7 +181,7 @@ FROM (SELECT
website_id,
session_id,
visit_id,
- hostname,
+ arrayFilter(x -> x != '', groupArray(hostname)) hostnames,
browser,
os,
device,
@@ -199,7 +199,7 @@ FROM (SELECT
arrayFilter(x -> x != '', groupArray(utm_campaign)) utm_campaign,
arrayFilter(x -> x != '', groupArray(utm_content)) utm_content,
arrayFilter(x -> x != '', groupArray(utm_term)) utm_term,
- arrayFilter(x -> x != '', groupArray(referrer_domain)) referrer_domain,
+ arrayFilter(x -> x != '' and x != hostname, groupArray(referrer_domain)) referrer_domain,
arrayFilter(x -> x != '', groupArray(page_title)) page_title,
arrayFilter(x -> x != '', groupArray(gclid)) gclid,
arrayFilter(x -> x != '', groupArray(fbclid)) fbclid,
diff --git a/src/components/metrics/PagesTable.tsx b/src/components/metrics/PagesTable.tsx
index 8163b3d9c..59cfda43d 100644
--- a/src/components/metrics/PagesTable.tsx
+++ b/src/components/metrics/PagesTable.tsx
@@ -62,7 +62,7 @@ export function PagesTable({ allowFilter, ...props }: PagesTableProps) {
{...props}
title={formatMessage(labels.pages)}
type={view}
- metric={formatMessage(labels.views)}
+ metric={formatMessage(labels.visitors)}
dataFilter={emptyFilter}
renderLabel={renderLink}
>
diff --git a/src/lib/constants.ts b/src/lib/constants.ts
index 39ecbc088..7fd1a85d2 100644
--- a/src/lib/constants.ts
+++ b/src/lib/constants.ts
@@ -33,7 +33,17 @@ export const FILTER_REFERRERS = 'filter-referrers';
export const FILTER_PAGES = 'filter-pages';
export const UNIT_TYPES = ['year', 'month', 'hour', 'day', 'minute'];
-export const EVENT_COLUMNS = ['url', 'entry', 'exit', 'referrer', 'title', 'query', 'event', 'tag'];
+export const EVENT_COLUMNS = [
+ 'url',
+ 'entry',
+ 'exit',
+ 'referrer',
+ 'title',
+ 'query',
+ 'event',
+ 'tag',
+ 'host',
+];
export const SESSION_COLUMNS = [
'browser',
@@ -44,7 +54,6 @@ export const SESSION_COLUMNS = [
'country',
'city',
'region',
- 'host',
];
export const FILTER_GROUPS = {
diff --git a/src/queries/sql/getValues.ts b/src/queries/sql/getValues.ts
index 2d1286f30..5322a5319 100644
--- a/src/queries/sql/getValues.ts
+++ b/src/queries/sql/getValues.ts
@@ -21,6 +21,12 @@ async function relationalQuery(
const { rawQuery, getSearchSQL } = prisma;
const params = {};
let searchQuery = '';
+ let excludeDomain = '';
+
+ if (column === 'referrer_domain') {
+ excludeDomain = `and website_event.referrer_domain != website_event.hostname
+ and website_event.referrer_domain != ''`;
+ }
if (search) {
if (decodeURIComponent(search).includes(',')) {
@@ -49,6 +55,7 @@ async function relationalQuery(
where website_event.website_id = {{websiteId::uuid}}
and website_event.created_at between {{startDate}} and {{endDate}}
${searchQuery}
+ ${excludeDomain}
group by 1
order by 2 desc
limit 10
@@ -73,6 +80,11 @@ async function clickhouseQuery(
const { rawQuery, getSearchSQL } = clickhouse;
const params = {};
let searchQuery = '';
+ let excludeDomain = '';
+
+ if (column === 'referrer_domain') {
+ excludeDomain = `and referrer_domain != hostname and referrer_domain != ''`;
+ }
if (search) {
searchQuery = `and positionCaseInsensitive(${column}, {search:String}) > 0`;
@@ -103,6 +115,7 @@ async function clickhouseQuery(
where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime64} and {endDate:DateTime64}
${searchQuery}
+ ${excludeDomain}
group by 1
order by 2 desc
limit 10
diff --git a/src/queries/sql/pageviews/getPageviewMetrics.ts b/src/queries/sql/pageviews/getPageviewMetrics.ts
index 72586eb79..67e18b75d 100644
--- a/src/queries/sql/pageviews/getPageviewMetrics.ts
+++ b/src/queries/sql/pageviews/getPageviewMetrics.ts
@@ -34,7 +34,7 @@ async function relationalQuery(
...filters,
eventType: column === 'event_name' ? EVENT_TYPE.customEvent : EVENT_TYPE.pageView,
},
- { joinSession: SESSION_COLUMNS.includes(type) || column === 'referrer_domain' },
+ { joinSession: SESSION_COLUMNS.includes(type) },
);
let entryExitQuery = '';
@@ -66,7 +66,7 @@ async function relationalQuery(
return rawQuery(
`
select ${column} x,
- ${column === 'referrer_domain' ? 'count(distinct website_event.session_id)' : 'count(*)'} as y
+ count(distinct website_event.session_id) as y
from website_event
${cohortQuery}
${joinSession}
@@ -126,7 +126,7 @@ async function clickhouseQuery(
sql = `
select ${column} x,
- ${column === 'referrer_domain' ? 'uniq(session_id)' : 'count(*)'} as y
+ uniq(website_event.session_id) as y
from website_event
${cohortQuery}
${entryExitQuery}
@@ -142,28 +142,27 @@ async function clickhouseQuery(
`;
} else {
let groupByQuery = '';
- let columnQuery = `arrayJoin(${column})`;
+ let columnQuery = `session_id s, arrayJoin(${column})`;
if (column === 'referrer_domain') {
- excludeDomain = `and t != hostname and t != ''`;
- columnQuery = `session_id s, arrayJoin(${column})`;
+ excludeDomain = `and t != ''`;
}
if (type === 'entry') {
- columnQuery = `visit_id x, argMinMerge(entry_url)`;
+ columnQuery = `session_id s, argMinMerge(entry_url)`;
}
if (type === 'exit') {
- columnQuery = `visit_id x, argMaxMerge(exit_url)`;
+ columnQuery = `session_id s, argMaxMerge(exit_url)`;
}
if (type === 'entry' || type === 'exit') {
- groupByQuery = 'group by x';
+ groupByQuery = 'group by s';
}
sql = `
select g.t as x,
- ${column === 'referrer_domain' ? 'uniq(s)' : 'count(*)'} as y
+ uniq(s) as y
from (
select ${columnQuery} as t
from website_event_stats_hourly website_event
diff --git a/src/queries/sql/sessions/getWebsiteSession.ts b/src/queries/sql/sessions/getWebsiteSession.ts
index 97850a2c1..8c549ca30 100644
--- a/src/queries/sql/sessions/getWebsiteSession.ts
+++ b/src/queries/sql/sessions/getWebsiteSession.ts
@@ -17,7 +17,6 @@ async function relationalQuery(websiteId: string, sessionId: string) {
select id,
distinct_id as "distinctId",
website_id as "websiteId",
- hostname,
browser,
os,
device,
@@ -37,7 +36,6 @@ async function relationalQuery(websiteId: string, sessionId: string) {
session.distinct_id,
website_event.visit_id,
session.website_id,
- website_event.hostname,
session.browser,
session.os,
session.device,
@@ -54,8 +52,8 @@ async function relationalQuery(websiteId: string, sessionId: string) {
join website_event on website_event.session_id = session.session_id
where session.website_id = {{websiteId::uuid}}
and session.session_id = {{sessionId::uuid}}
- group by session.session_id, session.distinct_id, visit_id, session.website_id, website_event.hostname, session.browser, session.os, session.device, session.screen, session.language, session.country, session.region, session.city) t
- group by id, distinct_id, website_id, hostname, browser, os, device, screen, language, country, region, city;
+ group by session.session_id, session.distinct_id, visit_id, session.website_id, session.browser, session.os, session.device, session.screen, session.language, session.country, session.region, session.city) t
+ group by id, distinct_id, website_id, browser, os, device, screen, language, country, region, city;
`,
{ websiteId, sessionId },
).then(result => result?.[0]);
@@ -69,7 +67,6 @@ async function clickhouseQuery(websiteId: string, sessionId: string) {
select id,
websiteId,
distinctId,
- hostname,
browser,
os,
device,
@@ -89,7 +86,6 @@ async function clickhouseQuery(websiteId: string, sessionId: string) {
distinct_id as distinctId,
visit_id,
website_id as websiteId,
- hostname,
browser,
os,
device,
@@ -105,8 +101,8 @@ async function clickhouseQuery(websiteId: string, sessionId: string) {
from website_event_stats_hourly
where website_id = {websiteId:UUID}
and session_id = {sessionId:UUID}
- group by session_id, distinct_id, visit_id, website_id, hostname, browser, os, device, screen, language, country, region, city) t
- group by id, websiteId, distinctId, hostname, browser, os, device, screen, language, country, region, city;
+ group by session_id, distinct_id, visit_id, website_id, browser, os, device, screen, language, country, region, city) t
+ group by id, websiteId, distinctId, browser, os, device, screen, language, country, region, city;
`,
{ websiteId, sessionId },
).then(result => result?.[0]);
diff --git a/src/queries/sql/sessions/getWebsiteSessions.ts b/src/queries/sql/sessions/getWebsiteSessions.ts
index 8f99df2d5..6171e5146 100644
--- a/src/queries/sql/sessions/getWebsiteSessions.ts
+++ b/src/queries/sql/sessions/getWebsiteSessions.ts
@@ -27,7 +27,6 @@ async function relationalQuery(websiteId: string, filters: QueryFilters, pagePar
select
session.session_id as "id",
session.website_id as "websiteId",
- website_event.hostname,
session.browser,
session.os,
session.device,
@@ -58,7 +57,6 @@ async function relationalQuery(websiteId: string, filters: QueryFilters, pagePar
}
group by session.session_id,
session.website_id,
- website_event.hostname,
session.browser,
session.os,
session.device,
@@ -84,7 +82,6 @@ async function clickhouseQuery(websiteId: string, filters: QueryFilters, pagePar
select
session_id as id,
website_id as websiteId,
- hostname,
browser,
os,
device,
@@ -112,7 +109,7 @@ async function clickhouseQuery(websiteId: string, filters: QueryFilters, pagePar
or (positionCaseInsensitive(device, {search:String}) > 0))`
: ''
}
- group by session_id, website_id, hostname, browser, os, device, screen, language, country, region, city
+ group by session_id, website_id, browser, os, device, screen, language, country, region, city
order by lastAt desc
`,
{ ...params, search },