Umami is a privacy-first web and product analytics platform. Traffic, campaigns, behavior, conversions, and revenue in one place. No cookies, no surveillance, self-hosted or in the cloud.
TypeScript analyticsaudience-segmentationchartscohort-analysisgoogle-analyticsproduct-analyticsstatisticsuser-journeyweb-analytics
change pagestable to visitors and update clickhouse hostname column to array
Francis Cao2dcb9e21bdf9
diff --git a/db/clickhouse/migrations/08_update_hostname_view.sql b/db/clickhouse/migrations/08_update_hostname_view.sqlnew file mode 100644index 000000000..061fa6285--- /dev/null+++ b/db/clickhouse/migrations/08_update_hostname_view.sql@@ -0,0 +1,253 @@+-- create new hourly table+CREATE TABLE umami.website_event_stats_hourly_new+(+ website_id UUID,+ session_id UUID,+ visit_id UUID,+ hostname SimpleAggregateFunction(groupArrayArray, Array(String)),+ browser LowCardinality(String),+ os LowCardinality(String),+ device LowCardinality(String),+ screen LowCardinality(String),+ language LowCardinality(String),+ country LowCardinality(String),+ region LowCardinality(String),+ city String,+ entry_url AggregateFunction(argMin, String, DateTime('UTC')),+ exit_url AggregateFunction(argMax, String, DateTime('UTC')),+ url_path SimpleAggregateFunction(groupArrayArray, Array(String)),+ url_query SimpleAggregateFunction(groupArrayArray, Array(String)),+ utm_source SimpleAggregateFunction(groupArrayArray, Array(String)),+ utm_medium SimpleAggregateFunction(groupArrayArray, Array(String)),+ utm_campaign SimpleAggregateFunction(groupArrayArray, Array(String)),+ utm_content SimpleAggregateFunction(groupArrayArray, Array(String)),+ utm_term SimpleAggregateFunction(groupArrayArray, Array(String)),+ referrer_domain SimpleAggregateFunction(groupArrayArray, Array(String)),+ page_title SimpleAggregateFunction(groupArrayArray, Array(String)),+ gclid SimpleAggregateFunction(groupArrayArray, Array(String)),+ fbclid SimpleAggregateFunction(groupArrayArray, Array(String)),+ msclkid SimpleAggregateFunction(groupArrayArray, Array(String)),+ ttclid SimpleAggregateFunction(groupArrayArray, Array(String)),+ li_fat_id SimpleAggregateFunction(groupArrayArray, Array(String)),+ twclid SimpleAggregateFunction(groupArrayArray, Array(String)),+ event_type UInt32,+ event_name SimpleAggregateFunction(groupArrayArray, Array(String)),+ views SimpleAggregateFunction(sum, UInt64),+ min_time SimpleAggregateFunction(min, DateTime('UTC')),+ max_time SimpleAggregateFunction(max, DateTime('UTC')),+ tag SimpleAggregateFunction(groupArrayArray, Array(String)),+ distinct_id String,+ created_at Datetime('UTC')+)+ENGINE = AggregatingMergeTree+ PARTITION BY toYYYYMM(created_at)+ ORDER BY (+ website_id,+ event_type,+ toStartOfHour(created_at),+ cityHash64(visit_id),+ visit_id+ )+ SAMPLE BY cityHash64(visit_id);++-- create view+CREATE MATERIALIZED VIEW umami.website_event_stats_hourly_mv_new+TO umami.website_event_stats_hourly_new+AS+SELECT+ website_id,+ session_id,+ visit_id,+ hostnames as hostname,+ browser,+ os,+ device,+ screen,+ language,+ country,+ region,+ city,+ entry_url,+ exit_url,+ url_paths as url_path,+ url_query,+ utm_source,+ utm_medium,+ utm_campaign,+ utm_content,+ utm_term,+ referrer_domain,+ page_title,+ gclid,+ fbclid,+ msclkid,+ ttclid,+ li_fat_id,+ twclid,+ event_type,+ event_name,+ views,+ min_time,+ max_time,+ tag,+ distinct_id,+ timestamp as created_at+FROM (SELECT+ website_id,+ session_id,+ visit_id,+ arrayFilter(x -> x != '', groupArray(hostname)) hostnames,+ browser,+ os,+ device,+ screen,+ language,+ country,+ region,+ city,+ argMinState(url_path, created_at) entry_url,+ argMaxState(url_path, created_at) exit_url,+ arrayFilter(x -> x != '', groupArray(url_path)) as url_paths,+ arrayFilter(x -> x != '', groupArray(url_query)) url_query,+ arrayFilter(x -> x != '', groupArray(utm_source)) utm_source,+ arrayFilter(x -> x != '', groupArray(utm_medium)) utm_medium,+ arrayFilter(x -> x != '', groupArray(utm_campaign)) utm_campaign,+ arrayFilter(x -> x != '', groupArray(utm_content)) utm_content,+ arrayFilter(x -> x != '', groupArray(utm_term)) utm_term,+ arrayFilter(x -> x != '' and x != hostname, groupArray(referrer_domain)) referrer_domain,+ arrayFilter(x -> x != '', groupArray(page_title)) page_title,+ arrayFilter(x -> x != '', groupArray(gclid)) gclid,+ arrayFilter(x -> x != '', groupArray(fbclid)) fbclid,+ arrayFilter(x -> x != '', groupArray(msclkid)) msclkid,+ arrayFilter(x -> x != '', groupArray(ttclid)) ttclid,+ arrayFilter(x -> x != '', groupArray(li_fat_id)) li_fat_id,+ arrayFilter(x -> x != '', groupArray(twclid)) twclid,+ event_type,+ if(event_type = 2, groupArray(event_name), []) event_name,+ sumIf(1, event_type = 1) views,+ min(created_at) min_time,+ max(created_at) max_time,+ arrayFilter(x -> x != '', groupArray(tag)) tag,+ distinct_id,+ toStartOfHour(created_at) timestamp+FROM umami.website_event+GROUP BY website_id,+ session_id,+ visit_id,+ hostname,+ browser,+ os,+ device,+ screen,+ language,+ country,+ region,+ city,+ event_type,+ distinct_id,+ timestamp);++-- rename tables+RENAME TABLE umami.website_event_stats_hourly TO umami.website_event_stats_hourly_old;+RENAME TABLE umami.website_event_stats_hourly_new TO umami.website_event_stats_hourly;++-- drop views+DROP TABLE umami.website_event_stats_hourly_mv;+DROP TABLE umami.website_event_stats_hourly_mv_new;++-- recreate view+CREATE MATERIALIZED VIEW umami.website_event_stats_hourly_mv+TO umami.website_event_stats_hourly+AS+SELECT+ website_id,+ session_id,+ visit_id,+ hostnames as hostname,+ browser,+ os,+ device,+ screen,+ language,+ country,+ region,+ city,+ entry_url,+ exit_url,+ url_paths as url_path,+ url_query,+ utm_source,+ utm_medium,+ utm_campaign,+ utm_content,+ utm_term,+ referrer_domain,+ page_title,+ gclid,+ fbclid,+ msclkid,+ ttclid,+ li_fat_id,+ twclid,+ event_type,+ event_name,+ views,+ min_time,+ max_time,+ tag,+ distinct_id,+ timestamp as created_at+FROM (SELECT+ website_id,+ session_id,+ visit_id,+ arrayFilter(x -> x != '', groupArray(hostname)) hostnames,+ browser,+ os,+ device,+ screen,+ language,+ country,+ region,+ city,+ argMinState(url_path, created_at) entry_url,+ argMaxState(url_path, created_at) exit_url,+ arrayFilter(x -> x != '', groupArray(url_path)) as url_paths,+ arrayFilter(x -> x != '', groupArray(url_query)) url_query,+ arrayFilter(x -> x != '', groupArray(utm_source)) utm_source,+ arrayFilter(x -> x != '', groupArray(utm_medium)) utm_medium,+ arrayFilter(x -> x != '', groupArray(utm_campaign)) utm_campaign,+ arrayFilter(x -> x != '', groupArray(utm_content)) utm_content,+ arrayFilter(x -> x != '', groupArray(utm_term)) utm_term,+ arrayFilter(x -> x != '' and x != hostname, groupArray(referrer_domain)) referrer_domain,+ arrayFilter(x -> x != '', groupArray(page_title)) page_title,+ arrayFilter(x -> x != '', groupArray(gclid)) gclid,+ arrayFilter(x -> x != '', groupArray(fbclid)) fbclid,+ arrayFilter(x -> x != '', groupArray(msclkid)) msclkid,+ arrayFilter(x -> x != '', groupArray(ttclid)) ttclid,+ arrayFilter(x -> x != '', groupArray(li_fat_id)) li_fat_id,+ arrayFilter(x -> x != '', groupArray(twclid)) twclid,+ event_type,+ if(event_type = 2, groupArray(event_name), []) event_name,+ sumIf(1, event_type = 1) views,+ min(created_at) min_time,+ max(created_at) max_time,+ arrayFilter(x -> x != '', groupArray(tag)) tag,+ distinct_id,+ toStartOfHour(created_at) timestamp+FROM umami.website_event+GROUP BY website_id,+ session_id,+ visit_id,+ hostname,+ browser,+ os,+ device,+ screen,+ language,+ country,+ region,+ city,+ event_type,+ distinct_id,+ timestamp);diff --git a/db/clickhouse/schema.sql b/db/clickhouse/schema.sqlindex d9831f57a..41b3a1b3e 100644--- a/db/clickhouse/schema.sql+++ b/db/clickhouse/schema.sql@@ -90,7 +90,7 @@ CREATE TABLE umami.website_event_stats_hourlywebsite_id UUID,session_id UUID,visit_id UUID,- hostname LowCardinality(String),+ hostname SimpleAggregateFunction(groupArrayArray, Array(String)),browser LowCardinality(String),os LowCardinality(String),device LowCardinality(String),@@ -143,7 +143,7 @@ SELECTwebsite_id,session_id,visit_id,- hostname,+ hostnames as hostname,browser,os,device,@@ -181,7 +181,7 @@ FROM (SELECTwebsite_id,session_id,visit_id,- hostname,+ arrayFilter(x -> x != '', groupArray(hostname)) hostnames,browser,os,device,@@ -199,7 +199,7 @@ FROM (SELECTarrayFilter(x -> x != '', groupArray(utm_campaign)) utm_campaign,arrayFilter(x -> x != '', groupArray(utm_content)) utm_content,arrayFilter(x -> x != '', groupArray(utm_term)) utm_term,- arrayFilter(x -> x != '', groupArray(referrer_domain)) referrer_domain,+ arrayFilter(x -> x != '' and x != hostname, groupArray(referrer_domain)) referrer_domain,arrayFilter(x -> x != '', groupArray(page_title)) page_title,arrayFilter(x -> x != '', groupArray(gclid)) gclid,arrayFilter(x -> x != '', groupArray(fbclid)) fbclid,diff --git a/src/components/metrics/PagesTable.tsx b/src/components/metrics/PagesTable.tsxindex 8163b3d9c..59cfda43d 100644--- a/src/components/metrics/PagesTable.tsx+++ b/src/components/metrics/PagesTable.tsx@@ -62,7 +62,7 @@ export function PagesTable({ allowFilter, ...props }: PagesTableProps) {{...props}title={formatMessage(labels.pages)}type={view}- metric={formatMessage(labels.views)}+ metric={formatMessage(labels.visitors)}dataFilter={emptyFilter}renderLabel={renderLink}>diff --git a/src/lib/constants.ts b/src/lib/constants.tsindex 39ecbc088..7fd1a85d2 100644--- a/src/lib/constants.ts+++ b/src/lib/constants.ts@@ -33,7 +33,17 @@ export const FILTER_REFERRERS = 'filter-referrers';export const FILTER_PAGES = 'filter-pages';export const UNIT_TYPES = ['year', 'month', 'hour', 'day', 'minute'];-export const EVENT_COLUMNS = ['url', 'entry', 'exit', 'referrer', 'title', 'query', 'event', 'tag'];+export const EVENT_COLUMNS = [+ 'url',+ 'entry',+ 'exit',+ 'referrer',+ 'title',+ 'query',+ 'event',+ 'tag',+ 'host',+];export const SESSION_COLUMNS = ['browser',@@ -44,7 +54,6 @@ export const SESSION_COLUMNS = ['country','city','region',- 'host',];export const FILTER_GROUPS = {diff --git a/src/queries/sql/getValues.ts b/src/queries/sql/getValues.tsindex 2d1286f30..5322a5319 100644--- a/src/queries/sql/getValues.ts+++ b/src/queries/sql/getValues.ts@@ -21,6 +21,12 @@ async function relationalQuery(const { rawQuery, getSearchSQL } = prisma;const params = {};let searchQuery = '';+ let excludeDomain = '';++ if (column === 'referrer_domain') {+ excludeDomain = `and website_event.referrer_domain != website_event.hostname+ and website_event.referrer_domain != ''`;+ }if (search) {if (decodeURIComponent(search).includes(',')) {@@ -49,6 +55,7 @@ async function relationalQuery(where website_event.website_id = {{websiteId::uuid}}and website_event.created_at between {{startDate}} and {{endDate}}${searchQuery}+ ${excludeDomain}group by 1order by 2 desclimit 10@@ -73,6 +80,11 @@ async function clickhouseQuery(const { rawQuery, getSearchSQL } = clickhouse;const params = {};let searchQuery = '';+ let excludeDomain = '';++ if (column === 'referrer_domain') {+ excludeDomain = `and referrer_domain != hostname and referrer_domain != ''`;+ }if (search) {searchQuery = `and positionCaseInsensitive(${column}, {search:String}) > 0`;@@ -103,6 +115,7 @@ async function clickhouseQuery(where website_id = {websiteId:UUID}and created_at between {startDate:DateTime64} and {endDate:DateTime64}${searchQuery}+ ${excludeDomain}group by 1order by 2 desclimit 10diff --git a/src/queries/sql/pageviews/getPageviewMetrics.ts b/src/queries/sql/pageviews/getPageviewMetrics.tsindex 72586eb79..67e18b75d 100644--- a/src/queries/sql/pageviews/getPageviewMetrics.ts+++ b/src/queries/sql/pageviews/getPageviewMetrics.ts@@ -34,7 +34,7 @@ async function relationalQuery(...filters,eventType: column === 'event_name' ? EVENT_TYPE.customEvent : EVENT_TYPE.pageView,},- { joinSession: SESSION_COLUMNS.includes(type) || column === 'referrer_domain' },+ { joinSession: SESSION_COLUMNS.includes(type) },);let entryExitQuery = '';@@ -66,7 +66,7 @@ async function relationalQuery(return rawQuery(`select ${column} x,- ${column === 'referrer_domain' ? 'count(distinct website_event.session_id)' : 'count(*)'} as y+ count(distinct website_event.session_id) as yfrom website_event${cohortQuery}${joinSession}@@ -126,7 +126,7 @@ async function clickhouseQuery(sql = `select ${column} x,- ${column === 'referrer_domain' ? 'uniq(session_id)' : 'count(*)'} as y+ uniq(website_event.session_id) as yfrom website_event${cohortQuery}${entryExitQuery}@@ -142,28 +142,27 @@ async function clickhouseQuery(`;} else {let groupByQuery = '';- let columnQuery = `arrayJoin(${column})`;+ let columnQuery = `session_id s, arrayJoin(${column})`;if (column === 'referrer_domain') {- excludeDomain = `and t != hostname and t != ''`;- columnQuery = `session_id s, arrayJoin(${column})`;+ excludeDomain = `and t != ''`;}if (type === 'entry') {- columnQuery = `visit_id x, argMinMerge(entry_url)`;+ columnQuery = `session_id s, argMinMerge(entry_url)`;}if (type === 'exit') {- columnQuery = `visit_id x, argMaxMerge(exit_url)`;+ columnQuery = `session_id s, argMaxMerge(exit_url)`;}if (type === 'entry' || type === 'exit') {- groupByQuery = 'group by x';+ groupByQuery = 'group by s';}sql = `select g.t as x,- ${column === 'referrer_domain' ? 'uniq(s)' : 'count(*)'} as y+ uniq(s) as yfrom (select ${columnQuery} as tfrom website_event_stats_hourly website_eventdiff --git a/src/queries/sql/sessions/getWebsiteSession.ts b/src/queries/sql/sessions/getWebsiteSession.tsindex 97850a2c1..8c549ca30 100644--- a/src/queries/sql/sessions/getWebsiteSession.ts+++ b/src/queries/sql/sessions/getWebsiteSession.ts@@ -17,7 +17,6 @@ async function relationalQuery(websiteId: string, sessionId: string) {select id,distinct_id as "distinctId",website_id as "websiteId",- hostname,browser,os,device,@@ -37,7 +36,6 @@ async function relationalQuery(websiteId: string, sessionId: string) {session.distinct_id,website_event.visit_id,session.website_id,- website_event.hostname,session.browser,session.os,session.device,@@ -54,8 +52,8 @@ async function relationalQuery(websiteId: string, sessionId: string) {join website_event on website_event.session_id = session.session_idwhere session.website_id = {{websiteId::uuid}}and session.session_id = {{sessionId::uuid}}- group by session.session_id, session.distinct_id, visit_id, session.website_id, website_event.hostname, session.browser, session.os, session.device, session.screen, session.language, session.country, session.region, session.city) t- group by id, distinct_id, website_id, hostname, browser, os, device, screen, language, country, region, city;+ group by session.session_id, session.distinct_id, visit_id, session.website_id, session.browser, session.os, session.device, session.screen, session.language, session.country, session.region, session.city) t+ group by id, distinct_id, website_id, browser, os, device, screen, language, country, region, city;`,{ websiteId, sessionId },).then(result => result?.[0]);@@ -69,7 +67,6 @@ async function clickhouseQuery(websiteId: string, sessionId: string) {select id,websiteId,distinctId,- hostname,browser,os,device,@@ -89,7 +86,6 @@ async function clickhouseQuery(websiteId: string, sessionId: string) {distinct_id as distinctId,visit_id,website_id as websiteId,- hostname,browser,os,device,@@ -105,8 +101,8 @@ async function clickhouseQuery(websiteId: string, sessionId: string) {from website_event_stats_hourlywhere website_id = {websiteId:UUID}and session_id = {sessionId:UUID}- group by session_id, distinct_id, visit_id, website_id, hostname, browser, os, device, screen, language, country, region, city) t- group by id, websiteId, distinctId, hostname, browser, os, device, screen, language, country, region, city;+ group by session_id, distinct_id, visit_id, website_id, browser, os, device, screen, language, country, region, city) t+ group by id, websiteId, distinctId, browser, os, device, screen, language, country, region, city;`,{ websiteId, sessionId },).then(result => result?.[0]);diff --git a/src/queries/sql/sessions/getWebsiteSessions.ts b/src/queries/sql/sessions/getWebsiteSessions.tsindex 8f99df2d5..6171e5146 100644--- a/src/queries/sql/sessions/getWebsiteSessions.ts+++ b/src/queries/sql/sessions/getWebsiteSessions.ts@@ -27,7 +27,6 @@ async function relationalQuery(websiteId: string, filters: QueryFilters, pageParselectsession.session_id as "id",session.website_id as "websiteId",- website_event.hostname,session.browser,session.os,session.device,@@ -58,7 +57,6 @@ async function relationalQuery(websiteId: string, filters: QueryFilters, pagePar}group by session.session_id,session.website_id,- website_event.hostname,session.browser,session.os,session.device,@@ -84,7 +82,6 @@ async function clickhouseQuery(websiteId: string, filters: QueryFilters, pageParselectsession_id as id,website_id as websiteId,- hostname,browser,os,device,@@ -112,7 +109,7 @@ async function clickhouseQuery(websiteId: string, filters: QueryFilters, pageParor (positionCaseInsensitive(device, {search:String}) > 0))`: ''}- group by session_id, website_id, hostname, browser, os, device, screen, language, country, region, city+ group by session_id, website_id, browser, os, device, screen, language, country, region, cityorder by lastAt desc`,{ ...params, search },