Umami is a privacy-first web and product analytics platform. Traffic, campaigns, behavior, conversions, and revenue in one place. No cookies, no surveillance, self-hosted or in the cloud.
TypeScript analyticsaudience-segmentationchartscohort-analysisgoogle-analyticsproduct-analyticsstatisticsuser-journeyweb-analytics
route dashboard queries based on filters selected
Francis Cao9882ff24f6fe
diff --git a/src/lib/clickhouse.ts b/src/lib/clickhouse.tsindex 2bda9bfa8..4c544a71d 100644--- a/src/lib/clickhouse.ts+++ b/src/lib/clickhouse.ts@@ -3,7 +3,7 @@ import dateFormat from 'dateformat';import debug from 'debug';import { CLICKHOUSE } from 'lib/db';import { PageParams, QueryFilters, QueryOptions } from './types';-import { DEFAULT_PAGE_SIZE, OPERATORS } from './constants';+import { EVENT_COLUMNS, DEFAULT_PAGE_SIZE, OPERATORS } from './constants';import { fetchWebsite } from './load';import { maxDate } from './date';import { filtersToArray } from './params';@@ -100,6 +100,26 @@ function getFilterQuery(filters: QueryFilters = {}, options: QueryOptions = {})return query.join('\n');}+function getSessionFilterQuery(filters: QueryFilters = {}, options: QueryOptions = {}) {+ const query = filtersToArray(filters, options).reduce((arr, { name, column, operator }) => {+ if (column) {+ if (EVENT_COLUMNS.includes(name)) {+ arr.push(`and has(${column}, {${name}:String})`);++ if (name === 'referrer') {+ arr.push('and not has(referrer_domain, {websiteDomain:String})');+ }+ } else {+ arr.push(`and ${mapFilter(column, operator, name)}`);+ }+ }++ return arr;+ }, []);++ return query.join('\n');+}+function getDateQuery(filters: QueryFilters = {}) {const { startDate, endDate } = filters;@@ -139,6 +159,25 @@ async function parseFilters(websiteId: string, filters: QueryFilters = {}, optio};}+async function parseSessionFilters(+ websiteId: string,+ filters: QueryFilters = {},+ options?: QueryOptions,+) {+ const website = await fetchWebsite(websiteId);++ return {+ filterQuery: getSessionFilterQuery(filters, options),+ dateQuery: getDateQuery(filters),+ params: {+ ...getFilterParams(filters),+ websiteId,+ startDate: maxDate(filters.startDate, new Date(website?.resetAt)),+ websiteDomain: website.domain,+ },+ };+}+async function pagedQuery(query: string,queryParams: { [key: string]: any },@@ -221,6 +260,7 @@ export default {getDateFormat,getFilterQuery,parseFilters,+ parseSessionFilters,pagedQuery,findUnique,findFirst,diff --git a/src/lib/constants.ts b/src/lib/constants.tsindex aa1b3c0f2..9d696f114 100644--- a/src/lib/constants.ts+++ b/src/lib/constants.ts@@ -33,16 +33,7 @@ export const FILTER_REFERRERS = 'filter-referrers';export const FILTER_PAGES = 'filter-pages';export const UNIT_TYPES = ['year', 'month', 'hour', 'day', 'minute'];-export const EVENT_COLUMNS = [- 'url',- 'entry',- 'exit',- 'referrer',- 'title',- 'query',- 'event',- 'host',-];+export const EVENT_COLUMNS = ['url', 'entry', 'exit', 'referrer', 'title', 'query', 'event'];export const SESSION_COLUMNS = ['browser',@@ -58,8 +49,8 @@ export const SESSION_COLUMNS = [export const FILTER_COLUMNS = {url: 'url_path',- entry: 'entry_url',- exit: 'exit_url',+ entry: 'url_path',+ exit: 'url_path',referrer: 'referrer_domain',host: 'hostname',title: 'page_title',diff --git a/src/queries/analytics/events/getEventMetrics.ts b/src/queries/analytics/events/getEventMetrics.tsindex 362321358..504cea119 100644--- a/src/queries/analytics/events/getEventMetrics.ts+++ b/src/queries/analytics/events/getEventMetrics.ts@@ -1,8 +1,8 @@-import prisma from 'lib/prisma';import clickhouse from 'lib/clickhouse';-import { runQuery, CLICKHOUSE, PRISMA } from 'lib/db';-import { WebsiteEventMetric, QueryFilters } from 'lib/types';import { EVENT_TYPE } from 'lib/constants';+import { CLICKHOUSE, PRISMA, runQuery } from 'lib/db';+import prisma from 'lib/prisma';+import { QueryFilters, WebsiteEventMetric } from 'lib/types';export async function getEventMetrics(...args: [websiteId: string, filters: QueryFilters]@@ -51,8 +51,24 @@ async function clickhouseQuery(eventType: EVENT_TYPE.customEvent,});- return rawQuery(- `+ let sql = '';++ if (filterQuery) {+ sql = `+ select+ event_name x,+ ${getDateSQL('created_at', unit, timezone)} t,+ count(*) y+ from website_event+ where website_id = {websiteId:UUID}+ and created_at between {startDate:DateTime64} and {endDate:DateTime64}+ and event_type = {eventType:UInt32}+ ${filterQuery}+ group by x, t+ order by t+ `;+ } else {+ sql = `selectevent_name x,${getDateSQL('created_at', unit, timezone)} t,@@ -64,13 +80,13 @@ async function clickhouseQuery(where website_id = {websiteId:UUID}and created_at between {startDate:DateTime64} and {endDate:DateTime64}and event_type = {eventType:UInt32}- ${filterQuery}) as ggroup by x, torder by t- `,- params,- ).then(a => {+ `;+ }++ return rawQuery(sql, params).then(a => {return Object.values(a).map(a => {return { x: a.x, t: a.t, y: Number(a.y) };});diff --git a/src/queries/analytics/getWebsiteStats.ts b/src/queries/analytics/getWebsiteStats.tsindex 7d8a76f69..09eebb919 100644--- a/src/queries/analytics/getWebsiteStats.ts+++ b/src/queries/analytics/getWebsiteStats.ts@@ -3,6 +3,7 @@ import { EVENT_TYPE } from 'lib/constants';import { CLICKHOUSE, PRISMA, runQuery } from 'lib/db';import prisma from 'lib/prisma';import { QueryFilters } from 'lib/types';+import { EVENT_COLUMNS } from 'lib/constants';export async function getWebsiteStats(...args: [websiteId: string, filters: QueryFilters]@@ -67,30 +68,33 @@ async function clickhouseQuery(eventType: EVENT_TYPE.pageView,});- return rawQuery(- // `- // select- // sum(t.c) as "pageviews",- // count(distinct t.session_id) as "visitors",- // count(distinct t.visit_id) as "visits",- // sum(if(t.c = 1, 1, 0)) as "bounces",- // sum(max_time-min_time) as "totaltime"- // from (- // select- // session_id,- // visit_id,- // count(*) c,- // min(created_at) min_time,- // max(created_at) max_time- // from website_event- // where website_id = {websiteId:UUID}- // and created_at between {startDate:DateTime64} and {endDate:DateTime64}- // and event_type = {eventType:UInt32}- // ${filterQuery}- // group by session_id, visit_id- // ) as t;- // `,- `+ let sql = '';++ if (EVENT_COLUMNS.some(item => Object.keys(filters).includes(item))) {+ sql = `+ select+ sum(t.c) as "pageviews",+ count(distinct t.session_id) as "visitors",+ count(distinct t.visit_id) as "visits",+ sum(if(t.c = 1, 1, 0)) as "bounces",+ sum(max_time-min_time) as "totaltime"+ from (+ select+ session_id,+ visit_id,+ count(*) c,+ min(created_at) min_time,+ max(created_at) max_time+ from website_event+ where website_id = {websiteId:UUID}+ and created_at between {startDate:DateTime64} and {endDate:DateTime64}+ and event_type = {eventType:UInt32}+ ${filterQuery}+ group by session_id, visit_id+ ) as t;+ `;+ } else {+ sql = `selectsum(views) as "pageviews",uniq(session_id) as "visitors",@@ -102,9 +106,10 @@ async function clickhouseQuery(and created_at between {startDate:DateTime64} and {endDate:DateTime64}and event_type = {eventType:UInt32}${filterQuery};- `,- params,- ).then(result => {+ `;+ }++ return rawQuery(sql, params).then(result => {return Object.values(result).map((a: any) => {return {pageviews: Number(a.pageviews),diff --git a/src/queries/analytics/pageviews/getPageviewMetrics.ts b/src/queries/analytics/pageviews/getPageviewMetrics.tsindex 4ac101b2c..f734b1ddc 100644--- a/src/queries/analytics/pageviews/getPageviewMetrics.ts+++ b/src/queries/analytics/pageviews/getPageviewMetrics.ts@@ -1,5 +1,5 @@import clickhouse from 'lib/clickhouse';-import { EVENT_TYPE, FILTER_COLUMNS, SESSION_COLUMNS } from 'lib/constants';+import { EVENT_COLUMNS, EVENT_TYPE, FILTER_COLUMNS, SESSION_COLUMNS } from 'lib/constants';import { CLICKHOUSE, PRISMA, runQuery } from 'lib/db';import prisma from 'lib/prisma';import { QueryFilters } from 'lib/types';@@ -91,46 +91,66 @@ async function clickhouseQuery(});let excludeDomain = '';- let groupByQuery = '';+ let sql = '';- if (column === 'referrer_domain') {- excludeDomain = `and t != {websiteDomain:String} and t != ''`;- }+ if (EVENT_COLUMNS.some(item => Object.keys(filters).includes(item))) {+ let entryExitQuery = '';- let columnQuery = `arrayJoin(${column})`;+ if (column === 'referrer_domain') {+ excludeDomain = `and referrer_domain != {websiteDomain:String} and referrer_domain != ''`;+ }- if (type === 'entry') {- columnQuery = `visit_id x, argMinMerge(${column})`;- }+ if (type === 'entry' || type === 'exit') {+ const aggregrate = type === 'entry' ? 'min' : 'max';- if (type === 'exit') {- columnQuery = `visit_id x, argMaxMerge(${column})`;- }+ entryExitQuery = `+ JOIN (select visit_id,+ ${aggregrate}(created_at) target_created_at+ from website_event+ where website_id = {websiteId:UUID}+ and created_at between {startDate:DateTime64} and {endDate:DateTime64}+ and event_type = {eventType:UInt32}+ group by visit_id) x+ ON x.visit_id = website_event.visit_id+ and x.target_created_at = website_event.created_at`;+ }- if (type === 'entry' || type === 'exit') {- groupByQuery = 'group by x';- }+ sql = `+ select ${column} x, count(*) y+ from website_event+ ${entryExitQuery}+ where website_id = {websiteId:UUID}+ and created_at between {startDate:DateTime64} and {endDate:DateTime64}+ and event_type = {eventType:UInt32}+ ${excludeDomain}+ ${filterQuery}+ group by x+ order by y desc+ limit ${limit}+ offset ${offset}+ `;+ } else {+ let groupByQuery = '';- // let excludeDomain = '';- // if (column === 'referrer_domain') {- // excludeDomain = `and referrer_domain != {websiteDomain:String} and referrer_domain != ''`;- // }+ if (column === 'referrer_domain') {+ excludeDomain = `and t != {websiteDomain:String} and t != ''`;+ }- return rawQuery(- // `- // select ${column} x, count(*) y- // from website_event- // where website_id = {websiteId:UUID}- // and created_at between {startDate:DateTime64} and {endDate:DateTime64}- // and event_type = {eventType:UInt32}- // ${excludeDomain}- // ${filterQuery}- // group by x- // order by y desc- // limit ${limit}- // offset ${offset}- // `,- `+ let columnQuery = `arrayJoin(${column})`;++ if (type === 'entry') {+ columnQuery = `visit_id x, argMinMerge(entry_url)`;+ }++ if (type === 'exit') {+ columnQuery = `visit_id x, argMaxMerge(exit_url)`;+ }++ if (type === 'entry' || type === 'exit') {+ groupByQuery = 'group by x';+ }++ sql = `select g.t as x,count(*) as yfrom (@@ -146,9 +166,10 @@ async function clickhouseQuery(order by y desclimit ${limit}offset ${offset}- `,- params,- ).then((result: any) => {+ `;+ }++ return rawQuery(sql, params).then((result: any) => {return Object.values(result).map((a: any) => {return { x: a.x, y: Number(a.y) };});diff --git a/src/queries/analytics/pageviews/getPageviewStats.ts b/src/queries/analytics/pageviews/getPageviewStats.tsindex c378d48a6..1d027e794 100644--- a/src/queries/analytics/pageviews/getPageviewStats.ts+++ b/src/queries/analytics/pageviews/getPageviewStats.ts@@ -1,7 +1,7 @@import clickhouse from 'lib/clickhouse';import { CLICKHOUSE, PRISMA, runQuery } from 'lib/db';import prisma from 'lib/prisma';-import { EVENT_TYPE } from 'lib/constants';+import { EVENT_COLUMNS, EVENT_TYPE } from 'lib/constants';import { QueryFilters } from 'lib/types';export async function getPageviewStats(...args: [websiteId: string, filters: QueryFilters]) {@@ -47,36 +47,18 @@ async function clickhouseQuery(eventType: EVENT_TYPE.pageView,});- const table = unit === 'minute' ? 'website_event' : 'website_event_stats_hourly';- const columnQuery = unit === 'minute' ? 'count(*)' : 'sum(views)';+ let sql = '';- return rawQuery(- // `- // select- // ${getDateStringSQL('g.t', unit)} as x,- // g.y as y- // from (- // select- // ${getDateSQL('created_at', unit, timezone)} as t,- // count(*) as y- // from website_event- // where website_id = {websiteId:UUID}- // and created_at between {startDate:DateTime64} and {endDate:DateTime64}- // and event_type = {eventType:UInt32}- // ${filterQuery}- // group by t- // ) as g- // order by t- // `,- `+ if (EVENT_COLUMNS.some(item => Object.keys(filters).includes(item)) || unit === 'minute') {+ sql = `select${getDateStringSQL('g.t', unit)} as x,g.y as yfrom (select${getDateSQL('created_at', unit, timezone)} as t,- ${columnQuery} as y- from ${table} website_event+ count(*) as y+ from website_eventwhere website_id = {websiteId:UUID}and created_at between {startDate:DateTime64} and {endDate:DateTime64}and event_type = {eventType:UInt32}@@ -84,9 +66,28 @@ async function clickhouseQuery(group by t) as gorder by t- `,- params,- ).then(result => {+ `;+ } else {+ sql = `+ select+ ${getDateStringSQL('g.t', unit)} as x,+ g.y as y+ from (+ select+ ${getDateSQL('created_at', unit, timezone)} as t,+ sum(views)as y+ from website_event_stats_hourly website_event+ where website_id = {websiteId:UUID}+ and created_at between {startDate:DateTime64} and {endDate:DateTime64}+ and event_type = {eventType:UInt32}+ ${filterQuery}+ group by t+ ) as g+ order by t+ `;+ }++ return rawQuery(sql, params).then(result => {return Object.values(result).map((a: any) => {return { x: a.x, y: Number(a.y) };});diff --git a/src/queries/analytics/sessions/getSessionMetrics.ts b/src/queries/analytics/sessions/getSessionMetrics.tsindex 9baf2a5c0..3e6f53c0a 100644--- a/src/queries/analytics/sessions/getSessionMetrics.ts+++ b/src/queries/analytics/sessions/getSessionMetrics.ts@@ -64,8 +64,8 @@ async function clickhouseQuery(offset: number = 0,): Promise<{ x: string; y: number }[]> {const column = FILTER_COLUMNS[type] || type;- const { parseFilters, rawQuery } = clickhouse;- const { filterQuery, params } = await parseFilters(websiteId, {+ const { parseSessionFilters, rawQuery } = clickhouse;+ const { filterQuery, params } = await parseSessionFilters(websiteId, {...filters,eventType: EVENT_TYPE.pageView,});diff --git a/src/queries/analytics/sessions/getSessionStats.ts b/src/queries/analytics/sessions/getSessionStats.tsindex 7bba14df1..fa0ed6a96 100644--- a/src/queries/analytics/sessions/getSessionStats.ts+++ b/src/queries/analytics/sessions/getSessionStats.ts@@ -41,8 +41,8 @@ async function clickhouseQuery(filters: QueryFilters,): Promise<{ x: string; y: number }[]> {const { timezone = 'UTC', unit = 'day' } = filters;- const { parseFilters, rawQuery, getDateStringSQL, getDateSQL } = clickhouse;- const { filterQuery, params } = await parseFilters(websiteId, {+ const { parseSessionFilters, rawQuery, getDateStringSQL, getDateSQL } = clickhouse;+ const { filterQuery, params } = await parseSessionFilters(websiteId, {...filters,eventType: EVENT_TYPE.pageView,});