Skip to content

armgit / umami

Public Imported from GitHub
0

Umami is a privacy-first web and product analytics platform. Traffic, campaigns, behavior, conversions, and revenue in one place. No cookies, no surveillance, self-hosted or in the cloud.

route dashboard queries based on filters selected

Francis Cao9882ff24f6fe
diff --git a/src/lib/clickhouse.ts b/src/lib/clickhouse.ts
index 2bda9bfa8..4c544a71d 100644
--- a/src/lib/clickhouse.ts
+++ b/src/lib/clickhouse.ts
@@ -3,7 +3,7 @@ import dateFormat from 'dateformat';
import debug from 'debug';
import { CLICKHOUSE } from 'lib/db';
import { PageParams, QueryFilters, QueryOptions } from './types';
-import { DEFAULT_PAGE_SIZE, OPERATORS } from './constants';
+import { EVENT_COLUMNS, DEFAULT_PAGE_SIZE, OPERATORS } from './constants';
import { fetchWebsite } from './load';
import { maxDate } from './date';
import { filtersToArray } from './params';
@@ -100,6 +100,26 @@ function getFilterQuery(filters: QueryFilters = {}, options: QueryOptions = {})
return query.join('\n');
}
+function getSessionFilterQuery(filters: QueryFilters = {}, options: QueryOptions = {}) {
+ const query = filtersToArray(filters, options).reduce((arr, { name, column, operator }) => {
+ if (column) {
+ if (EVENT_COLUMNS.includes(name)) {
+ arr.push(`and has(${column}, {${name}:String})`);
+
+ if (name === 'referrer') {
+ arr.push('and not has(referrer_domain, {websiteDomain:String})');
+ }
+ } else {
+ arr.push(`and ${mapFilter(column, operator, name)}`);
+ }
+ }
+
+ return arr;
+ }, []);
+
+ return query.join('\n');
+}
+
function getDateQuery(filters: QueryFilters = {}) {
const { startDate, endDate } = filters;
@@ -139,6 +159,25 @@ async function parseFilters(websiteId: string, filters: QueryFilters = {}, optio
};
}
+async function parseSessionFilters(
+ websiteId: string,
+ filters: QueryFilters = {},
+ options?: QueryOptions,
+) {
+ const website = await fetchWebsite(websiteId);
+
+ return {
+ filterQuery: getSessionFilterQuery(filters, options),
+ dateQuery: getDateQuery(filters),
+ params: {
+ ...getFilterParams(filters),
+ websiteId,
+ startDate: maxDate(filters.startDate, new Date(website?.resetAt)),
+ websiteDomain: website.domain,
+ },
+ };
+}
+
async function pagedQuery(
query: string,
queryParams: { [key: string]: any },
@@ -221,6 +260,7 @@ export default {
getDateFormat,
getFilterQuery,
parseFilters,
+ parseSessionFilters,
pagedQuery,
findUnique,
findFirst,
diff --git a/src/lib/constants.ts b/src/lib/constants.ts
index aa1b3c0f2..9d696f114 100644
--- a/src/lib/constants.ts
+++ b/src/lib/constants.ts
@@ -33,16 +33,7 @@ export const FILTER_REFERRERS = 'filter-referrers';
export const FILTER_PAGES = 'filter-pages';
export const UNIT_TYPES = ['year', 'month', 'hour', 'day', 'minute'];
-export const EVENT_COLUMNS = [
- 'url',
- 'entry',
- 'exit',
- 'referrer',
- 'title',
- 'query',
- 'event',
- 'host',
-];
+export const EVENT_COLUMNS = ['url', 'entry', 'exit', 'referrer', 'title', 'query', 'event'];
export const SESSION_COLUMNS = [
'browser',
@@ -58,8 +49,8 @@ export const SESSION_COLUMNS = [
export const FILTER_COLUMNS = {
url: 'url_path',
- entry: 'entry_url',
- exit: 'exit_url',
+ entry: 'url_path',
+ exit: 'url_path',
referrer: 'referrer_domain',
host: 'hostname',
title: 'page_title',
diff --git a/src/queries/analytics/events/getEventMetrics.ts b/src/queries/analytics/events/getEventMetrics.ts
index 362321358..504cea119 100644
--- a/src/queries/analytics/events/getEventMetrics.ts
+++ b/src/queries/analytics/events/getEventMetrics.ts
@@ -1,8 +1,8 @@
-import prisma from 'lib/prisma';
import clickhouse from 'lib/clickhouse';
-import { runQuery, CLICKHOUSE, PRISMA } from 'lib/db';
-import { WebsiteEventMetric, QueryFilters } from 'lib/types';
import { EVENT_TYPE } from 'lib/constants';
+import { CLICKHOUSE, PRISMA, runQuery } from 'lib/db';
+import prisma from 'lib/prisma';
+import { QueryFilters, WebsiteEventMetric } from 'lib/types';
export async function getEventMetrics(
...args: [websiteId: string, filters: QueryFilters]
@@ -51,8 +51,24 @@ async function clickhouseQuery(
eventType: EVENT_TYPE.customEvent,
});
- return rawQuery(
- `
+ let sql = '';
+
+ if (filterQuery) {
+ sql = `
+ select
+ event_name x,
+ ${getDateSQL('created_at', unit, timezone)} t,
+ count(*) y
+ from website_event
+ where website_id = {websiteId:UUID}
+ and created_at between {startDate:DateTime64} and {endDate:DateTime64}
+ and event_type = {eventType:UInt32}
+ ${filterQuery}
+ group by x, t
+ order by t
+ `;
+ } else {
+ sql = `
select
event_name x,
${getDateSQL('created_at', unit, timezone)} t,
@@ -64,13 +80,13 @@ async function clickhouseQuery(
where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime64} and {endDate:DateTime64}
and event_type = {eventType:UInt32}
- ${filterQuery}
) as g
group by x, t
order by t
- `,
- params,
- ).then(a => {
+ `;
+ }
+
+ return rawQuery(sql, params).then(a => {
return Object.values(a).map(a => {
return { x: a.x, t: a.t, y: Number(a.y) };
});
diff --git a/src/queries/analytics/getWebsiteStats.ts b/src/queries/analytics/getWebsiteStats.ts
index 7d8a76f69..09eebb919 100644
--- a/src/queries/analytics/getWebsiteStats.ts
+++ b/src/queries/analytics/getWebsiteStats.ts
@@ -3,6 +3,7 @@ import { EVENT_TYPE } from 'lib/constants';
import { CLICKHOUSE, PRISMA, runQuery } from 'lib/db';
import prisma from 'lib/prisma';
import { QueryFilters } from 'lib/types';
+import { EVENT_COLUMNS } from 'lib/constants';
export async function getWebsiteStats(
...args: [websiteId: string, filters: QueryFilters]
@@ -67,30 +68,33 @@ async function clickhouseQuery(
eventType: EVENT_TYPE.pageView,
});
- return rawQuery(
- // `
- // select
- // sum(t.c) as "pageviews",
- // count(distinct t.session_id) as "visitors",
- // count(distinct t.visit_id) as "visits",
- // sum(if(t.c = 1, 1, 0)) as "bounces",
- // sum(max_time-min_time) as "totaltime"
- // from (
- // select
- // session_id,
- // visit_id,
- // count(*) c,
- // min(created_at) min_time,
- // max(created_at) max_time
- // from website_event
- // where website_id = {websiteId:UUID}
- // and created_at between {startDate:DateTime64} and {endDate:DateTime64}
- // and event_type = {eventType:UInt32}
- // ${filterQuery}
- // group by session_id, visit_id
- // ) as t;
- // `,
- `
+ let sql = '';
+
+ if (EVENT_COLUMNS.some(item => Object.keys(filters).includes(item))) {
+ sql = `
+ select
+ sum(t.c) as "pageviews",
+ count(distinct t.session_id) as "visitors",
+ count(distinct t.visit_id) as "visits",
+ sum(if(t.c = 1, 1, 0)) as "bounces",
+ sum(max_time-min_time) as "totaltime"
+ from (
+ select
+ session_id,
+ visit_id,
+ count(*) c,
+ min(created_at) min_time,
+ max(created_at) max_time
+ from website_event
+ where website_id = {websiteId:UUID}
+ and created_at between {startDate:DateTime64} and {endDate:DateTime64}
+ and event_type = {eventType:UInt32}
+ ${filterQuery}
+ group by session_id, visit_id
+ ) as t;
+ `;
+ } else {
+ sql = `
select
sum(views) as "pageviews",
uniq(session_id) as "visitors",
@@ -102,9 +106,10 @@ async function clickhouseQuery(
and created_at between {startDate:DateTime64} and {endDate:DateTime64}
and event_type = {eventType:UInt32}
${filterQuery};
- `,
- params,
- ).then(result => {
+ `;
+ }
+
+ return rawQuery(sql, params).then(result => {
return Object.values(result).map((a: any) => {
return {
pageviews: Number(a.pageviews),
diff --git a/src/queries/analytics/pageviews/getPageviewMetrics.ts b/src/queries/analytics/pageviews/getPageviewMetrics.ts
index 4ac101b2c..f734b1ddc 100644
--- a/src/queries/analytics/pageviews/getPageviewMetrics.ts
+++ b/src/queries/analytics/pageviews/getPageviewMetrics.ts
@@ -1,5 +1,5 @@
import clickhouse from 'lib/clickhouse';
-import { EVENT_TYPE, FILTER_COLUMNS, SESSION_COLUMNS } from 'lib/constants';
+import { EVENT_COLUMNS, EVENT_TYPE, FILTER_COLUMNS, SESSION_COLUMNS } from 'lib/constants';
import { CLICKHOUSE, PRISMA, runQuery } from 'lib/db';
import prisma from 'lib/prisma';
import { QueryFilters } from 'lib/types';
@@ -91,46 +91,66 @@ async function clickhouseQuery(
});
let excludeDomain = '';
- let groupByQuery = '';
+ let sql = '';
- if (column === 'referrer_domain') {
- excludeDomain = `and t != {websiteDomain:String} and t != ''`;
- }
+ if (EVENT_COLUMNS.some(item => Object.keys(filters).includes(item))) {
+ let entryExitQuery = '';
- let columnQuery = `arrayJoin(${column})`;
+ if (column === 'referrer_domain') {
+ excludeDomain = `and referrer_domain != {websiteDomain:String} and referrer_domain != ''`;
+ }
- if (type === 'entry') {
- columnQuery = `visit_id x, argMinMerge(${column})`;
- }
+ if (type === 'entry' || type === 'exit') {
+ const aggregrate = type === 'entry' ? 'min' : 'max';
- if (type === 'exit') {
- columnQuery = `visit_id x, argMaxMerge(${column})`;
- }
+ entryExitQuery = `
+ JOIN (select visit_id,
+ ${aggregrate}(created_at) target_created_at
+ from website_event
+ where website_id = {websiteId:UUID}
+ and created_at between {startDate:DateTime64} and {endDate:DateTime64}
+ and event_type = {eventType:UInt32}
+ group by visit_id) x
+ ON x.visit_id = website_event.visit_id
+ and x.target_created_at = website_event.created_at`;
+ }
- if (type === 'entry' || type === 'exit') {
- groupByQuery = 'group by x';
- }
+ sql = `
+ select ${column} x, count(*) y
+ from website_event
+ ${entryExitQuery}
+ where website_id = {websiteId:UUID}
+ and created_at between {startDate:DateTime64} and {endDate:DateTime64}
+ and event_type = {eventType:UInt32}
+ ${excludeDomain}
+ ${filterQuery}
+ group by x
+ order by y desc
+ limit ${limit}
+ offset ${offset}
+ `;
+ } else {
+ let groupByQuery = '';
- // let excludeDomain = '';
- // if (column === 'referrer_domain') {
- // excludeDomain = `and referrer_domain != {websiteDomain:String} and referrer_domain != ''`;
- // }
+ if (column === 'referrer_domain') {
+ excludeDomain = `and t != {websiteDomain:String} and t != ''`;
+ }
- return rawQuery(
- // `
- // select ${column} x, count(*) y
- // from website_event
- // where website_id = {websiteId:UUID}
- // and created_at between {startDate:DateTime64} and {endDate:DateTime64}
- // and event_type = {eventType:UInt32}
- // ${excludeDomain}
- // ${filterQuery}
- // group by x
- // order by y desc
- // limit ${limit}
- // offset ${offset}
- // `,
- `
+ let columnQuery = `arrayJoin(${column})`;
+
+ if (type === 'entry') {
+ columnQuery = `visit_id x, argMinMerge(entry_url)`;
+ }
+
+ if (type === 'exit') {
+ columnQuery = `visit_id x, argMaxMerge(exit_url)`;
+ }
+
+ if (type === 'entry' || type === 'exit') {
+ groupByQuery = 'group by x';
+ }
+
+ sql = `
select g.t as x,
count(*) as y
from (
@@ -146,9 +166,10 @@ async function clickhouseQuery(
order by y desc
limit ${limit}
offset ${offset}
- `,
- params,
- ).then((result: any) => {
+ `;
+ }
+
+ return rawQuery(sql, params).then((result: any) => {
return Object.values(result).map((a: any) => {
return { x: a.x, y: Number(a.y) };
});
diff --git a/src/queries/analytics/pageviews/getPageviewStats.ts b/src/queries/analytics/pageviews/getPageviewStats.ts
index c378d48a6..1d027e794 100644
--- a/src/queries/analytics/pageviews/getPageviewStats.ts
+++ b/src/queries/analytics/pageviews/getPageviewStats.ts
@@ -1,7 +1,7 @@
import clickhouse from 'lib/clickhouse';
import { CLICKHOUSE, PRISMA, runQuery } from 'lib/db';
import prisma from 'lib/prisma';
-import { EVENT_TYPE } from 'lib/constants';
+import { EVENT_COLUMNS, EVENT_TYPE } from 'lib/constants';
import { QueryFilters } from 'lib/types';
export async function getPageviewStats(...args: [websiteId: string, filters: QueryFilters]) {
@@ -47,36 +47,18 @@ async function clickhouseQuery(
eventType: EVENT_TYPE.pageView,
});
- const table = unit === 'minute' ? 'website_event' : 'website_event_stats_hourly';
- const columnQuery = unit === 'minute' ? 'count(*)' : 'sum(views)';
+ let sql = '';
- return rawQuery(
- // `
- // select
- // ${getDateStringSQL('g.t', unit)} as x,
- // g.y as y
- // from (
- // select
- // ${getDateSQL('created_at', unit, timezone)} as t,
- // count(*) as y
- // from website_event
- // where website_id = {websiteId:UUID}
- // and created_at between {startDate:DateTime64} and {endDate:DateTime64}
- // and event_type = {eventType:UInt32}
- // ${filterQuery}
- // group by t
- // ) as g
- // order by t
- // `,
- `
+ if (EVENT_COLUMNS.some(item => Object.keys(filters).includes(item)) || unit === 'minute') {
+ sql = `
select
${getDateStringSQL('g.t', unit)} as x,
g.y as y
from (
select
${getDateSQL('created_at', unit, timezone)} as t,
- ${columnQuery} as y
- from ${table} website_event
+ count(*) as y
+ from website_event
where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime64} and {endDate:DateTime64}
and event_type = {eventType:UInt32}
@@ -84,9 +66,28 @@ async function clickhouseQuery(
group by t
) as g
order by t
- `,
- params,
- ).then(result => {
+ `;
+ } else {
+ sql = `
+ select
+ ${getDateStringSQL('g.t', unit)} as x,
+ g.y as y
+ from (
+ select
+ ${getDateSQL('created_at', unit, timezone)} as t,
+ sum(views)as y
+ from website_event_stats_hourly website_event
+ where website_id = {websiteId:UUID}
+ and created_at between {startDate:DateTime64} and {endDate:DateTime64}
+ and event_type = {eventType:UInt32}
+ ${filterQuery}
+ group by t
+ ) as g
+ order by t
+ `;
+ }
+
+ return rawQuery(sql, params).then(result => {
return Object.values(result).map((a: any) => {
return { x: a.x, y: Number(a.y) };
});
diff --git a/src/queries/analytics/sessions/getSessionMetrics.ts b/src/queries/analytics/sessions/getSessionMetrics.ts
index 9baf2a5c0..3e6f53c0a 100644
--- a/src/queries/analytics/sessions/getSessionMetrics.ts
+++ b/src/queries/analytics/sessions/getSessionMetrics.ts
@@ -64,8 +64,8 @@ async function clickhouseQuery(
offset: number = 0,
): Promise<{ x: string; y: number }[]> {
const column = FILTER_COLUMNS[type] || type;
- const { parseFilters, rawQuery } = clickhouse;
- const { filterQuery, params } = await parseFilters(websiteId, {
+ const { parseSessionFilters, rawQuery } = clickhouse;
+ const { filterQuery, params } = await parseSessionFilters(websiteId, {
...filters,
eventType: EVENT_TYPE.pageView,
});
diff --git a/src/queries/analytics/sessions/getSessionStats.ts b/src/queries/analytics/sessions/getSessionStats.ts
index 7bba14df1..fa0ed6a96 100644
--- a/src/queries/analytics/sessions/getSessionStats.ts
+++ b/src/queries/analytics/sessions/getSessionStats.ts
@@ -41,8 +41,8 @@ async function clickhouseQuery(
filters: QueryFilters,
): Promise<{ x: string; y: number }[]> {
const { timezone = 'UTC', unit = 'day' } = filters;
- const { parseFilters, rawQuery, getDateStringSQL, getDateSQL } = clickhouse;
- const { filterQuery, params } = await parseFilters(websiteId, {
+ const { parseSessionFilters, rawQuery, getDateStringSQL, getDateSQL } = clickhouse;
+ const { filterQuery, params } = await parseSessionFilters(websiteId, {
...filters,
eventType: EVENT_TYPE.pageView,
});