Skip to content

armgit / umami

Public Imported from GitHub
0

Umami is a privacy-first web and product analytics platform. Traffic, campaigns, behavior, conversions, and revenue in one place. No cookies, no surveillance, self-hosted or in the cloud.

remove parse session filters

Francis Caod02dc6f99785
diff --git a/src/lib/clickhouse.ts b/src/lib/clickhouse.ts
index 4c544a71d..c25eff4e5 100644
--- a/src/lib/clickhouse.ts
+++ b/src/lib/clickhouse.ts
@@ -2,11 +2,11 @@ import { ClickHouseClient, createClient } from '@clickhouse/client';
import dateFormat from 'dateformat';
import debug from 'debug';
import { CLICKHOUSE } from 'lib/db';
-import { PageParams, QueryFilters, QueryOptions } from './types';
-import { EVENT_COLUMNS, DEFAULT_PAGE_SIZE, OPERATORS } from './constants';
-import { fetchWebsite } from './load';
+import { DEFAULT_PAGE_SIZE, OPERATORS } from './constants';
import { maxDate } from './date';
+import { fetchWebsite } from './load';
import { filtersToArray } from './params';
+import { PageParams, QueryFilters, QueryOptions } from './types';
export const CLICKHOUSE_DATE_FORMATS = {
second: '%Y-%m-%dT%H:%i:%S',
@@ -100,26 +100,6 @@ function getFilterQuery(filters: QueryFilters = {}, options: QueryOptions = {})
return query.join('\n');
}
-function getSessionFilterQuery(filters: QueryFilters = {}, options: QueryOptions = {}) {
- const query = filtersToArray(filters, options).reduce((arr, { name, column, operator }) => {
- if (column) {
- if (EVENT_COLUMNS.includes(name)) {
- arr.push(`and has(${column}, {${name}:String})`);
-
- if (name === 'referrer') {
- arr.push('and not has(referrer_domain, {websiteDomain:String})');
- }
- } else {
- arr.push(`and ${mapFilter(column, operator, name)}`);
- }
- }
-
- return arr;
- }, []);
-
- return query.join('\n');
-}
-
function getDateQuery(filters: QueryFilters = {}) {
const { startDate, endDate } = filters;
@@ -159,25 +139,6 @@ async function parseFilters(websiteId: string, filters: QueryFilters = {}, optio
};
}
-async function parseSessionFilters(
- websiteId: string,
- filters: QueryFilters = {},
- options?: QueryOptions,
-) {
- const website = await fetchWebsite(websiteId);
-
- return {
- filterQuery: getSessionFilterQuery(filters, options),
- dateQuery: getDateQuery(filters),
- params: {
- ...getFilterParams(filters),
- websiteId,
- startDate: maxDate(filters.startDate, new Date(website?.resetAt)),
- websiteDomain: website.domain,
- },
- };
-}
-
async function pagedQuery(
query: string,
queryParams: { [key: string]: any },
@@ -260,7 +221,6 @@ export default {
getDateFormat,
getFilterQuery,
parseFilters,
- parseSessionFilters,
pagedQuery,
findUnique,
findFirst,
diff --git a/src/queries/analytics/getWebsiteStats.ts b/src/queries/analytics/getWebsiteStats.ts
index 09eebb919..c5141d3be 100644
--- a/src/queries/analytics/getWebsiteStats.ts
+++ b/src/queries/analytics/getWebsiteStats.ts
@@ -74,8 +74,8 @@ async function clickhouseQuery(
sql = `
select
sum(t.c) as "pageviews",
- count(distinct t.session_id) as "visitors",
- count(distinct t.visit_id) as "visits",
+ uniq(t.session_id) as "visitors",
+ uniq(t.visit_id) as "visits",
sum(if(t.c = 1, 1, 0)) as "bounces",
sum(max_time-min_time) as "totaltime"
from (
@@ -96,16 +96,24 @@ async function clickhouseQuery(
} else {
sql = `
select
- sum(views) as "pageviews",
+ sum(t.c) as "pageviews",
uniq(session_id) as "visitors",
uniq(visit_id) as "visits",
- sumIf(1, views = 1) as "bounces",
+ sumIf(1, t.c = 1) as "bounces",
sum(max_time-min_time) as "totaltime"
- from website_event_stats_hourly "website_event"
+ from (select
+ session_id,
+ visit_id,
+ sum(views) c,
+ min(min_time) min_time,
+ max(max_time) max_time
+ from umami.website_event_stats_hourly "website_event"
where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime64} and {endDate:DateTime64}
and event_type = {eventType:UInt32}
- ${filterQuery};
+ ${filterQuery}
+ group by session_id, visit_id
+ ) as t;
`;
}
diff --git a/src/queries/analytics/sessions/getSessionMetrics.ts b/src/queries/analytics/sessions/getSessionMetrics.ts
index 3e6f53c0a..bb8bc4c5c 100644
--- a/src/queries/analytics/sessions/getSessionMetrics.ts
+++ b/src/queries/analytics/sessions/getSessionMetrics.ts
@@ -1,5 +1,5 @@
import clickhouse from 'lib/clickhouse';
-import { EVENT_TYPE, FILTER_COLUMNS, SESSION_COLUMNS } from 'lib/constants';
+import { EVENT_COLUMNS, EVENT_TYPE, FILTER_COLUMNS, SESSION_COLUMNS } from 'lib/constants';
import { CLICKHOUSE, PRISMA, runQuery } from 'lib/db';
import prisma from 'lib/prisma';
import { QueryFilters } from 'lib/types';
@@ -64,15 +64,34 @@ async function clickhouseQuery(
offset: number = 0,
): Promise<{ x: string; y: number }[]> {
const column = FILTER_COLUMNS[type] || type;
- const { parseSessionFilters, rawQuery } = clickhouse;
- const { filterQuery, params } = await parseSessionFilters(websiteId, {
+ const { parseFilters, rawQuery } = clickhouse;
+ const { filterQuery, params } = await parseFilters(websiteId, {
...filters,
eventType: EVENT_TYPE.pageView,
});
const includeCountry = column === 'city' || column === 'subdivision1';
- return rawQuery(
- `
+ let sql = '';
+
+ if (EVENT_COLUMNS.some(item => Object.keys(filters).includes(item))) {
+ sql = `
+ select
+ ${column} x,
+ count(distinct session_id) y
+ ${includeCountry ? ', country' : ''}
+ from website_event
+ where website_id = {websiteId:UUID}
+ and created_at between {startDate:DateTime64} and {endDate:DateTime64}
+ and event_type = {eventType:UInt32}
+ ${filterQuery}
+ group by x
+ ${includeCountry ? ', country' : ''}
+ order by y desc
+ limit ${limit}
+ offset ${offset}
+ `;
+ } else {
+ sql = `
select
${column} x,
uniq(session_id) y
@@ -87,9 +106,10 @@ async function clickhouseQuery(
order by y desc
limit ${limit}
offset ${offset}
- `,
- params,
- ).then(a => {
+ `;
+ }
+
+ return rawQuery(sql, params).then(a => {
return Object.values(a).map(a => {
return { x: a.x, y: Number(a.y), country: a.country };
});
diff --git a/src/queries/analytics/sessions/getSessionStats.ts b/src/queries/analytics/sessions/getSessionStats.ts
index fa0ed6a96..fa7483333 100644
--- a/src/queries/analytics/sessions/getSessionStats.ts
+++ b/src/queries/analytics/sessions/getSessionStats.ts
@@ -1,7 +1,7 @@
import clickhouse from 'lib/clickhouse';
+import { EVENT_COLUMNS, EVENT_TYPE } from 'lib/constants';
import { CLICKHOUSE, PRISMA, runQuery } from 'lib/db';
import prisma from 'lib/prisma';
-import { EVENT_TYPE } from 'lib/constants';
import { QueryFilters } from 'lib/types';
export async function getSessionStats(...args: [websiteId: string, filters: QueryFilters]) {
@@ -41,25 +41,24 @@ async function clickhouseQuery(
filters: QueryFilters,
): Promise<{ x: string; y: number }[]> {
const { timezone = 'UTC', unit = 'day' } = filters;
- const { parseSessionFilters, rawQuery, getDateStringSQL, getDateSQL } = clickhouse;
- const { filterQuery, params } = await parseSessionFilters(websiteId, {
+ const { parseFilters, rawQuery, getDateStringSQL, getDateSQL } = clickhouse;
+ const { filterQuery, params } = await parseFilters(websiteId, {
...filters,
eventType: EVENT_TYPE.pageView,
});
- const table = unit === 'minute' ? 'website_event' : 'website_event_stats_hourly';
- const columnQuery = unit === 'minute' ? 'count(distinct session_id)' : 'uniq(session_id)';
+ let sql = '';
- return rawQuery(
- `
+ if (EVENT_COLUMNS.some(item => Object.keys(filters).includes(item)) || unit === 'minute') {
+ sql = `
select
${getDateStringSQL('g.t', unit)} as x,
g.y as y
from (
select
${getDateSQL('created_at', unit, timezone)} as t,
- ${columnQuery} as y
- from ${table} website_event
+ count(distinct session_id) as y
+ from website_event
where website_id = {websiteId:UUID}
and created_at between {startDate:DateTime64} and {endDate:DateTime64}
and event_type = {eventType:UInt32}
@@ -67,9 +66,28 @@ async function clickhouseQuery(
group by t
) as g
order by t
- `,
- params,
- ).then(result => {
+ `;
+ } else {
+ sql = `
+ select
+ ${getDateStringSQL('g.t', unit)} as x,
+ g.y as y
+ from (
+ select
+ ${getDateSQL('created_at', unit, timezone)} as t,
+ uniq(session_id) as y
+ from website_event_stats_hourly website_event
+ where website_id = {websiteId:UUID}
+ and created_at between {startDate:DateTime64} and {endDate:DateTime64}
+ and event_type = {eventType:UInt32}
+ ${filterQuery}
+ group by t
+ ) as g
+ order by t
+ `;
+ }
+
+ return rawQuery(sql, params).then(result => {
return Object.values(result).map((a: any) => {
return { x: a.x, y: Number(a.y) };
});