Umami is a privacy-first web and product analytics platform. Traffic, campaigns, behavior, conversions, and revenue in one place. No cookies, no surveillance, self-hosted or in the cloud.
TypeScript analyticsaudience-segmentationchartscohort-analysisgoogle-analyticsproduct-analyticsstatisticsuser-journeyweb-analytics
remove parse session filters
Francis Caod02dc6f99785
diff --git a/src/lib/clickhouse.ts b/src/lib/clickhouse.tsindex 4c544a71d..c25eff4e5 100644--- a/src/lib/clickhouse.ts+++ b/src/lib/clickhouse.ts@@ -2,11 +2,11 @@ import { ClickHouseClient, createClient } from '@clickhouse/client';import dateFormat from 'dateformat';import debug from 'debug';import { CLICKHOUSE } from 'lib/db';-import { PageParams, QueryFilters, QueryOptions } from './types';-import { EVENT_COLUMNS, DEFAULT_PAGE_SIZE, OPERATORS } from './constants';-import { fetchWebsite } from './load';+import { DEFAULT_PAGE_SIZE, OPERATORS } from './constants';import { maxDate } from './date';+import { fetchWebsite } from './load';import { filtersToArray } from './params';+import { PageParams, QueryFilters, QueryOptions } from './types';export const CLICKHOUSE_DATE_FORMATS = {second: '%Y-%m-%dT%H:%i:%S',@@ -100,26 +100,6 @@ function getFilterQuery(filters: QueryFilters = {}, options: QueryOptions = {})return query.join('\n');}-function getSessionFilterQuery(filters: QueryFilters = {}, options: QueryOptions = {}) {- const query = filtersToArray(filters, options).reduce((arr, { name, column, operator }) => {- if (column) {- if (EVENT_COLUMNS.includes(name)) {- arr.push(`and has(${column}, {${name}:String})`);-- if (name === 'referrer') {- arr.push('and not has(referrer_domain, {websiteDomain:String})');- }- } else {- arr.push(`and ${mapFilter(column, operator, name)}`);- }- }-- return arr;- }, []);-- return query.join('\n');-}-function getDateQuery(filters: QueryFilters = {}) {const { startDate, endDate } = filters;@@ -159,25 +139,6 @@ async function parseFilters(websiteId: string, filters: QueryFilters = {}, optio};}-async function parseSessionFilters(- websiteId: string,- filters: QueryFilters = {},- options?: QueryOptions,-) {- const website = await fetchWebsite(websiteId);-- return {- filterQuery: getSessionFilterQuery(filters, options),- dateQuery: getDateQuery(filters),- params: {- ...getFilterParams(filters),- websiteId,- startDate: maxDate(filters.startDate, new Date(website?.resetAt)),- websiteDomain: website.domain,- },- };-}-async function pagedQuery(query: string,queryParams: { [key: string]: any },@@ -260,7 +221,6 @@ export default {getDateFormat,getFilterQuery,parseFilters,- parseSessionFilters,pagedQuery,findUnique,findFirst,diff --git a/src/queries/analytics/getWebsiteStats.ts b/src/queries/analytics/getWebsiteStats.tsindex 09eebb919..c5141d3be 100644--- a/src/queries/analytics/getWebsiteStats.ts+++ b/src/queries/analytics/getWebsiteStats.ts@@ -74,8 +74,8 @@ async function clickhouseQuery(sql = `selectsum(t.c) as "pageviews",- count(distinct t.session_id) as "visitors",- count(distinct t.visit_id) as "visits",+ uniq(t.session_id) as "visitors",+ uniq(t.visit_id) as "visits",sum(if(t.c = 1, 1, 0)) as "bounces",sum(max_time-min_time) as "totaltime"from (@@ -96,16 +96,24 @@ async function clickhouseQuery(} else {sql = `select- sum(views) as "pageviews",+ sum(t.c) as "pageviews",uniq(session_id) as "visitors",uniq(visit_id) as "visits",- sumIf(1, views = 1) as "bounces",+ sumIf(1, t.c = 1) as "bounces",sum(max_time-min_time) as "totaltime"- from website_event_stats_hourly "website_event"+ from (select+ session_id,+ visit_id,+ sum(views) c,+ min(min_time) min_time,+ max(max_time) max_time+ from umami.website_event_stats_hourly "website_event"where website_id = {websiteId:UUID}and created_at between {startDate:DateTime64} and {endDate:DateTime64}and event_type = {eventType:UInt32}- ${filterQuery};+ ${filterQuery}+ group by session_id, visit_id+ ) as t;`;}diff --git a/src/queries/analytics/sessions/getSessionMetrics.ts b/src/queries/analytics/sessions/getSessionMetrics.tsindex 3e6f53c0a..bb8bc4c5c 100644--- a/src/queries/analytics/sessions/getSessionMetrics.ts+++ b/src/queries/analytics/sessions/getSessionMetrics.ts@@ -1,5 +1,5 @@import clickhouse from 'lib/clickhouse';-import { EVENT_TYPE, FILTER_COLUMNS, SESSION_COLUMNS } from 'lib/constants';+import { EVENT_COLUMNS, EVENT_TYPE, FILTER_COLUMNS, SESSION_COLUMNS } from 'lib/constants';import { CLICKHOUSE, PRISMA, runQuery } from 'lib/db';import prisma from 'lib/prisma';import { QueryFilters } from 'lib/types';@@ -64,15 +64,34 @@ async function clickhouseQuery(offset: number = 0,): Promise<{ x: string; y: number }[]> {const column = FILTER_COLUMNS[type] || type;- const { parseSessionFilters, rawQuery } = clickhouse;- const { filterQuery, params } = await parseSessionFilters(websiteId, {+ const { parseFilters, rawQuery } = clickhouse;+ const { filterQuery, params } = await parseFilters(websiteId, {...filters,eventType: EVENT_TYPE.pageView,});const includeCountry = column === 'city' || column === 'subdivision1';- return rawQuery(- `+ let sql = '';++ if (EVENT_COLUMNS.some(item => Object.keys(filters).includes(item))) {+ sql = `+ select+ ${column} x,+ count(distinct session_id) y+ ${includeCountry ? ', country' : ''}+ from website_event+ where website_id = {websiteId:UUID}+ and created_at between {startDate:DateTime64} and {endDate:DateTime64}+ and event_type = {eventType:UInt32}+ ${filterQuery}+ group by x+ ${includeCountry ? ', country' : ''}+ order by y desc+ limit ${limit}+ offset ${offset}+ `;+ } else {+ sql = `select${column} x,uniq(session_id) y@@ -87,9 +106,10 @@ async function clickhouseQuery(order by y desclimit ${limit}offset ${offset}- `,- params,- ).then(a => {+ `;+ }++ return rawQuery(sql, params).then(a => {return Object.values(a).map(a => {return { x: a.x, y: Number(a.y), country: a.country };});diff --git a/src/queries/analytics/sessions/getSessionStats.ts b/src/queries/analytics/sessions/getSessionStats.tsindex fa0ed6a96..fa7483333 100644--- a/src/queries/analytics/sessions/getSessionStats.ts+++ b/src/queries/analytics/sessions/getSessionStats.ts@@ -1,7 +1,7 @@import clickhouse from 'lib/clickhouse';+import { EVENT_COLUMNS, EVENT_TYPE } from 'lib/constants';import { CLICKHOUSE, PRISMA, runQuery } from 'lib/db';import prisma from 'lib/prisma';-import { EVENT_TYPE } from 'lib/constants';import { QueryFilters } from 'lib/types';export async function getSessionStats(...args: [websiteId: string, filters: QueryFilters]) {@@ -41,25 +41,24 @@ async function clickhouseQuery(filters: QueryFilters,): Promise<{ x: string; y: number }[]> {const { timezone = 'UTC', unit = 'day' } = filters;- const { parseSessionFilters, rawQuery, getDateStringSQL, getDateSQL } = clickhouse;- const { filterQuery, params } = await parseSessionFilters(websiteId, {+ const { parseFilters, rawQuery, getDateStringSQL, getDateSQL } = clickhouse;+ const { filterQuery, params } = await parseFilters(websiteId, {...filters,eventType: EVENT_TYPE.pageView,});- const table = unit === 'minute' ? 'website_event' : 'website_event_stats_hourly';- const columnQuery = unit === 'minute' ? 'count(distinct session_id)' : 'uniq(session_id)';+ let sql = '';- return rawQuery(- `+ if (EVENT_COLUMNS.some(item => Object.keys(filters).includes(item)) || unit === 'minute') {+ sql = `select${getDateStringSQL('g.t', unit)} as x,g.y as yfrom (select${getDateSQL('created_at', unit, timezone)} as t,- ${columnQuery} as y- from ${table} website_event+ count(distinct session_id) as y+ from website_eventwhere website_id = {websiteId:UUID}and created_at between {startDate:DateTime64} and {endDate:DateTime64}and event_type = {eventType:UInt32}@@ -67,9 +66,28 @@ async function clickhouseQuery(group by t) as gorder by t- `,- params,- ).then(result => {+ `;+ } else {+ sql = `+ select+ ${getDateStringSQL('g.t', unit)} as x,+ g.y as y+ from (+ select+ ${getDateSQL('created_at', unit, timezone)} as t,+ uniq(session_id) as y+ from website_event_stats_hourly website_event+ where website_id = {websiteId:UUID}+ and created_at between {startDate:DateTime64} and {endDate:DateTime64}+ and event_type = {eventType:UInt32}+ ${filterQuery}+ group by t+ ) as g+ order by t+ `;+ }++ return rawQuery(sql, params).then(result => {return Object.values(result).map((a: any) => {return { x: a.x, y: Number(a.y) };});