Umami is a privacy-first web and product analytics platform. Traffic, campaigns, behavior, conversions, and revenue in one place. No cookies, no surveillance, self-hosted or in the cloud.
TypeScript analyticsaudience-segmentationchartscohort-analysisgoogle-analyticsproduct-analyticsstatisticsuser-journeyweb-analytics
implement pageviews, events, and channels queries
Francis Cao0a0c1f27c6b7
diff --git a/src/app/api/websites/[websiteId]/metrics/expanded/route.ts b/src/app/api/websites/[websiteId]/metrics/expanded/route.tsindex 395971b65..a327bd0b0 100644--- a/src/app/api/websites/[websiteId]/metrics/expanded/route.ts+++ b/src/app/api/websites/[websiteId]/metrics/expanded/route.ts@@ -4,9 +4,9 @@ import { getQueryFilters, parseRequest } from '@/lib/request';import { badRequest, json, unauthorized } from '@/lib/response';import { dateRangeParams, filterParams, searchParams } from '@/lib/schema';import {- getChannelMetrics,- getEventMetrics,- getPageviewMetrics,+ getChannelExpandedMetrics,+ getEventExpandedMetrics,+ getPageviewExpandedMetrics,getSessionExpandedMetrics,} from '@/queries';import { z } from 'zod';@@ -46,22 +46,6 @@ export async function GET(if (SESSION_COLUMNS.includes(type)) {const data = await getSessionExpandedMetrics(websiteId, { type, limit, offset }, filters);- // if (type === 'language') {- // const combined = {};-- // for (const { x, y } of data) {- // const key = String(x).toLowerCase().split('-')[0];-- // if (combined[key] === undefined) {- // combined[key] = { x: key, y };- // } else {- // combined[key].y += y;- // }- // }-- // return json(Object.values(combined));- // }-return json(data);}@@ -69,16 +53,16 @@ export async function GET(let data;if (type === 'event') {- data = await getEventMetrics(websiteId, { type, limit, offset }, filters);+ data = await getEventExpandedMetrics(websiteId, { type, limit, offset }, filters);} else {- data = await getPageviewMetrics(websiteId, { type, limit, offset }, filters);+ data = await getPageviewExpandedMetrics(websiteId, { type, limit, offset }, filters);}return json(data);}if (type === 'channel') {- const data = await getChannelMetrics(websiteId, filters);+ const data = await getChannelExpandedMetrics(websiteId, { limit, offset }, filters);return json(data);}diff --git a/src/app/api/websites/[websiteId]/metrics/route.ts b/src/app/api/websites/[websiteId]/metrics/route.tsindex 49592a0ed..bc295d790 100644--- a/src/app/api/websites/[websiteId]/metrics/route.ts+++ b/src/app/api/websites/[websiteId]/metrics/route.ts@@ -46,22 +46,6 @@ export async function GET(if (SESSION_COLUMNS.includes(type)) {const data = await getSessionMetrics(websiteId, { type, limit, offset }, filters);- if (type === 'language') {- const combined = {};-- for (const { x, y } of data) {- const key = String(x).toLowerCase().split('-')[0];-- if (combined[key] === undefined) {- combined[key] = { x: key, y };- } else {- combined[key].y += y;- }- }-- return json(Object.values(combined));- }-return json(data);}diff --git a/src/components/hooks/queries/useWebsiteExpandedMetricsQuery.ts b/src/components/hooks/queries/useWebsiteExpandedMetricsQuery.tsindex 0443fd5f6..9718dfe13 100644--- a/src/components/hooks/queries/useWebsiteExpandedMetricsQuery.ts+++ b/src/components/hooks/queries/useWebsiteExpandedMetricsQuery.ts@@ -5,7 +5,7 @@ import { useDateParameters } from '../useDateParameters';import { ReactQueryOptions } from '@/lib/types';export type WebsiteExpandedMetricsData = {- label: string;+ name: string;pageviews: number;visitors: number;visits: number;diff --git a/src/components/metrics/ListExpandedTable.module.css b/src/components/metrics/ListExpandedTable.module.cssnew file mode 100644index 000000000..4bd287c46--- /dev/null+++ b/src/components/metrics/ListExpandedTable.module.css@@ -0,0 +1,7 @@+.truncate {+ white-space: nowrap;+ overflow: hidden;+ text-overflow: ellipsis;+ width: 300px;+ display: block;+}diff --git a/src/components/metrics/ListExpandedTable.tsx b/src/components/metrics/ListExpandedTable.tsxindex fb3efdb54..dc27b17fe 100644--- a/src/components/metrics/ListExpandedTable.tsx+++ b/src/components/metrics/ListExpandedTable.tsx@@ -2,23 +2,25 @@ import { useMessages } from '@/components/hooks';import { formatShortTime } from '@/lib/format';import { DataColumn, DataTable } from '@umami/react-zen';import { ReactNode } from 'react';+import styles from './ListExpandedTable.module.css';export interface ListExpandedTableProps {data?: any[];title?: string;+ type?: string;renderLabel?: (row: any, index: number) => ReactNode;}-export function ListExpandedTable({ data = [], title, renderLabel }: ListExpandedTableProps) {+export function ListExpandedTable({ data = [], title, type, renderLabel }: ListExpandedTableProps) {const { formatMessage, labels } = useMessages();return (<DataTable data={data}>- <DataColumn id="label" label={title} align="start" width="300px">+ <DataColumn id="label" label={title} align="start" className={styles.truncate}>{row =>renderLabel- ? renderLabel({ x: row?.label, country: row?.['country'] }, Number(row.id))- : (row.label ?? formatMessage(labels.unknown))+ ? renderLabel({ x: row?.['name'], country: row?.['country'] }, Number(row.id))+ : (row?.['name'] ?? formatMessage(labels.unknown))}</DataColumn><DataColumn id="visitors" label={formatMessage(labels.visitors)} align="end">@@ -30,18 +32,26 @@ export function ListExpandedTable({ data = [], title, renderLabel }: ListExpande<DataColumn id="pageviews" label={formatMessage(labels.views)} align="end">{row => row?.['pageviews']?.toLocaleString()}</DataColumn>- <DataColumn id="bounceRate" label={formatMessage(labels.bounceRate)} align="end">- {row => {- const n = (Math.min(row?.['visits'], row?.['bounces']) / row?.['visits']) * 100;- return Math.round(+n) + '%';- }}- </DataColumn>- <DataColumn id="visitDuration" label={formatMessage(labels.visitDuration)} align="end">- {row => {- const n = (row?.['totaltime'] / row?.['visits']) * 100;- return `${+n < 0 ? '-' : ''}${formatShortTime(Math.abs(~~n), ['m', 's'], ' ')}`;- }}- </DataColumn>+ {type !== 'exit' && type !== 'entry' ? (+ <DataColumn id="bounceRate" label={formatMessage(labels.bounceRate)} align="end">+ {row => {+ const n = (Math.min(row?.['visits'], row?.['bounces']) / row?.['visits']) * 100;+ return Math.round(+n) + '%';+ }}+ </DataColumn>+ ) : (+ <></>+ )}+ {type !== 'exit' && type !== 'entry' ? (+ <DataColumn id="visitDuration" label={formatMessage(labels.visitDuration)} align="end">+ {row => {+ const n = (row?.['totaltime'] / row?.['visits']) * 100;+ return `${+n < 0 ? '-' : ''}${formatShortTime(Math.abs(~~n), ['m', 's'], ' ')}`;+ }}+ </DataColumn>+ ) : (+ <></>+ )}</DataTable>);}diff --git a/src/components/metrics/MetricsTable.tsx b/src/components/metrics/MetricsTable.tsxindex 760a0da61..fb97f8db2 100644--- a/src/components/metrics/MetricsTable.tsx+++ b/src/components/metrics/MetricsTable.tsx@@ -56,7 +56,6 @@ export function MetricsTable({websiteId,{type,- limit: 30,search: searchFormattedValues ? undefined : search,...params,},@@ -111,6 +110,8 @@ export function MetricsTable({return [];}, [data, dataFilter, search, limit, formatValue, type]);+ const downloadData = expanded ? data : filteredData;+return (<Column gap="3" justifyContent="space-between"><LoadingPanel data={data} isFetching={isFetching} isLoading={isLoading} error={error} gap>@@ -118,12 +119,12 @@ export function MetricsTable({{allowSearch && <SearchField value={search} onSearch={setSearch} delay={300} />}<Row>{children}- {allowDownload && <DownloadButton filename={type} data={filteredData} />}+ {allowDownload && <DownloadButton filename={type} data={downloadData} />}</Row></Row>{data &&(expanded ? (- <ListExpandedTable {...(props as ListExpandedTableProps)} data={data} />+ <ListExpandedTable {...(props as ListExpandedTableProps)} data={data} type={type} />) : (<ListTable {...(props as ListTableProps)} data={filteredData} />))}diff --git a/src/components/metrics/QueryParametersTable.tsx b/src/components/metrics/QueryParametersTable.tsxindex e6d85495b..c33c23cf6 100644--- a/src/components/metrics/QueryParametersTable.tsx+++ b/src/components/metrics/QueryParametersTable.tsx@@ -58,6 +58,7 @@ export function QueryParametersTable({dataFilter={filters[filter]}renderLabel={renderLabel}delay={0}+ expanded={false}>{allowFilter && <FilterButtons items={buttons} value={filter} onChange={setFilter} />}</MetricsTable>diff --git a/src/lib/clickhouse.ts b/src/lib/clickhouse.tsindex bb184d6bd..bcb02d0d0 100644--- a/src/lib/clickhouse.ts+++ b/src/lib/clickhouse.ts@@ -90,7 +90,11 @@ function mapFilter(column: string, operator: string, name: string, type: stringfunction getFilterQuery(filters: Record<string, any>, options: QueryOptions = {}) {const query = filtersToArray(filters, options).reduce((arr, { name, column, operator }) => {if (column) {- arr.push(`and ${mapFilter(column, operator, name)}`);+ if (name === 'eventType') {+ arr.push(`and ${mapFilter(column, operator, name, 'UInt32')}`);+ } else {+ arr.push(`and ${mapFilter(column, operator, name)}`);+ }if (name === 'referrer') {arr.push(`and referrer_domain != hostname`);diff --git a/src/lib/constants.ts b/src/lib/constants.tsindex 1564458d2..16bb71afa 100644--- a/src/lib/constants.ts+++ b/src/lib/constants.ts@@ -35,7 +35,7 @@ export const EVENT_COLUMNS = ['query','event','tag',- 'host',+ 'hostname',];export const SESSION_COLUMNS = [diff --git a/src/queries/index.ts b/src/queries/index.tsindex 52ca65132..707dc8742 100644--- a/src/queries/index.ts+++ b/src/queries/index.ts@@ -11,6 +11,7 @@ export * from '@/queries/sql/events/getEventDataValues';export * from '@/queries/sql/events/getEventDataStats';export * from '@/queries/sql/events/getEventDataUsage';export * from '@/queries/sql/events/getEventMetrics';+export * from '@/queries/sql/events/getEventExpandedMetrics';export * from '@/queries/sql/events/getEventStats';export * from '@/queries/sql/events/getWebsiteEvents';export * from '@/queries/sql/events/getEventUsage';@@ -21,6 +22,7 @@ export * from '@/queries/sql/reports/getRetention';export * from '@/queries/sql/reports/getBreakdown';export * from '@/queries/sql/reports/getUTM';export * from '@/queries/sql/pageviews/getPageviewMetrics';+export * from '@/queries/sql/pageviews/getPageviewExpandedMetrics';export * from '@/queries/sql/pageviews/getPageviewStats';export * from '@/queries/sql/sessions/createSession';export * from '@/queries/sql/sessions/getWebsiteSession';@@ -37,6 +39,7 @@ export * from '@/queries/sql/sessions/getSessionStats';export * from '@/queries/sql/sessions/saveSessionData';export * from '@/queries/sql/getActiveVisitors';export * from '@/queries/sql/getChannelMetrics';+export * from '@/queries/sql/getChannelExpandedMetrics';export * from '@/queries/sql/getRealtimeActivity';export * from '@/queries/sql/getRealtimeData';export * from '@/queries/sql/getValues';diff --git a/src/queries/sql/events/getEventExpandedMetrics.ts b/src/queries/sql/events/getEventExpandedMetrics.tsnew file mode 100644index 000000000..235822b82--- /dev/null+++ b/src/queries/sql/events/getEventExpandedMetrics.ts@@ -0,0 +1,113 @@+import clickhouse from '@/lib/clickhouse';+import { EVENT_TYPE, FILTER_COLUMNS, SESSION_COLUMNS } from '@/lib/constants';+import { CLICKHOUSE, PRISMA, runQuery } from '@/lib/db';+import prisma from '@/lib/prisma';+import { QueryFilters } from '@/lib/types';++export interface EventExpandedMetricParameters {+ type: string;+ limit?: string;+ offset?: string;+}++export interface EventExpandedMetricData {+ name: string;+ pageviews: number;+ visitors: number;+ visits: number;+ bounces: number;+ totaltime: number;+}++export async function getEventExpandedMetrics(+ ...args: [websiteId: string, parameters: EventExpandedMetricParameters, filters: QueryFilters]+): Promise<EventExpandedMetricData[]> {+ return runQuery({+ [PRISMA]: () => relationalQuery(...args),+ [CLICKHOUSE]: () => clickhouseQuery(...args),+ });+}++async function relationalQuery(+ websiteId: string,+ parameters: EventExpandedMetricParameters,+ filters: QueryFilters,+) {+ const { type, limit = 500, offset = 0 } = parameters;+ const column = FILTER_COLUMNS[type] || type;+ const { rawQuery, parseFilters } = prisma;+ const { filterQuery, cohortQuery, joinSessionQuery, queryParams } = parseFilters(+ {+ ...filters,+ websiteId,+ eventType: EVENT_TYPE.customEvent,+ },+ { joinSession: SESSION_COLUMNS.includes(type) },+ );++ return rawQuery(+ `+ select ${column} x,+ count(*) as y+ from website_event+ ${cohortQuery}+ ${joinSessionQuery}+ where website_event.website_id = {{websiteId::uuid}}+ and website_event.created_at between {{startDate}} and {{endDate}}+ and event_type = {{eventType}}+ ${filterQuery}+ group by 1+ order by 2 desc+ limit ${limit}+ offset ${offset}+ `,+ queryParams,+ );+}++async function clickhouseQuery(+ websiteId: string,+ parameters: EventExpandedMetricParameters,+ filters: QueryFilters,+): Promise<EventExpandedMetricData[]> {+ const { type, limit = 500, offset = 0 } = parameters;+ const column = FILTER_COLUMNS[type] || type;+ const { rawQuery, parseFilters } = clickhouse;+ const { filterQuery, cohortQuery, queryParams } = parseFilters({+ ...filters,+ websiteId,+ });++ return rawQuery(+ `+ select+ name,+ sum(t.c) as "pageviews",+ uniq(t.session_id) as "visitors",+ uniq(t.visit_id) as "visits",+ sum(if(t.c = 1, 1, 0)) as "bounces",+ sum(max_time-min_time) as "totaltime"+ from (+ select+ ${column} name,+ session_id,+ visit_id,+ count(*) c,+ min(created_at) min_time,+ max(created_at) max_time+ from website_event+ ${cohortQuery}+ where website_id = {websiteId:UUID}+ and created_at between {startDate:DateTime64} and {endDate:DateTime64}+ and name != ''+ ${filterQuery}+ group by name, session_id, visit_id+ ) as t+ group by name+ order by visitors desc, visits desc+ limit ${limit}+ offset ${offset}+ `,+ { ...queryParams, ...parameters },+ );+}diff --git a/src/queries/sql/events/getEventMetrics.ts b/src/queries/sql/events/getEventMetrics.tsindex 73af22789..3d148edf9 100644--- a/src/queries/sql/events/getEventMetrics.ts+++ b/src/queries/sql/events/getEventMetrics.ts@@ -83,7 +83,6 @@ async function clickhouseQuery(${cohortQuery}where website_id = {websiteId:UUID}and created_at between {startDate:DateTime64} and {endDate:DateTime64}- and event_type = {eventType:UInt32}${filterQuery}group by xorder by y descdiff --git a/src/queries/sql/events/getEventStats.ts b/src/queries/sql/events/getEventStats.tsindex ad6b155d5..8d26dcfc7 100644--- a/src/queries/sql/events/getEventStats.ts+++ b/src/queries/sql/events/getEventStats.ts@@ -72,7 +72,6 @@ async function clickhouseQuery(${cohortQuery}where website_id = {websiteId:UUID}and created_at between {startDate:DateTime64} and {endDate:DateTime64}- and event_type = {eventType:UInt32}${filterQuery}group by x, torder by tdiff --git a/src/queries/sql/getChannelExpandedMetrics.ts b/src/queries/sql/getChannelExpandedMetrics.tsnew file mode 100644index 000000000..d8fce3d27--- /dev/null+++ b/src/queries/sql/getChannelExpandedMetrics.ts@@ -0,0 +1,162 @@+import clickhouse from '@/lib/clickhouse';+import {+ EMAIL_DOMAINS,+ EVENT_TYPE,+ PAID_AD_PARAMS,+ SEARCH_DOMAINS,+ SHOPPING_DOMAINS,+ SOCIAL_DOMAINS,+ VIDEO_DOMAINS,+} from '@/lib/constants';+import { CLICKHOUSE, PRISMA, runQuery } from '@/lib/db';+import prisma from '@/lib/prisma';+import { QueryFilters } from '@/lib/types';++export interface ChannelExpandedMetricsParameters {+ limit?: number | string;+ offset?: number | string;+}++export interface ChannelExpandedMetricsData {+ name: string;+ pageviews: number;+ visitors: number;+ visits: number;+ bounces: number;+ totaltime: number;+}++export async function getChannelExpandedMetrics(+ ...args: [websiteId: string, parameters: ChannelExpandedMetricsParameters, filters?: QueryFilters]+): Promise<ChannelExpandedMetricsData[]> {+ return runQuery({+ [PRISMA]: () => relationalQuery(...args),+ [CLICKHOUSE]: () => clickhouseQuery(...args),+ });+}++async function relationalQuery(+ websiteId: string,+ parameters: ChannelExpandedMetricsParameters,+ filters: QueryFilters,+): Promise<ChannelExpandedMetricsData[]> {+ const { rawQuery, parseFilters } = prisma;+ const { queryParams, filterQuery, cohortQuery, dateQuery } = parseFilters({+ ...filters,+ websiteId,+ eventType: EVENT_TYPE.pageView,+ });++ return rawQuery(+ `+ WITH channels as (+ select case when ${toPostgresPositionClause('utm_medium', ['cp', 'ppc', 'retargeting', 'paid'])} then 'paid' else 'organic' end prefix,+ case+ when referrer_domain = '' and url_query = '' then 'direct'+ when ${toPostgresPositionClause('url_query', PAID_AD_PARAMS)} then 'paidAds'+ when ${toPostgresPositionClause('utm_medium', ['referral', 'app', 'link'])} then 'referral'+ when position(utm_medium, 'affiliate') > 0 then 'affiliate'+ when position(utm_medium, 'sms') > 0 or position(utm_source, 'sms') > 0 then 'sms'+ when ${toPostgresPositionClause('referrer_domain', SEARCH_DOMAINS)} or position(utm_medium, 'organic') > 0 then concat(prefix, 'Search')+ when ${toPostgresPositionClause('referrer_domain', SOCIAL_DOMAINS)} then concat(prefix, 'Social')+ when ${toPostgresPositionClause('referrer_domain', EMAIL_DOMAINS)} or position(utm_medium, 'mail') > 0 then 'email'+ when ${toPostgresPositionClause('referrer_domain', SHOPPING_DOMAINS)} or position(utm_medium, 'shop') > 0 then concat(prefix, 'Shopping')+ when ${toPostgresPositionClause('referrer_domain', VIDEO_DOMAINS)} or position(utm_medium, 'video') > 0 then concat(prefix, 'Video')+ else '' end AS x,+ count(distinct session_id) y+ from website_event+ ${cohortQuery}+ where website_id = {{websiteId::uuid}}+ and event_type = {{eventType}}+ ${dateQuery}+ ${filterQuery}+ group by 1, 2+ order by y desc)++ select x, sum(y) y+ from channels+ where x != ''+ group by x+ order by y desc;+ `,+ { ...queryParams, ...parameters },+ );+}++async function clickhouseQuery(+ websiteId: string,+ parameters: ChannelExpandedMetricsParameters,+ filters: QueryFilters,+): Promise<ChannelExpandedMetricsData[]> {+ const { limit = 500, offset = 0 } = parameters;+ const { rawQuery, parseFilters } = clickhouse;+ const { queryParams, filterQuery, cohortQuery } = parseFilters({+ ...filters,+ websiteId,+ eventType: EVENT_TYPE.pageView,+ });++ return rawQuery(+ `+ select+ name,+ sum(t.c) as "pageviews",+ uniq(t.session_id) as "visitors",+ uniq(t.visit_id) as "visits",+ sum(if(t.c = 1, 1, 0)) as "bounces",+ sum(max_time-min_time) as "totaltime"+ from (+ select case when multiSearchAny(utm_medium, ['cp', 'ppc', 'retargeting', 'paid']) != 0 then 'paid' else 'organic' end prefix,+ case+ when referrer_domain = '' and url_query = '' then 'direct'+ when multiSearchAny(url_query, [${toClickHouseStringArray(+ PAID_AD_PARAMS,+ )}]) != 0 then 'paidAds'+ when multiSearchAny(utm_medium, ['referral', 'app','link']) != 0 then 'referral'+ when position(utm_medium, 'affiliate') > 0 then 'affiliate'+ when position(utm_medium, 'sms') > 0 or position(utm_source, 'sms') > 0 then 'sms'+ when multiSearchAny(referrer_domain, [${toClickHouseStringArray(+ SEARCH_DOMAINS,+ )}]) != 0 or position(utm_medium, 'organic') > 0 then concat(prefix, 'Search')+ when multiSearchAny(referrer_domain, [${toClickHouseStringArray(+ SOCIAL_DOMAINS,+ )}]) != 0 then concat(prefix, 'Social')+ when multiSearchAny(referrer_domain, [${toClickHouseStringArray(+ EMAIL_DOMAINS,+ )}]) != 0 or position(utm_medium, 'mail') > 0 then 'email'+ when multiSearchAny(referrer_domain, [${toClickHouseStringArray(+ SHOPPING_DOMAINS,+ )}]) != 0 or position(utm_medium, 'shop') > 0 then concat(prefix, 'Shopping')+ when multiSearchAny(referrer_domain, [${toClickHouseStringArray(+ VIDEO_DOMAINS,+ )}]) != 0 or position(utm_medium, 'video') > 0 then concat(prefix, 'Video')+ else '' end AS name,+ session_id,+ visit_id,+ count(*) c,+ min(created_at) min_time,+ max(created_at) max_time+ from website_event+ ${cohortQuery}+ where website_id = {websiteId:UUID}+ and created_at between {startDate:DateTime64} and {endDate:DateTime64}+ and name != ''+ ${filterQuery}+ group by prefix, name, session_id, visit_id+ ) as t+ group by name+ order by visitors desc, visits desc+ limit ${limit}+ offset ${offset}+ `,+ { ...queryParams, ...parameters },+ );+}++function toClickHouseStringArray(arr: string[]): string {+ return arr.map(p => `'${p.replace(/'/g, "\\'")}'`).join(', ');+}++function toPostgresPositionClause(column: string, arr: string[]) {+ return arr.map(val => `position(${column}, '${val.replace(/'/g, "''")}') > 0`).join(' OR\n ');+}diff --git a/src/queries/sql/getChannelMetrics.ts b/src/queries/sql/getChannelMetrics.tsindex c240cc862..9e6ba82c4 100644--- a/src/queries/sql/getChannelMetrics.ts+++ b/src/queries/sql/getChannelMetrics.ts@@ -105,7 +105,6 @@ async function clickhouseQuery(from website_event${cohortQuery}where website_id = {websiteId:UUID}- and event_type = {eventType:UInt32}${dateQuery}${filterQuery}group by 1, 2diff --git a/src/queries/sql/getWebsiteStats.ts b/src/queries/sql/getWebsiteStats.tsindex 24242ad65..4fa15a3c0 100644--- a/src/queries/sql/getWebsiteStats.ts+++ b/src/queries/sql/getWebsiteStats.ts@@ -94,7 +94,6 @@ async function clickhouseQuery(${cohortQuery}where website_id = {websiteId:UUID}and created_at between {startDate:DateTime64} and {endDate:DateTime64}- and event_type = {eventType:UInt32}${filterQuery}group by session_id, visit_id) as t;@@ -117,7 +116,6 @@ async function clickhouseQuery(${cohortQuery}where website_id = {websiteId:UUID}and created_at between {startDate:DateTime64} and {endDate:DateTime64}- and event_type = {eventType:UInt32}${filterQuery}group by session_id, visit_id) as t;diff --git a/src/queries/sql/pageviews/getPageviewExpandedMetrics.ts b/src/queries/sql/pageviews/getPageviewExpandedMetrics.tsnew file mode 100644index 000000000..ebe31d7be--- /dev/null+++ b/src/queries/sql/pageviews/getPageviewExpandedMetrics.ts@@ -0,0 +1,166 @@+import clickhouse from '@/lib/clickhouse';+import { EVENT_TYPE, FILTER_COLUMNS, SESSION_COLUMNS } from '@/lib/constants';+import { CLICKHOUSE, PRISMA, runQuery } from '@/lib/db';+import prisma from '@/lib/prisma';+import { QueryFilters } from '@/lib/types';++export interface PageviewExpandedMetricsParameters {+ type: string;+ limit?: number | string;+ offset?: number | string;+}++export interface PageviewExpandedMetricsData {+ name: string;+ pageviews: number;+ visitors: number;+ visits: number;+ bounces: number;+ totaltime: number;+}++export async function getPageviewExpandedMetrics(+ ...args: [websiteId: string, parameters: PageviewExpandedMetricsParameters, filters: QueryFilters]+) {+ return runQuery({+ [PRISMA]: () => relationalQuery(...args),+ [CLICKHOUSE]: () => clickhouseQuery(...args),+ });+}++async function relationalQuery(+ websiteId: string,+ parameters: PageviewExpandedMetricsParameters,+ filters: QueryFilters,+): Promise<PageviewExpandedMetricsData[]> {+ const { type, limit = 500, offset = 0 } = parameters;+ const column = FILTER_COLUMNS[type] || type;+ const { rawQuery, parseFilters } = prisma;+ const { filterQuery, joinSessionQuery, cohortQuery, queryParams } = parseFilters(+ {+ ...filters,+ websiteId,+ eventType: column === 'event_name' ? EVENT_TYPE.customEvent : EVENT_TYPE.pageView,+ },+ { joinSession: SESSION_COLUMNS.includes(type) },+ );++ let entryExitQuery = '';+ let excludeDomain = '';++ if (column === 'referrer_domain') {+ excludeDomain = `and website_event.referrer_domain != website_event.hostname+ and website_event.referrer_domain != ''`;+ }++ if (type === 'entry' || type === 'exit') {+ const aggregrate = type === 'entry' ? 'min' : 'max';++ entryExitQuery = `+ join (+ select visit_id,+ ${aggregrate}(created_at) target_created_at+ from website_event+ where website_event.website_id = {{websiteId::uuid}}+ and website_event.created_at between {{startDate}} and {{endDate}}+ and event_type = {{eventType}}+ group by visit_id+ ) x+ on x.visit_id = website_event.visit_id+ and x.target_created_at = website_event.created_at+ `;+ }++ return rawQuery(+ `+ select ${column} x,+ count(distinct website_event.session_id) as y+ from website_event+ ${joinSessionQuery}+ ${cohortQuery}+ ${entryExitQuery}+ where website_event.website_id = {{websiteId::uuid}}+ and website_event.created_at between {{startDate}} and {{endDate}}+ and event_type = {{eventType}}+ ${excludeDomain}+ ${filterQuery}+ group by 1+ order by 2 desc+ limit ${limit}+ offset ${offset}+ `,+ queryParams,+ );+}++async function clickhouseQuery(+ websiteId: string,+ parameters: PageviewExpandedMetricsParameters,+ filters: QueryFilters,+): Promise<{ x: string; y: number }[]> {+ const { type, limit = 500, offset = 0 } = parameters;+ const column = FILTER_COLUMNS[type] || type;+ const { rawQuery, parseFilters } = clickhouse;+ const { filterQuery, cohortQuery, queryParams } = parseFilters({+ ...filters,+ websiteId,+ eventType: column === 'event_name' ? EVENT_TYPE.customEvent : EVENT_TYPE.pageView,+ });++ let excludeDomain = '';+ let entryExitQuery = '';++ if (column === 'referrer_domain') {+ excludeDomain = `and referrer_domain != hostname and referrer_domain != ''`;+ }++ if (type === 'entry' || type === 'exit') {+ const aggregrate = type === 'entry' ? 'min' : 'max';++ entryExitQuery = `+ JOIN (select visit_id,+ ${aggregrate}(created_at) target_created_at+ from website_event+ where website_id = {websiteId:UUID}+ and created_at between {startDate:DateTime64} and {endDate:DateTime64}+ and event_type = {eventType:UInt32}+ group by visit_id) x+ ON x.visit_id = website_event.visit_id+ and x.target_created_at = website_event.created_at`;+ }++ return rawQuery(+ `+ select+ name,+ sum(t.c) as "pageviews",+ uniq(t.session_id) as "visitors",+ uniq(t.visit_id) as "visits",+ sum(if(t.c = 1, 1, 0)) as "bounces",+ sum(max_time-min_time) as "totaltime"+ from (+ select+ ${column} name,+ session_id,+ visit_id,+ count(*) c,+ min(created_at) min_time,+ max(created_at) max_time+ from website_event+ ${cohortQuery}+ ${entryExitQuery}+ where website_id = {websiteId:UUID}+ and created_at between {startDate:DateTime64} and {endDate:DateTime64}+ and name != ''+ ${excludeDomain}+ ${filterQuery}+ group by name, session_id, visit_id+ ) as t+ group by name+ order by visitors desc, visits desc+ limit ${limit}+ offset ${offset}+ `,+ { ...queryParams, ...parameters },+ );+}diff --git a/src/queries/sql/pageviews/getPageviewMetrics.ts b/src/queries/sql/pageviews/getPageviewMetrics.tsindex fd4856854..4ebd4cd9e 100644--- a/src/queries/sql/pageviews/getPageviewMetrics.ts+++ b/src/queries/sql/pageviews/getPageviewMetrics.ts@@ -136,7 +136,6 @@ async function clickhouseQuery(${entryExitQuery}where website_id = {websiteId:UUID}and created_at between {startDate:DateTime64} and {endDate:DateTime64}- and event_type = {eventType:UInt32}${excludeDomain}${filterQuery}group by x@@ -174,7 +173,6 @@ async function clickhouseQuery(${cohortQuery}where website_id = {websiteId:UUID}and created_at between {startDate:DateTime64} and {endDate:DateTime64}- and event_type = {eventType:UInt32}${excludeDomain}${filterQuery}${groupByQuery}) as gdiff --git a/src/queries/sql/pageviews/getPageviewStats.ts b/src/queries/sql/pageviews/getPageviewStats.tsindex 704dea1e5..e9e763e87 100644--- a/src/queries/sql/pageviews/getPageviewStats.ts+++ b/src/queries/sql/pageviews/getPageviewStats.ts@@ -66,7 +66,6 @@ async function clickhouseQuery(${cohortQuery}where website_id = {websiteId:UUID}and created_at between {startDate:DateTime64} and {endDate:DateTime64}- and event_type = {eventType:UInt32}${filterQuery}group by t) as g@@ -85,7 +84,6 @@ async function clickhouseQuery(${cohortQuery}where website_id = {websiteId:UUID}and created_at between {startDate:DateTime64} and {endDate:DateTime64}- and event_type = {eventType:UInt32}${filterQuery}group by t) as gdiff --git a/src/queries/sql/reports/getAttribution.ts b/src/queries/sql/reports/getAttribution.tsindex df452343f..0bfbd58b4 100644--- a/src/queries/sql/reports/getAttribution.ts+++ b/src/queries/sql/reports/getAttribution.ts@@ -477,7 +477,6 @@ async function clickhouseQuery(where website_id = {websiteId:UUID}and created_at between {startDate:DateTime64} and {endDate:DateTime64}and ${column} = {step:String}- and event_type = {eventType:UInt32}${filterQuery}`,queryParams,diff --git a/src/queries/sql/reports/getBreakdown.ts b/src/queries/sql/reports/getBreakdown.tsindex a39c49eff..7cae67a1a 100644--- a/src/queries/sql/reports/getBreakdown.ts+++ b/src/queries/sql/reports/getBreakdown.ts@@ -115,7 +115,6 @@ async function clickhouseQuery(${cohortQuery}where website_id = {websiteId:UUID}and created_at between {startDate:DateTime64} and {endDate:DateTime64}- and event_type = {eventType:UInt32}${filterQuery}group by ${parseFieldsByName(fields)},session_id, visit_iddiff --git a/src/queries/sql/sessions/getSessionExpandedMetrics.ts b/src/queries/sql/sessions/getSessionExpandedMetrics.tsindex ba0841dc8..7c6d98f6f 100644--- a/src/queries/sql/sessions/getSessionExpandedMetrics.ts+++ b/src/queries/sql/sessions/getSessionExpandedMetrics.ts@@ -11,7 +11,7 @@ export interface SessionExpandedMetricsParameters {}export interface SessionExpandedMetricsData {- label: string;+ name: string;pageviews: number;visitors: number;visits: number;@@ -34,7 +34,7 @@ async function relationalQuery(filters: QueryFilters,): Promise<SessionExpandedMetricsData[]> {const { type, limit = 500, offset = 0 } = parameters;- const column = FILTER_COLUMNS[type] || type;+ let column = FILTER_COLUMNS[type] || type;const { parseFilters, rawQuery } = prisma;const { filterQuery, joinSessionQuery, cohortQuery, queryParams } = parseFilters({@@ -48,6 +48,10 @@ async function relationalQuery();const includeCountry = column === 'city' || column === 'region';+ if (type === 'language') {+ column = `lower(left(${type}, 2))`;+ }+return rawQuery(`select@@ -77,7 +81,7 @@ async function clickhouseQuery(filters: QueryFilters,): Promise<SessionExpandedMetricsData[]> {const { type, limit = 500, offset = 0 } = parameters;- const column = FILTER_COLUMNS[type] || type;+ let column = FILTER_COLUMNS[type] || type;const { parseFilters, rawQuery } = clickhouse;const { filterQuery, cohortQuery, queryParams } = parseFilters({...filters,@@ -86,10 +90,14 @@ async function clickhouseQuery(});const includeCountry = column === 'city' || column === 'region';+ if (type === 'language') {+ column = `lower(left(${type}, 2))`;+ }+return rawQuery(`select- label,+ name,${includeCountry ? 'country,' : ''}sum(t.c) as "pageviews",uniq(t.session_id) as "visitors",@@ -98,7 +106,7 @@ async function clickhouseQuery(sum(max_time-min_time) as "totaltime"from (select- ${column} label,+ ${column} name,${includeCountry ? 'country,' : ''}session_id,visit_id,@@ -109,13 +117,12 @@ async function clickhouseQuery(${cohortQuery}where website_id = {websiteId:UUID}and created_at between {startDate:DateTime64} and {endDate:DateTime64}- and event_type = {eventType:UInt32}- and label != ''+ and name != ''${filterQuery}- group by label, session_id, visit_id+ group by name, session_id, visit_id${includeCountry ? ', country' : ''}) as t- group by label+ group by name${includeCountry ? ', country' : ''}order by visitors desc, visits desclimit ${limit}diff --git a/src/queries/sql/sessions/getSessionMetrics.ts b/src/queries/sql/sessions/getSessionMetrics.tsindex 559623102..725c73793 100644--- a/src/queries/sql/sessions/getSessionMetrics.ts+++ b/src/queries/sql/sessions/getSessionMetrics.ts@@ -25,7 +25,7 @@ async function relationalQuery(filters: QueryFilters,) {const { type, limit = 500, offset = 0 } = parameters;- const column = FILTER_COLUMNS[type] || type;+ let column = FILTER_COLUMNS[type] || type;const { parseFilters, rawQuery } = prisma;const { filterQuery, joinSessionQuery, cohortQuery, queryParams } = parseFilters({@@ -39,6 +39,10 @@ async function relationalQuery();const includeCountry = column === 'city' || column === 'region';+ if (type === 'language') {+ column = `lower(left(${type}, 2))`;+ }+return rawQuery(`select@@ -68,7 +72,7 @@ async function clickhouseQuery(filters: QueryFilters,): Promise<{ x: string; y: number }[]> {const { type, limit = 500, offset = 0 } = parameters;- const column = FILTER_COLUMNS[type] || type;+ let column = FILTER_COLUMNS[type] || type;const { parseFilters, rawQuery } = clickhouse;const { filterQuery, cohortQuery, queryParams } = parseFilters({...filters,@@ -77,6 +81,10 @@ async function clickhouseQuery(});const includeCountry = column === 'city' || column === 'region';+ if (type === 'language') {+ column = `lower(left(${type}, 2))`;+ }+let sql = '';if (EVENT_COLUMNS.some(item => Object.keys(filters).includes(item))) {@@ -89,7 +97,6 @@ async function clickhouseQuery(${cohortQuery}where website_id = {websiteId:UUID}and created_at between {startDate:DateTime64} and {endDate:DateTime64}- and event_type = {eventType:UInt32}${filterQuery}group by x${includeCountry ? ', country' : ''}@@ -107,7 +114,6 @@ async function clickhouseQuery(${cohortQuery}where website_id = {websiteId:UUID}and created_at between {startDate:DateTime64} and {endDate:DateTime64}- and event_type = {eventType:UInt32}${filterQuery}group by x${includeCountry ? ', country' : ''}diff --git a/src/queries/sql/sessions/getSessionStats.ts b/src/queries/sql/sessions/getSessionStats.tsindex 258de2906..97a8755ed 100644--- a/src/queries/sql/sessions/getSessionStats.ts+++ b/src/queries/sql/sessions/getSessionStats.ts@@ -66,7 +66,6 @@ async function clickhouseQuery(${cohortQuery}where website_id = {websiteId:UUID}and created_at between {startDate:DateTime64} and {endDate:DateTime64}- and event_type = {eventType:UInt32}${filterQuery}group by t) as g@@ -84,7 +83,6 @@ async function clickhouseQuery(from website_event_stats_hourly as website_eventwhere website_id = {websiteId:UUID}and created_at between {startDate:DateTime64} and {endDate:DateTime64}- and event_type = {eventType:UInt32}${filterQuery}group by t) as g