Umami is a privacy-first web and product analytics platform. Traffic, campaigns, behavior, conversions, and revenue in one place. No cookies, no surveillance, self-hosted or in the cloud.
TypeScript analyticsaudience-segmentationchartscohort-analysisgoogle-analyticsproduct-analyticsstatisticsuser-journeyweb-analytics
finish expanded queries and ui.
Francis Cao38f251ead51f
diff --git a/db/clickhouse/schema.sql b/db/clickhouse/schema.sqlindex 41b3a1b3e..be277743e 100644--- a/db/clickhouse/schema.sql+++ b/db/clickhouse/schema.sql@@ -57,7 +57,7 @@ CREATE TABLE umami.event_dataevent_name String,data_key String,string_value Nullable(String),- number_value Nullable(Decimal64(4)),+ number_value Nullable(Decimal64(22, 4)),date_value Nullable(DateTime('UTC')),data_type UInt32,created_at DateTime('UTC'),@@ -73,7 +73,7 @@ CREATE TABLE umami.session_datasession_id UUID,data_key String,string_value Nullable(String),- number_value Nullable(Decimal64(4)),+ number_value Nullable(Decimal64(22, 4)),date_value Nullable(DateTime('UTC')),data_type UInt32,distinct_id String,diff --git a/src/app/api/reports/utm/route.ts b/src/app/api/reports/utm/route.tsindex 17c5c1c67..150773f79 100644--- a/src/app/api/reports/utm/route.ts+++ b/src/app/api/reports/utm/route.ts@@ -17,8 +17,8 @@ export async function POST(request: Request) {return unauthorized();}- const parameters = await setWebsiteDate(websiteId, body.parameters);const filters = await getQueryFilters(body.filters, websiteId);+ const parameters = await setWebsiteDate(websiteId, body.parameters);const data = await getUTM(websiteId, parameters as UTMParameters, filters);diff --git a/src/app/api/websites/[websiteId]/sessions/route.ts b/src/app/api/websites/[websiteId]/sessions/route.tsindex bcc4b8bcc..5f81e14c8 100644--- a/src/app/api/websites/[websiteId]/sessions/route.ts+++ b/src/app/api/websites/[websiteId]/sessions/route.ts@@ -2,7 +2,7 @@ import { z } from 'zod';import { getQueryFilters, parseRequest } from '@/lib/request';import { unauthorized, json } from '@/lib/response';import { canViewWebsite } from '@/lib/auth';-import { dateRangeParams, filterParams, pagingParams } from '@/lib/schema';+import { dateRangeParams, filterParams, pagingParams, searchParams } from '@/lib/schema';import { getWebsiteSessions } from '@/queries';export async function GET(@@ -13,6 +13,7 @@ export async function GET(...dateRangeParams,...filterParams,...pagingParams,+ ...searchParams,});const { auth, query, error } = await parseRequest(request, schema);diff --git a/src/components/metrics/ListExpandedTable.tsx b/src/components/metrics/ListExpandedTable.tsxindex dc27b17fe..822852373 100644--- a/src/components/metrics/ListExpandedTable.tsx+++ b/src/components/metrics/ListExpandedTable.tsx@@ -7,11 +7,10 @@ import styles from './ListExpandedTable.module.css';export interface ListExpandedTableProps {data?: any[];title?: string;- type?: string;renderLabel?: (row: any, index: number) => ReactNode;}-export function ListExpandedTable({ data = [], title, type, renderLabel }: ListExpandedTableProps) {+export function ListExpandedTable({ data = [], title, renderLabel }: ListExpandedTableProps) {const { formatMessage, labels } = useMessages();return (@@ -32,26 +31,18 @@ export function ListExpandedTable({ data = [], title, type, renderLabel }: ListE<DataColumn id="pageviews" label={formatMessage(labels.views)} align="end">{row => row?.['pageviews']?.toLocaleString()}</DataColumn>- {type !== 'exit' && type !== 'entry' ? (- <DataColumn id="bounceRate" label={formatMessage(labels.bounceRate)} align="end">- {row => {- const n = (Math.min(row?.['visits'], row?.['bounces']) / row?.['visits']) * 100;- return Math.round(+n) + '%';- }}- </DataColumn>- ) : (- <></>- )}- {type !== 'exit' && type !== 'entry' ? (- <DataColumn id="visitDuration" label={formatMessage(labels.visitDuration)} align="end">- {row => {- const n = (row?.['totaltime'] / row?.['visits']) * 100;- return `${+n < 0 ? '-' : ''}${formatShortTime(Math.abs(~~n), ['m', 's'], ' ')}`;- }}- </DataColumn>- ) : (- <></>- )}+ <DataColumn id="bounceRate" label={formatMessage(labels.bounceRate)} align="end">+ {row => {+ const n = (Math.min(row?.['visits'], row?.['bounces']) / row?.['visits']) * 100;+ return Math.round(+n) + '%';+ }}+ </DataColumn>+ <DataColumn id="visitDuration" label={formatMessage(labels.visitDuration)} align="end">+ {row => {+ const n = (row?.['totaltime'] / row?.['visits']) * 100;+ return `${+n < 0 ? '-' : ''}${formatShortTime(Math.abs(~~n), ['m', 's'], ' ')}`;+ }}+ </DataColumn></DataTable>);}diff --git a/src/components/metrics/MetricsTable.tsx b/src/components/metrics/MetricsTable.tsxindex fb97f8db2..c18e46e44 100644--- a/src/components/metrics/MetricsTable.tsx+++ b/src/components/metrics/MetricsTable.tsx@@ -114,7 +114,7 @@ export function MetricsTable({return (<Column gap="3" justifyContent="space-between">- <LoadingPanel data={data} isFetching={isFetching} isLoading={isLoading} error={error} gap>+ <LoadingPanel isFetching={isFetching} isLoading={isLoading} error={error} gap><Row alignItems="center" justifyContent="space-between">{allowSearch && <SearchField value={search} onSearch={setSearch} delay={300} />}<Row>@@ -124,7 +124,7 @@ export function MetricsTable({</Row>{data &&(expanded ? (- <ListExpandedTable {...(props as ListExpandedTableProps)} data={data} type={type} />+ <ListExpandedTable {...(props as ListExpandedTableProps)} data={data} />) : (<ListTable {...(props as ListTableProps)} data={filteredData} />))}diff --git a/src/components/metrics/ReferrersTable.tsx b/src/components/metrics/ReferrersTable.tsxindex 0b3f1ee30..718f13ee7 100644--- a/src/components/metrics/ReferrersTable.tsx+++ b/src/components/metrics/ReferrersTable.tsx@@ -1,11 +1,11 @@-import { Row } from '@umami/react-zen';-import { FilterLink } from '@/components/common/FilterLink';import { Favicon } from '@/components/common/Favicon';import { FilterButtons } from '@/components/common/FilterButtons';+import { FilterLink } from '@/components/common/FilterLink';import { useMessages, useNavigation } from '@/components/hooks';-import { MetricsTable, MetricsTableProps } from './MetricsTable';-import thenby from 'thenby';import { GROUPED_DOMAINS } from '@/lib/constants';+import { emptyFilter } from '@/lib/filters';+import { Row } from '@umami/react-zen';+import { MetricsTable, MetricsTableProps } from './MetricsTable';export interface ReferrersTableProps extends MetricsTableProps {allowFilter?: boolean;@@ -36,7 +36,7 @@ export function ReferrersTable({ allowFilter, ...props }: ReferrersTableProps) {const renderLink = ({ x: referrer }) => {if (view === 'grouped') {- if (referrer === '_other') {+ if (referrer === 'Other') {return `(${formatMessage(labels.other)})`;} else {return (@@ -60,38 +60,13 @@ export function ReferrersTable({ allowFilter, ...props }: ReferrersTableProps) {);};- const getDomain = (x: string) => {- for (const { domain, match } of GROUPED_DOMAINS) {- if (Array.isArray(match) ? match.some(str => x.includes(str)) : x.includes(match)) {- return domain;- }- }- return '_other';- };-- const groupedFilter = (data: any[]) => {- const groups = { _other: 0 };-- for (const { x, y } of data) {- const domain = getDomain(x);- if (!groups[domain]) {- groups[domain] = 0;- }- groups[domain] += +y;- }-- return Object.keys(groups)- .map((key: any) => ({ x: key, y: groups[key] }))- .sort(thenby.firstBy('y', -1));- };-return (<MetricsTable{...props}title={formatMessage(labels.referrers)}- type="referrer"+ type={view}metric={formatMessage(labels.visitors)}- dataFilter={view === 'grouped' ? groupedFilter : undefined}+ dataFilter={emptyFilter}renderLabel={renderLink}>{allowFilter && <FilterButtons items={buttons} value={view} onChange={handleSelect} />}diff --git a/src/lib/constants.ts b/src/lib/constants.tsindex 16bb71afa..18bda293f 100644--- a/src/lib/constants.ts+++ b/src/lib/constants.ts@@ -26,11 +26,13 @@ export const REALTIME_RANGE = 30;export const REALTIME_INTERVAL = 10000;export const UNIT_TYPES = ['year', 'month', 'hour', 'day', 'minute'];+export const EVENT_COLUMNS = ['path','entry','exit','referrer',+ 'grouped','title','query','event',@@ -59,6 +61,7 @@ export const FILTER_COLUMNS = {entry: 'url_path',exit: 'url_path',referrer: 'referrer_domain',+ grouped: 'referrer_domain',hostname: 'hostname',title: 'page_title',query: 'url_query',@@ -386,6 +389,9 @@ export const GROUPED_DOMAINS = [{ name: 'Snapchat', domain: 'snapchat.com', match: 'snapchat.' },{ name: 'Pinterest', domain: 'pinterest.com', match: 'pinterest.' },{ name: 'ChatGPT', domain: 'chatgpt.com', match: 'chatgpt.' },+ { name: 'Yahoo', domain: 'yahoo.com', match: 'yahoo.' },+ { name: 'Yandex', domain: 'yandex.ru', match: 'yandex.' },+ { name: 'Baidu', domain: 'baidu.com', match: 'baidu.' },];export const MAP_FILE = '/datamaps.world.json';diff --git a/src/queries/sql/events/getWebsiteEvents.ts b/src/queries/sql/events/getWebsiteEvents.tsindex 3b620018a..579593d66 100644--- a/src/queries/sql/events/getWebsiteEvents.ts+++ b/src/queries/sql/events/getWebsiteEvents.ts@@ -19,7 +19,7 @@ async function relationalQuery(websiteId: string, filters: QueryFilters) {search: `%${search}%`,});- const searchQuery = filters.search+ const searchQuery = search? `and ((event_name ilike {{search}} and event_type = 2)or (url_path ilike {{search}} and event_type = 1))`: '';@@ -59,12 +59,13 @@ async function relationalQuery(websiteId: string, filters: QueryFilters) {async function clickhouseQuery(websiteId: string, filters: QueryFilters) {const { pagedRawQuery, parseFilters } = clickhouse;+ const { search } = filters;const { queryParams, dateQuery, cohortQuery, filterQuery } = parseFilters({...filters,websiteId,});- const searchQuery = filters.search+ const searchQuery = search? `and ((positionCaseInsensitive(event_name, {search:String}) > 0 and event_type = 2)or (positionCaseInsensitive(url_path, {search:String}) > 0 and event_type = 1))`: '';diff --git a/src/queries/sql/pageviews/getPageviewExpandedMetrics.ts b/src/queries/sql/pageviews/getPageviewExpandedMetrics.tsindex ebe31d7be..ce1f1df9f 100644--- a/src/queries/sql/pageviews/getPageviewExpandedMetrics.ts+++ b/src/queries/sql/pageviews/getPageviewExpandedMetrics.ts@@ -1,5 +1,5 @@import clickhouse from '@/lib/clickhouse';-import { EVENT_TYPE, FILTER_COLUMNS, SESSION_COLUMNS } from '@/lib/constants';+import { EVENT_TYPE, FILTER_COLUMNS, GROUPED_DOMAINS, SESSION_COLUMNS } from '@/lib/constants';import { CLICKHOUSE, PRISMA, runQuery } from '@/lib/db';import prisma from '@/lib/prisma';import { QueryFilters } from '@/lib/types';@@ -99,7 +99,7 @@ async function clickhouseQuery(filters: QueryFilters,): Promise<{ x: string; y: number }[]> {const { type, limit = 500, offset = 0 } = parameters;- const column = FILTER_COLUMNS[type] || type;+ let column = FILTER_COLUMNS[type] || type;const { rawQuery, parseFilters } = clickhouse;const { filterQuery, cohortQuery, queryParams } = parseFilters({...filters,@@ -112,21 +112,24 @@ async function clickhouseQuery(if (column === 'referrer_domain') {excludeDomain = `and referrer_domain != hostname and referrer_domain != ''`;+ if (type === 'grouped') {+ column = toClickHouseGroupedReferrer(GROUPED_DOMAINS);+ }}if (type === 'entry' || type === 'exit') {- const aggregrate = type === 'entry' ? 'min' : 'max';+ const aggregrate = type === 'entry' ? 'argMin' : 'argMax';+ column = `x.${column}`;entryExitQuery = `JOIN (select visit_id,- ${aggregrate}(created_at) target_created_at+ ${aggregrate}(url_path, created_at) url_pathfrom website_eventwhere website_id = {websiteId:UUID}and created_at between {startDate:DateTime64} and {endDate:DateTime64}and event_type = {eventType:UInt32}group by visit_id) x- ON x.visit_id = website_event.visit_id- and x.target_created_at = website_event.created_at`;+ ON x.visit_id = website_event.visit_id`;}return rawQuery(@@ -164,3 +167,19 @@ async function clickhouseQuery({ ...queryParams, ...parameters },);}++export function toClickHouseGroupedReferrer(+ domains: any[],+ column: string = 'referrer_domain',+): string {+ return [+ 'CASE',+ ...domains.map(group => {+ const matches = Array.isArray(group.match) ? group.match : [group.match];+ const formattedArray = matches.map(m => `'${m}'`).join(', ');+ return ` WHEN multiSearchAny(${column}, [${formattedArray}]) != 0 THEN '${group.domain}'`;+ }),+ " ELSE 'Other'",+ 'END',+ ].join('\n');+}diff --git a/src/queries/sql/pageviews/getPageviewMetrics.ts b/src/queries/sql/pageviews/getPageviewMetrics.tsindex 4ebd4cd9e..2403a8e8f 100644--- a/src/queries/sql/pageviews/getPageviewMetrics.ts+++ b/src/queries/sql/pageviews/getPageviewMetrics.ts@@ -30,7 +30,7 @@ async function relationalQuery(filters: QueryFilters,): Promise<PageviewMetricsData[]> {const { type, limit = 500, offset = 0 } = parameters;- const column = FILTER_COLUMNS[type] || type;+ let column = FILTER_COLUMNS[type] || type;const { rawQuery, parseFilters } = prisma;const { filterQuery, joinSessionQuery, cohortQuery, queryParams } = parseFilters({@@ -50,20 +50,21 @@ async function relationalQuery(}if (type === 'entry' || type === 'exit') {- const aggregrate = type === 'entry' ? 'min' : 'max';+ const order = type === 'entry' ? 'asc' : 'desc';+ column = `x.${column}`;entryExitQuery = `join (- select visit_id,- ${aggregrate}(created_at) target_created_at+ select distinct on (visit_id)+ visit_id,+ url_pathfrom website_eventwhere website_event.website_id = {{websiteId::uuid}}and website_event.created_at between {{startDate}} and {{endDate}}and event_type = {{eventType}}- group by visit_id+ order by visit_id, created_at ${order}) xon x.visit_id = website_event.visit_id- and x.target_created_at = website_event.created_at`;}@@ -95,7 +96,7 @@ async function clickhouseQuery(filters: QueryFilters,): Promise<{ x: string; y: number }[]> {const { type, limit = 500, offset = 0 } = parameters;- const column = FILTER_COLUMNS[type] || type;+ let column = FILTER_COLUMNS[type] || type;const { rawQuery, parseFilters } = clickhouse;const { filterQuery, cohortQuery, queryParams } = parseFilters({...filters,@@ -114,18 +115,18 @@ async function clickhouseQuery(}if (type === 'entry' || type === 'exit') {- const aggregrate = type === 'entry' ? 'min' : 'max';+ const aggregrate = type === 'entry' ? 'argMin' : 'argMax';+ column = `x.${column}`;entryExitQuery = `JOIN (select visit_id,- ${aggregrate}(created_at) target_created_at+ ${aggregrate}(url_path, created_at) url_pathfrom website_eventwhere website_id = {websiteId:UUID}and created_at between {startDate:DateTime64} and {endDate:DateTime64}and event_type = {eventType:UInt32}group by visit_id) x- ON x.visit_id = website_event.visit_id- and x.target_created_at = website_event.created_at`;+ ON x.visit_id = website_event.visit_id`;}sql = `diff --git a/src/queries/sql/reports/getUTM.ts b/src/queries/sql/reports/getUTM.tsindex f96c62d3c..f690be1c8 100644--- a/src/queries/sql/reports/getUTM.ts+++ b/src/queries/sql/reports/getUTM.ts@@ -1,16 +1,15 @@import clickhouse from '@/lib/clickhouse';import { CLICKHOUSE, PRISMA, runQuery } from '@/lib/db';import prisma from '@/lib/prisma';+import { QueryFilters } from '@/lib/types';++export interface UTMParameters {+ startDate: Date;+ endDate: Date;+}export async function getUTM(- ...args: [- websiteId: string,- filters: {- startDate: Date;- endDate: Date;- timezone?: string;- },- ]+ ...args: [websiteId: string, parameters: UTMParameters, filters: QueryFilters]) {return runQuery({[PRISMA]: () => relationalQuery(...args),@@ -20,14 +19,18 @@ export async function getUTM(async function relationalQuery(websiteId: string,- filters: {- startDate: Date;- endDate: Date;- timezone?: string;- },+ parameters: UTMParameters,+ filters: QueryFilters,) {- const { startDate, endDate } = filters;- const { rawQuery } = prisma;+ const { startDate, endDate } = parameters;+ const { parseFilters, rawQuery } = prisma;++ const { filterQuery, queryParams } = parseFilters({+ ...filters,+ websiteId,+ startDate,+ endDate,+ });return rawQuery(`@@ -37,26 +40,26 @@ async function relationalQuery(and created_at between {{startDate}} and {{endDate}}and coalesce(url_query, '') != ''and event_type = 1+ ${filterQuery}group by 1`,- {- websiteId,- startDate,- endDate,- },+ queryParams,);}async function clickhouseQuery(websiteId: string,- filters: {- startDate: Date;- endDate: Date;- timezone?: string;- },+ parameters: UTMParameters,+ filters: QueryFilters,) {- const { startDate, endDate } = filters;- const { rawQuery } = clickhouse;+ const { startDate, endDate } = parameters;+ const { parseFilters, rawQuery } = clickhouse;+ const { filterQuery, queryParams } = parseFilters({+ ...filters,+ websiteId,+ startDate,+ endDate,+ });return rawQuery(`@@ -66,12 +69,9 @@ async function clickhouseQuery(and created_at between {startDate:DateTime64} and {endDate:DateTime64}and url_query != ''and event_type = 1+ ${filterQuery}group by 1`,- {- websiteId,- startDate,- endDate,- },+ queryParams,);}diff --git a/src/queries/sql/sessions/getWebsiteSessions.ts b/src/queries/sql/sessions/getWebsiteSessions.tsindex fdc1bf9cd..c0942b207 100644--- a/src/queries/sql/sessions/getWebsiteSessions.ts+++ b/src/queries/sql/sessions/getWebsiteSessions.ts@@ -20,7 +20,13 @@ async function relationalQuery(websiteId: string, filters: QueryFilters) {search: search ? `%${search}%` : undefined,});- const searchQuery = search ? `and session.distinct_id ilike {{search}}` : '';+ const searchQuery = search+ ? `and (distinct_id ilike {{search}}+ or city ilike {{search}}+ or browser ilike {{search}}+ or os ilike {{search}}+ or device ilike {{search}})`+ : '';return pagedRawQuery(`@@ -74,7 +80,13 @@ async function clickhouseQuery(websiteId: string, filters: QueryFilters) {websiteId,});- const searchQuery = search ? `and positionCaseInsensitive(distinct_id, {search:String}) > 0` : '';+ const searchQuery = search+ ? `and ((positionCaseInsensitive(distinct_id, {search:String}) > 0)+ or (positionCaseInsensitive(city, {search:String}) > 0)+ or (positionCaseInsensitive(browser, {search:String}) > 0)+ or (positionCaseInsensitive(os, {search:String}) > 0)+ or (positionCaseInsensitive(device, {search:String}) > 0))`+ : '';let sql = '';