Umami is a privacy-first web and product analytics platform. Traffic, campaigns, behavior, conversions, and revenue in one place. No cookies, no surveillance, self-hosted or in the cloud.
TypeScript analyticsaudience-segmentationchartscohort-analysisgoogle-analyticsproduct-analyticsstatisticsuser-journeyweb-analytics
Add MAX_PAGING_RESULTS on hooks for performance improvements.
Francis Caofbac7518fc35
diff --git a/src/components/common/DataGrid.tsx b/src/components/common/DataGrid.tsxindex 993929fd2..8d18321cf 100644--- a/src/components/common/DataGrid.tsx+++ b/src/components/common/DataGrid.tsx@@ -41,7 +41,7 @@ export function DataGrid({const { data, error, isLoading, isFetching } = query;const { router, updateParams, query: queryParams } = useNavigation();const [search, setSearch] = useState(queryParams?.search || data?.search || '');- const showPager = allowPaging && data && data.count > data.pageSize;+ const showPager = allowPaging && data && (data.count > data.pageSize || data.isCapped);const { isMobile } = useMobile();const displayMode = isMobile ? 'cards' : undefined;@@ -95,6 +95,7 @@ export function DataGrid({page={data.page}pageSize={data.pageSize}count={data.count}+ isCapped={data.isCapped}onPageChange={handlePageChange}/></Row>diff --git a/src/components/common/Pager.tsx b/src/components/common/Pager.tsxindex 128d33e24..ff22dc70e 100644--- a/src/components/common/Pager.tsx+++ b/src/components/common/Pager.tsx@@ -6,11 +6,12 @@ export interface PagerProps {page: string | number;pageSize: string | number;count: string | number;+ isCapped?: boolean;onPageChange: (nextPage: number) => void;className?: string;}-export function Pager({ page, pageSize, count, onPageChange }: PagerProps) {+export function Pager({ page, pageSize, count, isCapped, onPageChange }: PagerProps) {const { t, labels } = useMessages();const maxPage = pageSize && count ? Math.ceil(+count / +pageSize) : 0;const lastPage = page === maxPage;@@ -28,13 +29,15 @@ export function Pager({ page, pageSize, count, onPageChange }: PagerProps) {}};- if (maxPage === 1) {+ if (maxPage === 1 && !isCapped) {return null;}+ const displayCount = isCapped ? `10,000+` : (+count).toLocaleString();+return (<Row alignItems="center" justifyContent="space-between" gap="3" flexGrow={1}>- <Text>{t(labels.numberOfRecords, { x: count.toLocaleString() })}</Text>+ <Text>{t(labels.numberOfRecords, { x: displayCount })}</Text><Row alignItems="center" justifyContent="flex-end" gap="3"><Text>{t(labels.pageOf, {diff --git a/src/components/hooks/queries/useEventDataPivotQuery.ts b/src/components/hooks/queries/useEventDataPivotQuery.tsindex e9f689ac7..a6ef902bf 100644--- a/src/components/hooks/queries/useEventDataPivotQuery.ts+++ b/src/components/hooks/queries/useEventDataPivotQuery.ts@@ -1,3 +1,4 @@+import { MAX_PAGING_RESULTS } from '@/lib/constants';import { serializeEventPropertyFilters } from '@/lib/params';import type { EventPropertyFilter, ReactQueryOptions } from '@/lib/types';import { useApi } from '../useApi';@@ -28,6 +29,7 @@ export function useEventDataPivotQuery(timezone,...serializeEventPropertyFilters(eventFilters),...params,+ maxResults: MAX_PAGING_RESULTS,}),enabled: !!(websiteId && eventName),...options,diff --git a/src/components/hooks/queries/useReplaysQuery.ts b/src/components/hooks/queries/useReplaysQuery.tsindex 79c1a04bf..795512896 100644--- a/src/components/hooks/queries/useReplaysQuery.ts+++ b/src/components/hooks/queries/useReplaysQuery.ts@@ -1,3 +1,4 @@+import { MAX_PAGING_RESULTS } from '@/lib/constants';import { useApi } from '../useApi';import { useDateParameters } from '../useDateParameters';import { useFilterParameters } from '../useFilterParameters';@@ -24,7 +25,7 @@ export function useReplaysQuery(websiteId: string, params?: Record<string, strin...filters,...pageParams,...params,- pageSize: 20,+ maxResults: MAX_PAGING_RESULTS,});},});diff --git a/src/components/hooks/queries/useRevenueSessionsQuery.ts b/src/components/hooks/queries/useRevenueSessionsQuery.tsindex a1c12190d..21016965b 100644--- a/src/components/hooks/queries/useRevenueSessionsQuery.ts+++ b/src/components/hooks/queries/useRevenueSessionsQuery.ts@@ -1,3 +1,4 @@+import { MAX_PAGING_RESULTS } from '@/lib/constants';import { useApi } from '../useApi';import { useDateParameters } from '../useDateParameters';import { useFilterParameters } from '../useFilterParameters';@@ -29,7 +30,7 @@ export function useRevenueSessionsQuery(...filters,...pageParams,...params,- pageSize: 20,+ maxResults: MAX_PAGING_RESULTS,});},});diff --git a/src/components/hooks/queries/useSavedReplaysQuery.ts b/src/components/hooks/queries/useSavedReplaysQuery.tsindex b6aef525c..c47922203 100644--- a/src/components/hooks/queries/useSavedReplaysQuery.ts+++ b/src/components/hooks/queries/useSavedReplaysQuery.ts@@ -11,7 +11,6 @@ export function useSavedReplaysQuery(websiteId: string) {queryFn: pageParams => {return get(`/websites/${websiteId}/replays/saved`, {...pageParams,- pageSize: 20,});},});diff --git a/src/components/hooks/queries/useSessionDataPivotQuery.ts b/src/components/hooks/queries/useSessionDataPivotQuery.tsindex db5fecccf..80a6bee89 100644--- a/src/components/hooks/queries/useSessionDataPivotQuery.ts+++ b/src/components/hooks/queries/useSessionDataPivotQuery.ts@@ -1,3 +1,4 @@+import { MAX_PAGING_RESULTS } from '@/lib/constants';import { serializePropertyFilters } from '@/lib/params';import type { PropertyFilter, ReactQueryOptions } from '@/lib/types';import { useApi } from '../useApi';@@ -28,6 +29,7 @@ export function useSessionDataPivotQuery(propertyName,...serializePropertyFilters(propertyFilters),...params,+ maxResults: MAX_PAGING_RESULTS,}),enabled: !!(websiteId && propertyName),...options,diff --git a/src/components/hooks/queries/useWebsiteEventsQuery.ts b/src/components/hooks/queries/useWebsiteEventsQuery.tsindex fc4dad5b1..e28e03369 100644--- a/src/components/hooks/queries/useWebsiteEventsQuery.ts+++ b/src/components/hooks/queries/useWebsiteEventsQuery.ts@@ -1,3 +1,4 @@+import { MAX_PAGING_RESULTS } from '@/lib/constants';import type { ReactQueryOptions } from '@/lib/types';import { useApi } from '../useApi';import { useDateParameters } from '../useDateParameters';@@ -32,6 +33,7 @@ export function useWebsiteEventsQuery(...filters,...pageParams,eventType: EVENT_TYPES[params.view],+ maxResults: MAX_PAGING_RESULTS,}),enabled: !!websiteId,...options,diff --git a/src/components/hooks/queries/useWebsiteSessionsQuery.ts b/src/components/hooks/queries/useWebsiteSessionsQuery.tsindex 31906be90..9de64850c 100644--- a/src/components/hooks/queries/useWebsiteSessionsQuery.ts+++ b/src/components/hooks/queries/useWebsiteSessionsQuery.ts@@ -1,3 +1,4 @@+import { MAX_PAGING_RESULTS } from '@/lib/constants';import { useApi } from '../useApi';import { useDateParameters } from '../useDateParameters';import { useFilterParameters } from '../useFilterParameters';@@ -27,7 +28,7 @@ export function useWebsiteSessionsQuery(...filters,...pageParams,...params,- pageSize: 20,+ maxResults: MAX_PAGING_RESULTS,});},});diff --git a/src/lib/clickhouse.ts b/src/lib/clickhouse.tsindex 025e4273d..ae44c9147 100644--- a/src/lib/clickhouse.ts+++ b/src/lib/clickhouse.ts@@ -380,13 +380,15 @@ async function pagedRawQuery(.filter(n => n).join('\n');- const count = await rawQuery(`select count(*) as num from (${query}) t`, queryParams).then(- res => res[0].num,- );+ const { maxResults } = filters;+ const countQuery = maxResults+ ? `select count(*) as num from (select 1 from (${query}) t limit ${+maxResults}) t2`+ : `select count(*) as num from (${query}) t`;+ const count = await rawQuery(countQuery, queryParams).then(res => res[0].num);const data = await rawQuery(`${query}${statements}`, queryParams, name);- return { data, count, page: +page, pageSize: size, orderBy, search };+ return { data, count, page: +page, pageSize: size, orderBy, search, isCapped: !!maxResults && +count >= +maxResults };}async function rawQuery<T = unknown>(diff --git a/src/lib/constants.ts b/src/lib/constants.tsindex f7c182b15..dc01a6d48 100644--- a/src/lib/constants.ts+++ b/src/lib/constants.ts@@ -26,6 +26,7 @@ export const DEFAULT_DATE_RANGE_VALUE = '24hour';export const DEFAULT_WEBSITE_LIMIT = 10;export const DEFAULT_RESET_DATE = '2000-01-01';export const DEFAULT_PAGE_SIZE = 20;+export const MAX_PAGING_RESULTS = 10000;export const DEFAULT_DATE_COMPARE = 'prev';export const DEFAULT_CURRENCY = 'USD';diff --git a/src/lib/prisma.ts b/src/lib/prisma.tsindex 95d585c1e..7f400d05b 100644--- a/src/lib/prisma.ts+++ b/src/lib/prisma.ts@@ -470,13 +470,15 @@ async function pagedRawQuery(.filter(n => n).join('\n');- const count = await rawQuery(`select count(*) as num from (${query}) t`, queryParams).then(- res => res[0].num,- );+ const { maxResults } = filters;+ const countQuery = maxResults+ ? `select count(*) as num from (select 1 from (${query}) t limit ${+maxResults}) t2`+ : `select count(*) as num from (${query}) t`;+ const count = await rawQuery(countQuery, queryParams).then(res => Number(res[0].num));const data = await rawQuery(`${query}${statements}`, queryParams, name);- return { data, count, page: +page, pageSize: size, orderBy };+ return { data, count, page: +page, pageSize: size, orderBy, isCapped: !!maxResults && +count >= +maxResults };}function getSearchParameters(query: string, filters: Record<string, any>[]) {diff --git a/src/lib/request.ts b/src/lib/request.tsindex 9b32b6736..87c4872ad 100644--- a/src/lib/request.ts+++ b/src/lib/request.ts@@ -174,5 +174,6 @@ export async function getQueryFilters(sortDescending: params?.sortDescending,search: params?.search,compare: params?.compare,+ maxResults: params?.maxResults,};}diff --git a/src/lib/schema.ts b/src/lib/schema.tsindex 2567c3de6..113b24cf8 100644--- a/src/lib/schema.ts+++ b/src/lib/schema.ts@@ -77,6 +77,7 @@ export const searchParams = {export const pagingParams = {page: z.coerce.number().int().positive().optional(),pageSize: z.coerce.number().int().positive().optional(),+ maxResults: z.coerce.number().int().positive().optional(),};export const sortingParams = {diff --git a/src/lib/types.ts b/src/lib/types.tsindex dd365aeab..6c8bb92ff 100644--- a/src/lib/types.ts+++ b/src/lib/types.ts@@ -157,6 +157,7 @@ export interface SortParams {export interface PageParams {page?: number;pageSize?: number;+ maxResults?: number;}export interface SegmentParams {@@ -172,6 +173,7 @@ export interface PageResult<T> {orderBy?: string;sortDescending?: boolean;search?: string;+ isCapped?: boolean;}export interface RealtimeData {diff --git a/src/queries/sql/events/getEventDataPivot.ts b/src/queries/sql/events/getEventDataPivot.tsindex 8699d3269..210a7cdc6 100644--- a/src/queries/sql/events/getEventDataPivot.ts+++ b/src/queries/sql/events/getEventDataPivot.ts@@ -1,5 +1,4 @@import clickhouse from '@/lib/clickhouse';-import { DEFAULT_PAGE_SIZE } from '@/lib/constants';import { CLICKHOUSE, PRISMA, runQuery } from '@/lib/db';import prisma from '@/lib/prisma';import type { EventPropertyFilter, QueryFilters } from '@/lib/types';@@ -27,10 +26,7 @@ async function relationalQuery(eventFilters: EventPropertyFilter[] = [],) {const { timezone = 'utc' } = filters;- const { rawQuery, parseFilters, getPropertyFilterQuery, getDateStringSQL } = prisma;- const { page = 1, pageSize } = filters;- const size = +pageSize || DEFAULT_PAGE_SIZE;- const offset = +size * (+page - 1);+ const { pagedRawQuery, parseFilters, getPropertyFilterQuery, getDateStringSQL } = prisma;const { filterQuery, cohortQuery, joinSessionQuery, queryParams } = parseFilters({...filters,@@ -39,30 +35,10 @@ async function relationalQuery(});const { sql: pfSQL, params: pfParams } = getPropertyFilterQuery(eventFilters, 'event', timezone);- const countResult = await rawQuery(- `- select count(distinct website_event.event_id) as num- from website_event- join event_data on event_data.website_event_id = website_event.event_id- and event_data.website_id = {{websiteId::uuid}}- and event_data.created_at between {{startDate}} and {{endDate}}- ${cohortQuery}- ${joinSessionQuery}- where website_event.website_id = {{websiteId::uuid}}- and website_event.created_at between {{startDate}} and {{endDate}}- and website_event.event_name = {{eventName}}- ${filterQuery}- ${pfSQL}- `,- { ...queryParams, eventName, ...pfParams },- );-- const count = countResult[0].num;-- const rows = await rawQuery(+ return pagedRawQuery(`with paged_events as (- select website_event.event_id+ select website_event.event_id, max(website_event.created_at) as sort_created_atfrom website_eventjoin event_data on event_data.website_event_id = website_event.event_idand event_data.website_id = {{websiteId::uuid}}@@ -75,68 +51,43 @@ async function relationalQuery(${filterQuery}${pfSQL}group by website_event.event_id- order by max(website_event.created_at) desc- limit ${size} offset ${offset})selectwebsite_event.event_id as "eventId",website_event.session_id as "sessionId",website_event.event_name as "eventName",website_event.url_path as "urlPath",- website_event.created_at as "createdAt",- event_data.data_key as "dataKey",- coalesce(- case when event_data.data_type = 1 then event_data.string_value end,- case when event_data.data_type = 2 then cast(event_data.number_value as varchar) end,- case when event_data.data_type = 3 then event_data.string_value end,- case when event_data.data_type = 4 then ${getDateStringSQL('event_data.date_value', 'second', timezone)} end,- case when event_data.data_type = 5 then event_data.string_value end,- ''- ) as "value",- event_data.data_type as "dataType"+ max(website_event.created_at) as "createdAt",+ array_agg(event_data.data_key order by event_data.data_key asc) as "propertyKeys",+ array_agg(+ coalesce(+ case when event_data.data_type = 1 then event_data.string_value end,+ case when event_data.data_type = 2 then cast(event_data.number_value as varchar) end,+ case when event_data.data_type = 3 then event_data.string_value end,+ case when event_data.data_type = 4 then ${getDateStringSQL('event_data.date_value', 'second', timezone)} end,+ case when event_data.data_type = 5 then event_data.string_value end,+ ''+ )+ order by event_data.data_key asc+ ) as "propertyValues"from event_datajoin website_event on website_event.event_id = event_data.website_event_idand website_event.website_id = {{websiteId::uuid}}join paged_events on paged_events.event_id = event_data.website_event_idwhere event_data.website_id = {{websiteId::uuid}}and event_data.created_at between {{startDate}} and {{endDate}}- order by website_event.created_at desc+ group by+ website_event.event_id,+ website_event.session_id,+ website_event.event_name,+ website_event.url_path,+ paged_events.sort_created_at+ order by paged_events.sort_created_at desc`,{ ...queryParams, eventName, ...pfParams },+ filters,FUNCTION_NAME,);-- // Pivot flat rows into one record per event- const eventMap = new Map<- string,- {- eventId: string;- sessionId: string;- eventName: string;- urlPath: string;- createdAt: Date;- propertyKeys: string[];- propertyValues: string[];- }- >();- for (const { eventId, sessionId, eventName: name, urlPath, createdAt, dataKey, value } of rows) {- if (!eventMap.has(eventId)) {- eventMap.set(eventId, {- eventId,- sessionId,- eventName: name,- urlPath,- createdAt,- propertyKeys: [],- propertyValues: [],- });- }- const entry = eventMap.get(eventId);- entry.propertyKeys.push(dataKey);- entry.propertyValues.push(value ?? '');- }-- return { data: [...eventMap.values()], count, page: +page, pageSize: size };}async function clickhouseQuery(@@ -146,10 +97,7 @@ async function clickhouseQuery(eventFilters: EventPropertyFilter[] = [],) {const { timezone = 'UTC' } = filters;- const { rawQuery, parseFilters, getPropertyFilterQuery, getDateStringSQL } = clickhouse;- const { page = 1, pageSize } = filters;- const size = +pageSize || DEFAULT_PAGE_SIZE;- const offset = +size * (+page - 1);+ const { pagedRawQuery, parseFilters, getPropertyFilterQuery, getDateStringSQL } = clickhouse;const { filterQuery, cohortQuery, queryParams } = parseFilters({...filters,@@ -158,31 +106,7 @@ async function clickhouseQuery(});const { sql: pfSQL, params: pfParams } = getPropertyFilterQuery(eventFilters, 'event', timezone);- const count = await rawQuery(- `- select count() as num- from umami.event_data_pivot- any left join (- select *- from website_event- where website_id = {websiteId:UUID}- and created_at between {startDate:DateTime64} and {endDate:DateTime64}- and event_type = 2- and event_name = {eventName:String}) website_event- on website_event.event_id = event_data_pivot.event_id- and website_event.session_id = event_data_pivot.session_id- and website_event.website_id = event_data_pivot.website_id- ${cohortQuery}- where event_data_pivot.website_id = {websiteId:UUID}- and event_data_pivot.created_at between {startDate:DateTime64} and {endDate:DateTime64}- and event_data_pivot.event_name = {eventName:String}- ${filterQuery}- ${pfSQL}- `,- { ...queryParams, eventName, ...pfParams },- ).then((res: any) => res[0].num);-- const data = await rawQuery(+ return pagedRawQuery(`selectevent_data_pivot.event_id as eventId,@@ -224,11 +148,9 @@ async function clickhouseQuery(event_data_pivot.url_path,event_data_pivot.created_atorder by event_data_pivot.created_at desc- limit ${size} offset ${offset}`,{ ...queryParams, eventName, ...pfParams },+ filters,FUNCTION_NAME,);-- return { data, count, page: +page, pageSize: size };}diff --git a/src/queries/sql/sessions/getSessionDataPivot.ts b/src/queries/sql/sessions/getSessionDataPivot.tsindex ac4528123..c4e460b0e 100644--- a/src/queries/sql/sessions/getSessionDataPivot.ts+++ b/src/queries/sql/sessions/getSessionDataPivot.ts@@ -1,5 +1,4 @@import clickhouse from '@/lib/clickhouse';-import { DEFAULT_PAGE_SIZE } from '@/lib/constants';import { CLICKHOUSE, PRISMA, runQuery } from '@/lib/db';import prisma from '@/lib/prisma';import type { PageResult, PropertyFilter, QueryFilters, SessionDataPivotRow } from '@/lib/types';@@ -25,12 +24,9 @@ async function relationalQuery(propertyName: string,filters: QueryFilters,propertyFilters: PropertyFilter[] = [],-): Promise<PageResult<SessionDataPivotRow[]>> {+) {const { timezone = 'utc' } = filters;- const { rawQuery, parseFilters, getPropertyFilterQuery, getDateStringSQL } = prisma;- const { page = 1, pageSize } = filters;- const size = +pageSize || DEFAULT_PAGE_SIZE;- const offset = +size * (+page - 1);+ const { pagedRawQuery, parseFilters, getPropertyFilterQuery, getDateStringSQL } = prisma;const { filterQuery, cohortQuery, joinSessionQuery, queryParams } = parseFilters({...filters,@@ -43,47 +39,7 @@ async function relationalQuery(timezone,);- const countResult = (await rawQuery(- `- with filtered_sessions as (- select distinct website_event.session_id- from website_event- ${cohortQuery}- ${joinSessionQuery}- where website_event.website_id = {{websiteId::uuid}}- and website_event.created_at between {{startDate}} and {{endDate}}- ${filterQuery}- ${pfSQL}- ),- latest_session_properties as (- select- ranked.session_id,- ranked.data_key- from (- select- session_data.session_id,- session_data.data_key,- row_number() over (- partition by session_data.session_id, session_data.data_key- order by session_data.created_at desc, session_data.session_data_id desc- ) as row_num- from session_data- join filtered_sessions- on filtered_sessions.session_id = session_data.session_id- where session_data.website_id = {{websiteId::uuid}}- ) ranked- where ranked.row_num = 1- )- select count(*) as num- from latest_session_properties- where latest_session_properties.data_key = {{propertyName}}- `,- { ...queryParams, websiteId, propertyName, ...pfParams },- )) as { num: number }[];-- const count = countResult[0].num;-- const rows = (await rawQuery(+ return pagedRawQuery(`with filtered_sessions as (select distinct website_event.session_id@@ -132,8 +88,6 @@ async function relationalQuery(latest_session_properties.created_at as sort_created_atfrom latest_session_propertieswhere latest_session_properties.data_key = {{propertyName}}- order by latest_session_properties.created_at desc- limit ${size} offset ${offset})selectlatest_session_properties.session_id as "sessionId",@@ -158,10 +112,9 @@ async function relationalQuery(order by paged_sessions.sort_created_at desc`,{ ...queryParams, websiteId, propertyName, ...pfParams },+ filters,FUNCTION_NAME,- )) as SessionDataPivotRow[];-- return { data: rows, count, page: +page, pageSize: size };+ );}async function clickhouseQuery(@@ -169,12 +122,9 @@ async function clickhouseQuery(propertyName: string,filters: QueryFilters,propertyFilters: PropertyFilter[] = [],-): Promise<PageResult<SessionDataPivotRow[]>> {+) {const { timezone = 'UTC' } = filters;- const { rawQuery, parseFilters, getPropertyFilterQuery, getDateStringSQL } = clickhouse;- const { page = 1, pageSize } = filters;- const size = +pageSize || DEFAULT_PAGE_SIZE;- const offset = +size * (+page - 1);+ const { pagedRawQuery, parseFilters, getPropertyFilterQuery, getDateStringSQL } = clickhouse;const { filterQuery, cohortQuery, queryParams } = parseFilters({ ...filters, websiteId, timezone });const { sql: pfSQL, params: pfParams } = getPropertyFilterQuery(@@ -183,38 +133,7 @@ async function clickhouseQuery(timezone,);- const countResult = (await rawQuery(- `- with filtered_sessions as (- select distinct website_event.session_id- from website_event- ${cohortQuery}- where website_event.website_id = {websiteId:UUID}- and website_event.created_at between {startDate:DateTime64} and {endDate:DateTime64}- ${filterQuery}- ${pfSQL}- ),- latest_session_properties as (- select- session_data.session_id as session_id,- session_data.data_key as data_key- from session_data final- join filtered_sessions- on filtered_sessions.session_id = session_data.session_id- where session_data.website_id = {websiteId:UUID}- group by- session_data.session_id,- session_data.data_key- )- select count() as num- from latest_session_properties- where latest_session_properties.data_key = {propertyName:String}- `,- { ...queryParams, websiteId, propertyName, ...pfParams },- )) as { num: number }[];- const count = countResult[0].num;-- const data = (await rawQuery(+ return pagedRawQuery(`with filtered_sessions as (select distinct website_event.session_id@@ -249,8 +168,6 @@ async function clickhouseQuery(latest_session_properties.created_at as sort_created_atfrom latest_session_propertieswhere latest_session_properties.data_key = {propertyName:String}- order by latest_session_properties.created_at desc- limit ${size} offset ${offset})selectlatest_session_properties.session_id as sessionId,@@ -274,8 +191,7 @@ async function clickhouseQuery(order by paged_sessions.sort_created_at desc`,{ ...queryParams, websiteId, propertyName, ...pfParams },+ filters,FUNCTION_NAME,- )) as SessionDataPivotRow[];-- return { data, count, page: +page, pageSize: size };+ );}