Skip to content

armgit / umami

Public Imported from GitHub
0

Umami is a privacy-first web and product analytics platform. Traffic, campaigns, behavior, conversions, and revenue in one place. No cookies, no surveillance, self-hosted or in the cloud.

Improve website events query performance and fix activity search persistence

Francis Cao079cfdde8461
diff --git a/src/app/(main)/websites/[websiteId]/WebsiteTabs.tsx b/src/app/(main)/websites/[websiteId]/WebsiteTabs.tsx
index 782d55ad8..7d63b4889 100644
--- a/src/app/(main)/websites/[websiteId]/WebsiteTabs.tsx
+++ b/src/app/(main)/websites/[websiteId]/WebsiteTabs.tsx
@@ -7,6 +7,10 @@ export function WebsiteTabs() {
const website = useWebsite();
const { pathname, renderUrl } = useNavigation();
const { t, labels } = useMessages();
+ const resetParams = {
+ search: undefined,
+ page: undefined,
+ };
const links = [
{
@@ -49,7 +53,7 @@ export function WebsiteTabs() {
<TabList>
{links.map(({ id, label, icon, path }) => {
return (
- <Tab key={id} id={id} href={renderUrl(`/websites/${website.id}${path}`)}>
+ <Tab key={id} id={id} href={renderUrl(`/websites/${website.id}${path}`, resetParams)}>
<Row alignItems="center" gap>
<Icon>{icon}</Icon>
<Text>{label}</Text>
diff --git a/src/app/(main)/websites/[websiteId]/events/EventsTable.tsx b/src/app/(main)/websites/[websiteId]/events/EventsTable.tsx
index bec5289cd..5035f0e4c 100644
--- a/src/app/(main)/websites/[websiteId]/events/EventsTable.tsx
+++ b/src/app/(main)/websites/[websiteId]/events/EventsTable.tsx
@@ -1,5 +1,6 @@
import {
Button,
+ Column,
DataColumn,
DataTable,
type DataTableProps,
@@ -43,7 +44,7 @@ export function EventsTable(props: DataTableProps) {
<DataColumn id="event" label={t(labels.event)} width="2fr">
{(row: any) => {
return (
- <Row alignItems="center" wrap="wrap" gap>
+ <Column gap="2">
<Row>
<IconLabel
icon={row.eventName ? <Lightning /> : <Eye />}
@@ -51,16 +52,18 @@ export function EventsTable(props: DataTableProps) {
labelProps={{ wrap: 'nowrap' }}
/>
</Row>
- <Text
- weight="bold"
- style={{ maxWidth: '300px' }}
- title={row.eventName || row.urlPath}
- truncate
- >
- {row.eventName || renderLink(row.urlPath, row.hostname)}
- </Text>
- {row.hasData > 0 && <PropertiesButton websiteId={row.websiteId} eventId={row.id} />}
- </Row>
+ <Row alignItems="center" gap>
+ <Text
+ weight="bold"
+ style={{ maxWidth: '300px' }}
+ title={row.eventName || row.urlPath}
+ truncate
+ >
+ {row.eventName || renderLink(row.urlPath, row.hostname)}
+ </Text>
+ {row.hasData > 0 && <PropertiesButton websiteId={row.websiteId} eventId={row.id} />}
+ </Row>
+ </Column>
);
}}
</DataColumn>
diff --git a/src/components/common/DataGrid.tsx b/src/components/common/DataGrid.tsx
index 0717a4d09..f730c7ed6 100644
--- a/src/components/common/DataGrid.tsx
+++ b/src/components/common/DataGrid.tsx
@@ -16,6 +16,7 @@ import {
type ReactElement,
type ReactNode,
useCallback,
+ useEffect,
useState,
} from 'react';
import { Empty } from '@/components/common/Empty';
@@ -73,7 +74,6 @@ export function DataGrid({
const handleSearch = (value: string) => {
if (value !== search) {
setSearch(value);
- router.push(updateParams({ search: value, page: 1 }));
}
};
@@ -84,6 +84,24 @@ export function DataGrid({
[router, updateParams, search],
);
+ useEffect(() => {
+ setSearch(queryParams?.search || '');
+ }, [queryParams?.search]);
+
+ useEffect(() => {
+ const currentSearch = queryParams?.search || '';
+
+ if (search === currentSearch) {
+ return;
+ }
+
+ const timer = window.setTimeout(() => {
+ router.push(updateParams({ search: search || undefined, page: 1 }));
+ }, searchDelay || DEFAULT_SEARCH_DELAY);
+
+ return () => window.clearTimeout(timer);
+ }, [queryParams?.search, router, search, searchDelay, updateParams]);
+
const child = data ? (typeof children === 'function' ? children(data) : children) : null;
const viewToggleButton = (
@@ -106,7 +124,7 @@ export function DataGrid({
<SearchField
value={search}
onSearch={handleSearch}
- delay={searchDelay || DEFAULT_SEARCH_DELAY}
+ delay={0}
autoFocus={autoFocus}
placeholder={t(labels.search)}
className="w-full max-w-md"
diff --git a/src/components/hooks/useNavigation.ts b/src/components/hooks/useNavigation.ts
index f5eae008c..ba9702320 100644
--- a/src/components/hooks/useNavigation.ts
+++ b/src/components/hooks/useNavigation.ts
@@ -1,5 +1,5 @@
import { usePathname, useRouter, useSearchParams } from 'next/navigation';
-import { useCallback, useEffect, useState } from 'react';
+import { useCallback, useMemo } from 'react';
import { buildPath } from '@/lib/url';
export function useNavigation() {
@@ -11,7 +11,8 @@ export function useNavigation() {
const [, linkId] = pathname.match(/\/links\/([a-f0-9-]+)/) || [];
const [, pixelId] = pathname.match(/\/pixels\/([a-f0-9-]+)/) || [];
const [, boardId] = pathname.match(/\/boards\/([a-f0-9-]+)/) || [];
- const [queryParams, setQueryParams] = useState(Object.fromEntries(searchParams));
+ const searchParamsString = searchParams.toString();
+ const queryParams = useMemo(() => Object.fromEntries(searchParams), [searchParamsString]);
const updateParams = useCallback(
(params?: Record<string, string | number>) => {
@@ -37,10 +38,6 @@ export function useNavigation() {
[teamId, queryParams],
);
- useEffect(() => {
- setQueryParams(Object.fromEntries(searchParams));
- }, [searchParams.toString()]);
-
return {
router,
pathname,
diff --git a/src/components/hooks/useWebsiteNavItems.tsx b/src/components/hooks/useWebsiteNavItems.tsx
index 09731f101..0d0adb2db 100644
--- a/src/components/hooks/useWebsiteNavItems.tsx
+++ b/src/components/hooks/useWebsiteNavItems.tsx
@@ -17,9 +17,14 @@ import { useNavigation } from './useNavigation';
export function useWebsiteNavItems(websiteId: string) {
const { t, labels } = useMessages();
const { pathname, renderUrl } = useNavigation();
+ const resetParams = {
+ search: undefined,
+ page: undefined,
+ };
const renderPath = (path: string) =>
renderUrl(`/websites/${websiteId}${path}`, {
+ ...resetParams,
event: undefined,
compare: undefined,
view: undefined,
diff --git a/src/queries/sql/events/getWebsiteEvents.test.ts b/src/queries/sql/events/getWebsiteEvents.test.ts
index cc6cc7561..906d64be8 100644
--- a/src/queries/sql/events/getWebsiteEvents.test.ts
+++ b/src/queries/sql/events/getWebsiteEvents.test.ts
@@ -88,7 +88,6 @@ describe('getWebsiteEvents', () => {
expect(countQuery).not.toContain('exists(');
expect(dataQuery).toContain('with paged_events as (');
expect(dataQuery).toContain('paged_event_data as (');
- expect(dataQuery).toContain('join website_event on website_event.event_id = paged_events.event_id');
expect(dataQuery).toContain('join paged_events on paged_events.event_id = event_data.website_event_id');
expect(dataQuery).toContain('where event_data.website_id = {{websiteId::uuid}}');
expect(dataQuery).toContain('and event_data.created_at between {{startDate}} and {{endDate}}');
@@ -122,15 +121,19 @@ describe('getWebsiteEvents', () => {
expect(countQuery).toContain('select count(*) as num from (select 1 from (');
expect(countQuery).not.toContain('event_data');
expect(dataQuery).toContain('with paged_events as (');
+ expect(dataQuery).toContain('website_id as websiteId');
expect(dataQuery).toContain('paged_event_data as (');
- expect(dataQuery).toContain('inner join website_event on website_event.event_id = paged_events.event_id');
expect(dataQuery).toContain('where website_id = {websiteId:UUID}');
expect(dataQuery).toContain('from event_data');
- expect(dataQuery).toContain('inner join paged_events on paged_events.event_id = event_data.event_id');
- expect(dataQuery).toContain('and created_at between {startDate:DateTime64} and {endDate:DateTime64}');
- expect(dataQuery).toContain('left join paged_event_data on paged_event_data.event_id = website_event.event_id');
- expect(dataQuery).toContain('order by paged_events.created_at desc');
- expect(dataQuery).toContain('paged_event_data.event_id is not null as hasData');
+ expect(dataQuery).toContain('inner join paged_events on paged_events.id = event_data.event_id');
+ expect(dataQuery).toContain(
+ 'and event_data.created_at between {startDate:DateTime64} and {endDate:DateTime64}',
+ );
+ expect(dataQuery).toContain('from paged_events');
+ expect(dataQuery).toContain('left join paged_event_data on paged_event_data.event_id = paged_events.id');
+ expect(dataQuery).toContain('order by paged_events.createdAt desc');
+ expect(dataQuery).toContain('toUInt8(1) as has_data');
+ expect(dataQuery).toContain('ifNull(paged_event_data.has_data, 0) as hasData');
expect(result).toEqual({
data: [{ id: 'event-1' }],
count: '25',
@@ -182,7 +185,7 @@ describe('getWebsiteEvents', () => {
const [dataQuery] = clickhouseRawQuery.mock.calls[1];
expect(dataQuery).toContain(
- 'and created_at between toTimezone({startDate:DateTime64},{timezone:String}) and toTimezone({endDate:DateTime64},{timezone:String})',
+ 'and event_data.created_at between toTimezone({startDate:DateTime64},{timezone:String}) and toTimezone({endDate:DateTime64},{timezone:String})',
);
expect(dataQuery).not.toContain(
'and created_at between {startDate:DateTime64} and {endDate:DateTime64}',
diff --git a/src/queries/sql/events/getWebsiteEvents.ts b/src/queries/sql/events/getWebsiteEvents.ts
index 5b6509406..2680d608e 100644
--- a/src/queries/sql/events/getWebsiteEvents.ts
+++ b/src/queries/sql/events/getWebsiteEvents.ts
@@ -114,6 +114,7 @@ async function clickhouseQuery(websiteId: string, filters: QueryFilters) {
...filters,
websiteId,
});
+ const hasDataDateQuery = dateQuery.replaceAll('created_at', 'event_data.created_at');
const searchQuery = search
? `and ((positionCaseInsensitive(event_name, {search:String}) > 0 and event_type = ${EVENT_TYPE.customEvent})
@@ -142,41 +143,51 @@ async function clickhouseQuery(websiteId: string, filters: QueryFilters) {
const data = await rawQuery(
`
with paged_events as (
- ${eventQuery}
+ select
+ event_id as id,
+ website_id as websiteId,
+ session_id as sessionId,
+ created_at as createdAt,
+ hostname,
+ url_path as urlPath,
+ url_query as urlQuery,
+ referrer_path as referrerPath,
+ referrer_query as referrerQuery,
+ referrer_domain as referrerDomain,
+ country,
+ city,
+ device,
+ os,
+ browser,
+ page_title as pageTitle,
+ event_type as eventType,
+ event_name as eventName
+ from website_event
+ ${cohortQuery}
+ where website_id = {websiteId:UUID}
+ and event_type != ${EVENT_TYPE.performance}
+ ${dateQuery}
+ ${filterQuery}
+ ${searchQuery}
order by created_at desc
limit ${size} offset ${offset}
),
paged_event_data as (
- select distinct event_id
+ select
+ event_id,
+ toUInt8(1) as has_data
from event_data
- inner join paged_events on paged_events.event_id = event_data.event_id
+ inner join paged_events on paged_events.id = event_data.event_id
where website_id = {websiteId:UUID}
- ${dateQuery}
+ ${hasDataDateQuery}
+ group by event_id
)
select
- website_event.event_id as id,
- website_event.website_id as websiteId,
- website_event.session_id as sessionId,
- website_event.created_at as createdAt,
- website_event.hostname,
- website_event.url_path as urlPath,
- website_event.url_query as urlQuery,
- website_event.referrer_path as referrerPath,
- website_event.referrer_query as referrerQuery,
- website_event.referrer_domain as referrerDomain,
- website_event.country as country,
- website_event.city as city,
- website_event.device as device,
- website_event.os as os,
- website_event.browser as browser,
- website_event.page_title as pageTitle,
- website_event.event_type as eventType,
- website_event.event_name as eventName,
- paged_event_data.event_id is not null as hasData
+ paged_events.*,
+ ifNull(paged_event_data.has_data, 0) as hasData
from paged_events
- inner join website_event on website_event.event_id = paged_events.event_id
- left join paged_event_data on paged_event_data.event_id = website_event.event_id
- order by paged_events.created_at desc
+ left join paged_event_data on paged_event_data.event_id = paged_events.id
+ order by paged_events.createdAt desc
`,
queryParams,
FUNCTION_NAME,