Skip to content

armgit / umami

Public Imported from GitHub
0

Umami is a privacy-first web and product analytics platform. Traffic, campaigns, behavior, conversions, and revenue in one place. No cookies, no surveillance, self-hosted or in the cloud.

fix 500 websiteEvent

Francis Caob6b500ea4ff5
diff --git a/src/queries/sql/events/getWebsiteEventStats.test.ts b/src/queries/sql/events/getWebsiteEventStats.test.ts
index 2794386fb..e9f98f5e9 100644
--- a/src/queries/sql/events/getWebsiteEventStats.test.ts
+++ b/src/queries/sql/events/getWebsiteEventStats.test.ts
@@ -89,4 +89,42 @@ describe('getWebsiteEventStats', () => {
expect(query).not.toContain('group by session_id, visit_id, event_name');
expect(query).not.toContain('sum(t.c)');
});
+
+ test('clickhouse uses the hourly rollup when there are no extra filters', async () => {
+ vi.resetModules();
+
+ const clickhouseRawQuery = vi.fn().mockResolvedValue([{}]);
+
+ vi.doMock('@/lib/db', () => ({
+ CLICKHOUSE: 'clickhouse',
+ PRISMA: 'prisma',
+ runQuery: vi.fn((queries: Record<string, () => unknown>) => queries.clickhouse()),
+ }));
+
+ vi.doMock('@/lib/prisma', () => ({
+ default: {},
+ }));
+
+ vi.doMock('@/lib/clickhouse', () => ({
+ default: {
+ rawQuery: clickhouseRawQuery,
+ parseFilters: vi.fn().mockReturnValue({
+ queryParams: { websiteId: 'website-1' },
+ filterQuery: '',
+ cohortQuery: '',
+ }),
+ },
+ }));
+
+ const { getWebsiteEventStats } = await import('./getWebsiteEventStats');
+
+ await getWebsiteEventStats('website-1', {} as any);
+
+ const [query] = clickhouseRawQuery.mock.calls[0];
+
+ expect(query).toContain('sum(length(event_name)) as "events"');
+ expect(query).toContain('uniqArray(event_name) as "uniqueEvents"');
+ expect(query).toContain('from website_event_stats_hourly website_event');
+ expect(query).not.toContain('from website_event\n');
+ });
});
diff --git a/src/queries/sql/events/getWebsiteEventStats.ts b/src/queries/sql/events/getWebsiteEventStats.ts
index e73856020..a8fba94ad 100644
--- a/src/queries/sql/events/getWebsiteEventStats.ts
+++ b/src/queries/sql/events/getWebsiteEventStats.ts
@@ -61,21 +61,31 @@ async function clickhouseQuery(
websiteId,
});
- return rawQuery(
- `
- select
- count(*) as "events",
- uniq(session_id) as "visitors",
- uniq(visit_id) as "visits",
- uniq(event_name) as "uniqueEvents"
- from website_event
- ${cohortQuery}
- where website_id = {websiteId:UUID}
- and created_at between {startDate:DateTime64} and {endDate:DateTime64}
- and event_type = 2
- ${filterQuery};
- `,
- queryParams,
- FUNCTION_NAME,
- ).then(result => result?.[0]);
+ const sql = filterQuery || cohortQuery
+ ? `
+ select
+ count(*) as "events",
+ uniq(session_id) as "visitors",
+ uniq(visit_id) as "visits",
+ uniq(event_name) as "uniqueEvents"
+ from website_event
+ ${cohortQuery}
+ where website_id = {websiteId:UUID}
+ and created_at between {startDate:DateTime64} and {endDate:DateTime64}
+ and event_type = 2
+ ${filterQuery};
+ `
+ : `
+ select
+ sum(length(event_name)) as "events",
+ uniq(session_id) as "visitors",
+ uniq(visit_id) as "visits",
+ uniqArray(event_name) as "uniqueEvents"
+ from website_event_stats_hourly website_event
+ where website_id = {websiteId:UUID}
+ and created_at between {startDate:DateTime64} and {endDate:DateTime64}
+ and event_type = 2;
+ `;
+
+ return rawQuery(sql, queryParams, FUNCTION_NAME).then(result => result?.[0]);
}
diff --git a/src/queries/sql/events/getWebsiteEvents.test.ts b/src/queries/sql/events/getWebsiteEvents.test.ts
index 56db24015..cc6cc7561 100644
--- a/src/queries/sql/events/getWebsiteEvents.test.ts
+++ b/src/queries/sql/events/getWebsiteEvents.test.ts
@@ -87,13 +87,14 @@ describe('getWebsiteEvents', () => {
expect(countQuery).toContain('select count(*) as num from (select 1 from (');
expect(countQuery).not.toContain('exists(');
expect(dataQuery).toContain('with paged_events as (');
+ expect(dataQuery).toContain('paged_event_data as (');
expect(dataQuery).toContain('join website_event on website_event.event_id = paged_events.event_id');
- expect(dataQuery).toContain('exists(');
- expect(dataQuery).toContain('where event_data.website_event_id = website_event.event_id');
- expect(dataQuery).toContain('and event_data.website_id = {{websiteId::uuid}}');
+ expect(dataQuery).toContain('join paged_events on paged_events.event_id = event_data.website_event_id');
+ expect(dataQuery).toContain('where event_data.website_id = {{websiteId::uuid}}');
expect(dataQuery).toContain('and event_data.created_at between {{startDate}} and {{endDate}}');
+ expect(dataQuery).toContain('left join paged_event_data on paged_event_data.event_id = website_event.event_id');
expect(dataQuery).toContain('order by paged_events.created_at desc');
- expect(dataQuery).not.toContain('created_at between {{startDate}} and {{endDate}}) AS "hasData"');
+ expect(dataQuery).toContain('(paged_event_data.event_id is not null) as "hasData"');
expect(result).toEqual({
data: [{ id: 'event-1' }],
count: 25,
@@ -121,13 +122,15 @@ describe('getWebsiteEvents', () => {
expect(countQuery).toContain('select count(*) as num from (select 1 from (');
expect(countQuery).not.toContain('event_data');
expect(dataQuery).toContain('with paged_events as (');
+ expect(dataQuery).toContain('paged_event_data as (');
expect(dataQuery).toContain('inner join website_event on website_event.event_id = paged_events.event_id');
expect(dataQuery).toContain('where website_id = {websiteId:UUID}');
expect(dataQuery).toContain('from event_data');
+ expect(dataQuery).toContain('inner join paged_events on paged_events.event_id = event_data.event_id');
expect(dataQuery).toContain('and created_at between {startDate:DateTime64} and {endDate:DateTime64}');
- expect(dataQuery).toContain('and event_id in (select event_id from paged_events)');
+ expect(dataQuery).toContain('left join paged_event_data on paged_event_data.event_id = website_event.event_id');
expect(dataQuery).toContain('order by paged_events.created_at desc');
- expect(dataQuery).not.toContain('event_id IN (select event_id');
+ expect(dataQuery).toContain('paged_event_data.event_id is not null as hasData');
expect(result).toEqual({
data: [{ id: 'event-1' }],
count: '25',
@@ -182,7 +185,7 @@ describe('getWebsiteEvents', () => {
'and created_at between toTimezone({startDate:DateTime64},{timezone:String}) and toTimezone({endDate:DateTime64},{timezone:String})',
);
expect(dataQuery).not.toContain(
- 'and created_at between {startDate:DateTime64} and {endDate:DateTime64}\r\n and event_id in (select event_id from paged_events)',
+ 'and created_at between {startDate:DateTime64} and {endDate:DateTime64}',
);
});
});
diff --git a/src/queries/sql/events/getWebsiteEvents.ts b/src/queries/sql/events/getWebsiteEvents.ts
index a57d767ef..5b6509406 100644
--- a/src/queries/sql/events/getWebsiteEvents.ts
+++ b/src/queries/sql/events/getWebsiteEvents.ts
@@ -56,6 +56,13 @@ async function relationalQuery(websiteId: string, filters: QueryFilters) {
${eventQuery}
order by created_at desc
limit ${size} offset ${offset}
+ ),
+ paged_event_data as (
+ select distinct event_data.website_event_id as event_id
+ from event_data
+ join paged_events on paged_events.event_id = event_data.website_event_id
+ where event_data.website_id = {{websiteId::uuid}}
+ ${hasDataDateQuery}
)
select
website_event.event_id as "id",
@@ -76,17 +83,12 @@ async function relationalQuery(websiteId: string, filters: QueryFilters) {
website_event.page_title as "pageTitle",
website_event.event_type as "eventType",
website_event.event_name as "eventName",
- exists(
- select 1
- from event_data
- where event_data.website_event_id = website_event.event_id
- and event_data.website_id = {{websiteId::uuid}}
- ${hasDataDateQuery}
- ) as "hasData"
+ (paged_event_data.event_id is not null) as "hasData"
from paged_events
join website_event on website_event.event_id = paged_events.event_id
join session on session.session_id = website_event.session_id
and session.website_id = website_event.website_id
+ left join paged_event_data on paged_event_data.event_id = website_event.event_id
order by paged_events.created_at desc
`,
queryParams,
@@ -143,6 +145,13 @@ async function clickhouseQuery(websiteId: string, filters: QueryFilters) {
${eventQuery}
order by created_at desc
limit ${size} offset ${offset}
+ ),
+ paged_event_data as (
+ select distinct event_id
+ from event_data
+ inner join paged_events on paged_events.event_id = event_data.event_id
+ where website_id = {websiteId:UUID}
+ ${dateQuery}
)
select
website_event.event_id as id,
@@ -163,15 +172,10 @@ async function clickhouseQuery(websiteId: string, filters: QueryFilters) {
website_event.page_title as pageTitle,
website_event.event_type as eventType,
website_event.event_name as eventName,
- website_event.event_id in (
- select event_id
- from event_data
- where website_id = {websiteId:UUID}
- ${dateQuery}
- and event_id in (select event_id from paged_events)
- ) as hasData
+ paged_event_data.event_id is not null as hasData
from paged_events
inner join website_event on website_event.event_id = paged_events.event_id
+ left join paged_event_data on paged_event_data.event_id = website_event.event_id
order by paged_events.created_at desc
`,
queryParams,