Skip to content

armgit / umami

Public Imported from GitHub
0

Umami is a privacy-first web and product analytics platform. Traffic, campaigns, behavior, conversions, and revenue in one place. No cookies, no surveillance, self-hosted or in the cloud.

query optimizations on session property filters

Francis Caoa113f5be79ed
diff --git a/db/clickhouse/migrations/11_add_event_session_data_pivot.sql b/db/clickhouse/migrations/11_add_event_session_data_pivot.sql
index 2d44cf0bf..388649859 100644
--- a/db/clickhouse/migrations/11_add_event_session_data_pivot.sql
+++ b/db/clickhouse/migrations/11_add_event_session_data_pivot.sql
@@ -108,3 +108,19 @@ SELECT
groupArrayState(data_type)
FROM umami.session_data
GROUP BY website_id, session_id, distinct_id;
+
+ALTER TABLE umami.session_data
+ADD PROJECTION session_data_property_filter_projection (
+ SELECT *
+ ORDER BY (
+ website_id,
+ data_key,
+ data_type,
+ string_value,
+ number_value,
+ date_value,
+ session_id
+ )
+);
+
+ALTER TABLE umami.session_data MATERIALIZE PROJECTION session_data_property_filter_projection;
diff --git a/db/clickhouse/schema.sql b/db/clickhouse/schema.sql
index c5c8dce67..d4041dde9 100644
--- a/db/clickhouse/schema.sql
+++ b/db/clickhouse/schema.sql
@@ -90,6 +90,22 @@ ENGINE = ReplacingMergeTree
ORDER BY (website_id, session_id, data_key)
SETTINGS index_granularity = 8192;
+ALTER TABLE umami.session_data
+ADD PROJECTION session_data_property_filter_projection (
+ SELECT *
+ ORDER BY (
+ website_id,
+ data_key,
+ data_type,
+ string_value,
+ number_value,
+ date_value,
+ session_id
+ )
+);
+
+ALTER TABLE umami.session_data MATERIALIZE PROJECTION session_data_property_filter_projection;
+
-- stats hourly
CREATE TABLE umami.website_event_stats_hourly
(
diff --git a/src/lib/clickhouse.ts b/src/lib/clickhouse.ts
index b6fb4f874..025e4273d 100644
--- a/src/lib/clickhouse.ts
+++ b/src/lib/clickhouse.ts
@@ -253,6 +253,10 @@ function getPropertyFilterQuery(
const table = propertyType === 'event' ? 'event_data' : 'session_data final';
const column = propertyType === 'event' ? 'event_id' : 'session_id';
const outerColumn = propertyType === 'event' ? 'event_id' : 'website_event.session_id';
+ const dateFilter =
+ propertyType === 'event'
+ ? `and created_at between {startDate:DateTime64} and {endDate:DateTime64}`
+ : '';
filters.forEach(({ propertyName, dataType, operator, value }, i) => {
const keyParam = `pf_key_${i}`;
@@ -333,15 +337,26 @@ function getPropertyFilterQuery(
}
}
- parts.push(`and ${outerColumn} in (
+ if (propertyType === 'session') {
+ parts.push(`and tuple(website_event.website_id, website_event.session_id) in (
+ select website_id, session_id
+ from ${table}
+ where website_id = {websiteId:UUID}
+ and data_key = {${keyParam}:String}
+ and data_type = ${dataType}
+ and ${condition}
+ )`);
+ } else {
+ parts.push(`and ${outerColumn} in (
select ${column}
from ${table}
where website_id = {websiteId:UUID}
- and created_at between {startDate:DateTime64} and {endDate:DateTime64}
+ ${dateFilter}
and data_key = {${keyParam}:String}
and data_type = ${dataType}
and ${condition}
)`);
+ }
});
return { sql: parts.join('\n'), params };
diff --git a/src/lib/prisma.ts b/src/lib/prisma.ts
index 1da0f81b7..95d585c1e 100644
--- a/src/lib/prisma.ts
+++ b/src/lib/prisma.ts
@@ -289,6 +289,8 @@ function getPropertyFilterQuery(
const column = propertyType === 'event' ? 'website_event_id' : 'session_id';
const outerColumn =
propertyType === 'event' ? 'website_event.event_id' : 'website_event.session_id';
+ const dateFilter =
+ propertyType === 'event' ? `and created_at between {{startDate}} and {{endDate}}` : '';
filters.forEach(({ propertyName, dataType, operator, value }, i) => {
const keyParam = `pf_key_${i}`;
@@ -372,15 +374,27 @@ function getPropertyFilterQuery(
}
}
- parts.push(`and ${outerColumn} in (
+ if (propertyType === 'session') {
+ parts.push(`and exists (
+ select 1
+ from ${table}
+ where website_id = website_event.website_id
+ and session_id = website_event.session_id
+ and data_key = {{${keyParam}}}
+ and data_type = ${dataType}
+ and ${condition}
+ )`);
+ } else {
+ parts.push(`and ${outerColumn} in (
select ${column}
from ${table}
where website_id = {{websiteId::uuid}}
- and created_at between {{startDate}} and {{endDate}}
+ ${dateFilter}
and data_key = {{${keyParam}}}
and data_type = ${dataType}
and ${condition}
)`);
+ }
});
return { sql: parts.join('\n'), params };