Umami is a privacy-first web and product analytics platform. Traffic, campaigns, behavior, conversions, and revenue in one place. No cookies, no surveillance, self-hosted or in the cloud.
TypeScript analyticsaudience-segmentationchartscohort-analysisgoogle-analyticsproduct-analyticsstatisticsuser-journeyweb-analytics
query optimizations on session property filters
Francis Caoa113f5be79ed
diff --git a/db/clickhouse/migrations/11_add_event_session_data_pivot.sql b/db/clickhouse/migrations/11_add_event_session_data_pivot.sqlindex 2d44cf0bf..388649859 100644--- a/db/clickhouse/migrations/11_add_event_session_data_pivot.sql+++ b/db/clickhouse/migrations/11_add_event_session_data_pivot.sql@@ -108,3 +108,19 @@ SELECTgroupArrayState(data_type)FROM umami.session_dataGROUP BY website_id, session_id, distinct_id;++ALTER TABLE umami.session_data+ADD PROJECTION session_data_property_filter_projection (+ SELECT *+ ORDER BY (+ website_id,+ data_key,+ data_type,+ string_value,+ number_value,+ date_value,+ session_id+ )+);++ALTER TABLE umami.session_data MATERIALIZE PROJECTION session_data_property_filter_projection;diff --git a/db/clickhouse/schema.sql b/db/clickhouse/schema.sqlindex c5c8dce67..d4041dde9 100644--- a/db/clickhouse/schema.sql+++ b/db/clickhouse/schema.sql@@ -90,6 +90,22 @@ ENGINE = ReplacingMergeTreeORDER BY (website_id, session_id, data_key)SETTINGS index_granularity = 8192;+ALTER TABLE umami.session_data+ADD PROJECTION session_data_property_filter_projection (+ SELECT *+ ORDER BY (+ website_id,+ data_key,+ data_type,+ string_value,+ number_value,+ date_value,+ session_id+ )+);++ALTER TABLE umami.session_data MATERIALIZE PROJECTION session_data_property_filter_projection;+-- stats hourlyCREATE TABLE umami.website_event_stats_hourly(diff --git a/src/lib/clickhouse.ts b/src/lib/clickhouse.tsindex b6fb4f874..025e4273d 100644--- a/src/lib/clickhouse.ts+++ b/src/lib/clickhouse.ts@@ -253,6 +253,10 @@ function getPropertyFilterQuery(const table = propertyType === 'event' ? 'event_data' : 'session_data final';const column = propertyType === 'event' ? 'event_id' : 'session_id';const outerColumn = propertyType === 'event' ? 'event_id' : 'website_event.session_id';+ const dateFilter =+ propertyType === 'event'+ ? `and created_at between {startDate:DateTime64} and {endDate:DateTime64}`+ : '';filters.forEach(({ propertyName, dataType, operator, value }, i) => {const keyParam = `pf_key_${i}`;@@ -333,15 +337,26 @@ function getPropertyFilterQuery(}}- parts.push(`and ${outerColumn} in (+ if (propertyType === 'session') {+ parts.push(`and tuple(website_event.website_id, website_event.session_id) in (+ select website_id, session_id+ from ${table}+ where website_id = {websiteId:UUID}+ and data_key = {${keyParam}:String}+ and data_type = ${dataType}+ and ${condition}+ )`);+ } else {+ parts.push(`and ${outerColumn} in (select ${column}from ${table}where website_id = {websiteId:UUID}- and created_at between {startDate:DateTime64} and {endDate:DateTime64}+ ${dateFilter}and data_key = {${keyParam}:String}and data_type = ${dataType}and ${condition})`);+ }});return { sql: parts.join('\n'), params };diff --git a/src/lib/prisma.ts b/src/lib/prisma.tsindex 1da0f81b7..95d585c1e 100644--- a/src/lib/prisma.ts+++ b/src/lib/prisma.ts@@ -289,6 +289,8 @@ function getPropertyFilterQuery(const column = propertyType === 'event' ? 'website_event_id' : 'session_id';const outerColumn =propertyType === 'event' ? 'website_event.event_id' : 'website_event.session_id';+ const dateFilter =+ propertyType === 'event' ? `and created_at between {{startDate}} and {{endDate}}` : '';filters.forEach(({ propertyName, dataType, operator, value }, i) => {const keyParam = `pf_key_${i}`;@@ -372,15 +374,27 @@ function getPropertyFilterQuery(}}- parts.push(`and ${outerColumn} in (+ if (propertyType === 'session') {+ parts.push(`and exists (+ select 1+ from ${table}+ where website_id = website_event.website_id+ and session_id = website_event.session_id+ and data_key = {{${keyParam}}}+ and data_type = ${dataType}+ and ${condition}+ )`);+ } else {+ parts.push(`and ${outerColumn} in (select ${column}from ${table}where website_id = {{websiteId::uuid}}- and created_at between {{startDate}} and {{endDate}}+ ${dateFilter}and data_key = {{${keyParam}}}and data_type = ${dataType}and ${condition})`);+ }});return { sql: parts.join('\n'), params };