Umami is a privacy-first web and product analytics platform. Traffic, campaigns, behavior, conversions, and revenue in one place. No cookies, no surveillance, self-hosted or in the cloud.
TypeScript analyticsaudience-segmentationchartscohort-analysisgoogle-analyticsproduct-analyticsstatisticsuser-journeyweb-analytics
feat(funnels): add per-step event property filters to schema and SQL queries
Rudy B.96ae1de7fd82
diff --git a/src/lib/schema.ts b/src/lib/schema.tsindex 38f7339a1..99daaa780 100644--- a/src/lib/schema.ts+++ b/src/lib/schema.ts@@ -134,6 +134,15 @@ export const funnelReportSchema = z.object({z.object({type: z.enum(['path', 'event']),value: z.string(),+ filters: z+ .array(+ z.object({+ property: z.string().min(1),+ operator: z.enum(['eq', 'neq', 'c', 'dnc']),+ value: z.string(),+ }),+ )+ .optional(),}),).min(2)diff --git a/src/queries/sql/reports/getFunnel.ts b/src/queries/sql/reports/getFunnel.tsindex 4840123fe..a623fb5e5 100644--- a/src/queries/sql/reports/getFunnel.ts+++ b/src/queries/sql/reports/getFunnel.ts@@ -3,11 +3,23 @@ import { CLICKHOUSE, PRISMA, runQuery } from '@/lib/db';import prisma from '@/lib/prisma';import type { QueryFilters } from '@/lib/types';+export interface FunnelStepFilter {+ property: string;+ operator: string;+ value: string;+}++export interface FunnelStep {+ type: string;+ value: string;+ filters?: Array<FunnelStepFilter>;+}+export interface FunnelParameters {startDate: Date;endDate: Date;window: number;- steps: { type: string; value: string }[];+ steps: Array<FunnelStep>;}export interface FunnelResult {@@ -29,7 +41,7 @@ async function relationalQuery(websiteId: string,parameters: FunnelParameters,filters: QueryFilters,-): Promise<FunnelResult[]> {+): Promise<Array<FunnelResult>> {const { startDate, endDate, window, steps } = parameters;const { rawQuery, getAddIntervalQuery, parseFilters } = prisma;const { filterQuery, joinSessionQuery, cohortQuery, queryParams } = parseFilters({@@ -40,16 +52,55 @@ async function relationalQuery(});const { levelOneQuery, levelQuery, sumQuery, params } = getFunnelQuery(steps, window);+ function buildExistsFilters(+ stepIndex: number,+ stepFilters: Array<FunnelStepFilter> | undefined,+ eventAlias: string,+ extraParams: Record<string, string>,+ ): string {+ if (!stepFilters?.length) return '';++ return stepFilters+ .map((f, fi) => {+ const keyParam = `f_${stepIndex}_${fi}_k`;+ const valParam = `f_${stepIndex}_${fi}_v`;+ extraParams[keyParam] = f.property;++ let op = '=';+ let val = f.value;+ if (f.operator === 'neq') op = '!=';+ else if (f.operator === 'c') {+ op = 'ilike';+ val = `%${val}%`;+ } else if (f.operator === 'dnc') {+ op = 'not ilike';+ val = `%${val}%`;+ }+ extraParams[valParam] = val;++ return `and exists (+ select 1 from event_data _ed${stepIndex}_${fi}+ where _ed${stepIndex}_${fi}.website_event_id = ${eventAlias}.event_id+ and _ed${stepIndex}_${fi}.website_id = {{websiteId::uuid}}+ and _ed${stepIndex}_${fi}.data_key = {{${keyParam}}}+ and case when _ed${stepIndex}_${fi}.data_type = 2 then replace(_ed${stepIndex}_${fi}.string_value, '.0000', '') else _ed${stepIndex}_${fi}.string_value end ${op} {{${valParam}}}+ )`;+ })+ .join('\n');+ }+function getFunnelQuery(- steps: { type: string; value: string }[],+ steps: Array<FunnelStep>,window: number,): {levelOneQuery: string;levelQuery: string;sumQuery: string;- params: string[];+ params: Record<string, string>;} {- return steps.reduce(+ const extraParams: Record<string, string> = {};++ const result = steps.reduce((pv, cv, i) => {const levelNumber = i + 1;const startSum = i > 0 ? 'union ' : '';@@ -64,6 +115,16 @@ async function relationalQuery(paramValue = cv.value.replace(/^\*|\*$/g, '%');}+ const existsClause =+ !isURL && cv.filters?.length+ ? buildExistsFilters(+ i,+ cv.filters,+ levelNumber === 1 ? 'website_event' : 'we',+ extraParams,+ )+ : '';+if (levelNumber === 1) {pv.levelOneQuery = `WITH level1 AS (@@ -75,6 +136,7 @@ async function relationalQuery(and website_event.created_at between {{startDate}} and {{endDate}}and ${column} ${operator} {{${i}}}${filterQuery}+ ${existsClause})`;} else {pv.levelQuery += `@@ -90,11 +152,12 @@ async function relationalQuery()}and we.${column} ${operator} {{${i}}}and we.created_at <= {{endDate}}+ ${existsClause})`;}pv.sumQuery += `\n${startSum}select ${levelNumber} as level, count(distinct(session_id)) as count from level${levelNumber}`;- pv.params.push(paramValue);+ pv.params[i] = paramValue;return pv;},@@ -102,9 +165,11 @@ async function relationalQuery(levelOneQuery: '',levelQuery: '',sumQuery: '',- params: [],+ params: {} as Record<string, string>,},);++ return { ...result, params: { ...result.params, ...extraParams } };}return rawQuery(@@ -125,13 +190,7 @@ async function clickhouseQuery(websiteId: string,parameters: FunnelParameters,filters: QueryFilters,-): Promise<- {- value: string;- visitors: number;- dropoff: number;- }[]-> {+): Promise<Array<FunnelResult>> {const { startDate, endDate, window, steps } = parameters;const { rawQuery, parseFilters } = clickhouse;const { levelOneQuery, levelQuery, sumQuery, stepFilterQuery, params } = getFunnelQuery(@@ -145,8 +204,43 @@ async function clickhouseQuery(endDate,});+ function buildEventDataFilters(+ stepIndex: number,+ stepFilters: Array<FunnelStepFilter> | undefined,+ params: Record<string, string>,+ ): string {+ if (!stepFilters?.length) return '';++ return stepFilters+ .map((f, fi) => {+ const keyParam = `f_${stepIndex}_${fi}_k`;+ const valParam = `f_${stepIndex}_${fi}_v`;+ params[keyParam] = f.property;++ let op = '=';+ let val = f.value;+ if (f.operator === 'neq') op = '!=';+ else if (f.operator === 'c') {+ op = 'like';+ val = `%${val}%`;+ } else if (f.operator === 'dnc') {+ op = 'not like';+ val = `%${val}%`;+ }+ params[valParam] = val;++ return `and event_id in (+ select event_id from event_data+ where website_id = {websiteId:UUID}+ and data_key = {${keyParam}:String}+ and multiIf(data_type = 2, replaceAll(string_value, '.0000', ''), string_value) ${op} {${valParam}:String}+ )`;+ })+ .join('\n');+ }+function getFunnelQuery(- steps: { type: string; value: string }[],+ steps: Array<FunnelStep>,window: number,): {levelOneQuery: string;@@ -155,7 +249,9 @@ async function clickhouseQuery(stepFilterQuery: string;params: Record<string, string>;} {- return steps.reduce(+ const extraParams: Record<string, string> = {};++ const result = steps.reduce((pv, cv, i) => {const levelNumber = i + 1;const startSum = i > 0 ? 'union all ' : '';@@ -171,12 +267,16 @@ async function clickhouseQuery(paramValue = cv.value.replace(/^\*|\*$/g, '%');}+ const eventDataClause =+ !isURL && cv.filters?.length ? buildEventDataFilters(i, cv.filters, extraParams) : '';+if (levelNumber === 1) {pv.levelOneQuery = `\nlevel1 AS (select *from level0where ${column} ${operator} {param${i}:String}+ ${eventDataClause})`;} else {pv.levelQuery += `\n@@ -185,12 +285,14 @@ async function clickhouseQuery(y.url_path as url_path,y.referrer_path as referrer_path,y.event_name,+ y.event_id,y.created_at as created_atfrom level${i} xjoin level0 yon x.session_id = y.session_idwhere y.created_at between x.created_at and x.created_at + interval ${window} minuteand y.${column} ${operator} {param${i}:String}+ ${eventDataClause})`;}@@ -205,15 +307,17 @@ async function clickhouseQuery(levelQuery: '',sumQuery: '',stepFilterQuery: '',- params: {},+ params: {} as Record<string, string>,},);++ return { ...result, params: { ...result.params, ...extraParams } };}return rawQuery(`WITH level0 AS (- select distinct session_id, url_path, referrer_path, event_name, created_at+ select distinct event_id, session_id, url_path, referrer_path, event_name, created_atfrom website_event${cohortQuery}where (${stepFilterQuery})@@ -225,7 +329,7 @@ async function clickhouseQuery(${levelQuery}select *from (- ${sumQuery}+ ${sumQuery}) ORDER BY level;`,{@@ -235,8 +339,8 @@ async function clickhouseQuery().then(formatResults(steps));}-const formatResults = (steps: { type: string; value: string }[]) => (results: unknown) => {- return steps.map((step: { type: string; value: string }, i: number) => {+const formatResults = (steps: Array<FunnelStep>) => (results: unknown) => {+ return steps.map((step: FunnelStep, i: number) => {const visitors = Number(results[i]?.count) || 0;const previous = Number(results[i - 1]?.count) || 0;const dropped = previous > 0 ? previous - visitors : 0;