Skip to content

armgit / umami

Public Imported from GitHub
0

Umami is a privacy-first web and product analytics platform. Traffic, campaigns, behavior, conversions, and revenue in one place. No cookies, no surveillance, self-hosted or in the cloud.

feat(funnels): add per-step event property filters to schema and SQL queries

Rudy B.96ae1de7fd82
diff --git a/src/lib/schema.ts b/src/lib/schema.ts
index 38f7339a1..99daaa780 100644
--- a/src/lib/schema.ts
+++ b/src/lib/schema.ts
@@ -134,6 +134,15 @@ export const funnelReportSchema = z.object({
z.object({
type: z.enum(['path', 'event']),
value: z.string(),
+ filters: z
+ .array(
+ z.object({
+ property: z.string().min(1),
+ operator: z.enum(['eq', 'neq', 'c', 'dnc']),
+ value: z.string(),
+ }),
+ )
+ .optional(),
}),
)
.min(2)
diff --git a/src/queries/sql/reports/getFunnel.ts b/src/queries/sql/reports/getFunnel.ts
index 4840123fe..a623fb5e5 100644
--- a/src/queries/sql/reports/getFunnel.ts
+++ b/src/queries/sql/reports/getFunnel.ts
@@ -3,11 +3,23 @@ import { CLICKHOUSE, PRISMA, runQuery } from '@/lib/db';
import prisma from '@/lib/prisma';
import type { QueryFilters } from '@/lib/types';
+export interface FunnelStepFilter {
+ property: string;
+ operator: string;
+ value: string;
+}
+
+export interface FunnelStep {
+ type: string;
+ value: string;
+ filters?: Array<FunnelStepFilter>;
+}
+
export interface FunnelParameters {
startDate: Date;
endDate: Date;
window: number;
- steps: { type: string; value: string }[];
+ steps: Array<FunnelStep>;
}
export interface FunnelResult {
@@ -29,7 +41,7 @@ async function relationalQuery(
websiteId: string,
parameters: FunnelParameters,
filters: QueryFilters,
-): Promise<FunnelResult[]> {
+): Promise<Array<FunnelResult>> {
const { startDate, endDate, window, steps } = parameters;
const { rawQuery, getAddIntervalQuery, parseFilters } = prisma;
const { filterQuery, joinSessionQuery, cohortQuery, queryParams } = parseFilters({
@@ -40,16 +52,55 @@ async function relationalQuery(
});
const { levelOneQuery, levelQuery, sumQuery, params } = getFunnelQuery(steps, window);
+ function buildExistsFilters(
+ stepIndex: number,
+ stepFilters: Array<FunnelStepFilter> | undefined,
+ eventAlias: string,
+ extraParams: Record<string, string>,
+ ): string {
+ if (!stepFilters?.length) return '';
+
+ return stepFilters
+ .map((f, fi) => {
+ const keyParam = `f_${stepIndex}_${fi}_k`;
+ const valParam = `f_${stepIndex}_${fi}_v`;
+ extraParams[keyParam] = f.property;
+
+ let op = '=';
+ let val = f.value;
+ if (f.operator === 'neq') op = '!=';
+ else if (f.operator === 'c') {
+ op = 'ilike';
+ val = `%${val}%`;
+ } else if (f.operator === 'dnc') {
+ op = 'not ilike';
+ val = `%${val}%`;
+ }
+ extraParams[valParam] = val;
+
+ return `and exists (
+ select 1 from event_data _ed${stepIndex}_${fi}
+ where _ed${stepIndex}_${fi}.website_event_id = ${eventAlias}.event_id
+ and _ed${stepIndex}_${fi}.website_id = {{websiteId::uuid}}
+ and _ed${stepIndex}_${fi}.data_key = {{${keyParam}}}
+ and case when _ed${stepIndex}_${fi}.data_type = 2 then replace(_ed${stepIndex}_${fi}.string_value, '.0000', '') else _ed${stepIndex}_${fi}.string_value end ${op} {{${valParam}}}
+ )`;
+ })
+ .join('\n');
+ }
+
function getFunnelQuery(
- steps: { type: string; value: string }[],
+ steps: Array<FunnelStep>,
window: number,
): {
levelOneQuery: string;
levelQuery: string;
sumQuery: string;
- params: string[];
+ params: Record<string, string>;
} {
- return steps.reduce(
+ const extraParams: Record<string, string> = {};
+
+ const result = steps.reduce(
(pv, cv, i) => {
const levelNumber = i + 1;
const startSum = i > 0 ? 'union ' : '';
@@ -64,6 +115,16 @@ async function relationalQuery(
paramValue = cv.value.replace(/^\*|\*$/g, '%');
}
+ const existsClause =
+ !isURL && cv.filters?.length
+ ? buildExistsFilters(
+ i,
+ cv.filters,
+ levelNumber === 1 ? 'website_event' : 'we',
+ extraParams,
+ )
+ : '';
+
if (levelNumber === 1) {
pv.levelOneQuery = `
WITH level1 AS (
@@ -75,6 +136,7 @@ async function relationalQuery(
and website_event.created_at between {{startDate}} and {{endDate}}
and ${column} ${operator} {{${i}}}
${filterQuery}
+ ${existsClause}
)`;
} else {
pv.levelQuery += `
@@ -90,11 +152,12 @@ async function relationalQuery(
)}
and we.${column} ${operator} {{${i}}}
and we.created_at <= {{endDate}}
+ ${existsClause}
)`;
}
pv.sumQuery += `\n${startSum}select ${levelNumber} as level, count(distinct(session_id)) as count from level${levelNumber}`;
- pv.params.push(paramValue);
+ pv.params[i] = paramValue;
return pv;
},
@@ -102,9 +165,11 @@ async function relationalQuery(
levelOneQuery: '',
levelQuery: '',
sumQuery: '',
- params: [],
+ params: {} as Record<string, string>,
},
);
+
+ return { ...result, params: { ...result.params, ...extraParams } };
}
return rawQuery(
@@ -125,13 +190,7 @@ async function clickhouseQuery(
websiteId: string,
parameters: FunnelParameters,
filters: QueryFilters,
-): Promise<
- {
- value: string;
- visitors: number;
- dropoff: number;
- }[]
-> {
+): Promise<Array<FunnelResult>> {
const { startDate, endDate, window, steps } = parameters;
const { rawQuery, parseFilters } = clickhouse;
const { levelOneQuery, levelQuery, sumQuery, stepFilterQuery, params } = getFunnelQuery(
@@ -145,8 +204,43 @@ async function clickhouseQuery(
endDate,
});
+ function buildEventDataFilters(
+ stepIndex: number,
+ stepFilters: Array<FunnelStepFilter> | undefined,
+ params: Record<string, string>,
+ ): string {
+ if (!stepFilters?.length) return '';
+
+ return stepFilters
+ .map((f, fi) => {
+ const keyParam = `f_${stepIndex}_${fi}_k`;
+ const valParam = `f_${stepIndex}_${fi}_v`;
+ params[keyParam] = f.property;
+
+ let op = '=';
+ let val = f.value;
+ if (f.operator === 'neq') op = '!=';
+ else if (f.operator === 'c') {
+ op = 'like';
+ val = `%${val}%`;
+ } else if (f.operator === 'dnc') {
+ op = 'not like';
+ val = `%${val}%`;
+ }
+ params[valParam] = val;
+
+ return `and event_id in (
+ select event_id from event_data
+ where website_id = {websiteId:UUID}
+ and data_key = {${keyParam}:String}
+ and multiIf(data_type = 2, replaceAll(string_value, '.0000', ''), string_value) ${op} {${valParam}:String}
+ )`;
+ })
+ .join('\n');
+ }
+
function getFunnelQuery(
- steps: { type: string; value: string }[],
+ steps: Array<FunnelStep>,
window: number,
): {
levelOneQuery: string;
@@ -155,7 +249,9 @@ async function clickhouseQuery(
stepFilterQuery: string;
params: Record<string, string>;
} {
- return steps.reduce(
+ const extraParams: Record<string, string> = {};
+
+ const result = steps.reduce(
(pv, cv, i) => {
const levelNumber = i + 1;
const startSum = i > 0 ? 'union all ' : '';
@@ -171,12 +267,16 @@ async function clickhouseQuery(
paramValue = cv.value.replace(/^\*|\*$/g, '%');
}
+ const eventDataClause =
+ !isURL && cv.filters?.length ? buildEventDataFilters(i, cv.filters, extraParams) : '';
+
if (levelNumber === 1) {
pv.levelOneQuery = `\n
level1 AS (
select *
from level0
where ${column} ${operator} {param${i}:String}
+ ${eventDataClause}
)`;
} else {
pv.levelQuery += `\n
@@ -185,12 +285,14 @@ async function clickhouseQuery(
y.url_path as url_path,
y.referrer_path as referrer_path,
y.event_name,
+ y.event_id,
y.created_at as created_at
from level${i} x
join level0 y
on x.session_id = y.session_id
where y.created_at between x.created_at and x.created_at + interval ${window} minute
and y.${column} ${operator} {param${i}:String}
+ ${eventDataClause}
)`;
}
@@ -205,15 +307,17 @@ async function clickhouseQuery(
levelQuery: '',
sumQuery: '',
stepFilterQuery: '',
- params: {},
+ params: {} as Record<string, string>,
},
);
+
+ return { ...result, params: { ...result.params, ...extraParams } };
}
return rawQuery(
`
WITH level0 AS (
- select distinct session_id, url_path, referrer_path, event_name, created_at
+ select distinct event_id, session_id, url_path, referrer_path, event_name, created_at
from website_event
${cohortQuery}
where (${stepFilterQuery})
@@ -225,7 +329,7 @@ async function clickhouseQuery(
${levelQuery}
select *
from (
- ${sumQuery}
+ ${sumQuery}
) ORDER BY level;
`,
{
@@ -235,8 +339,8 @@ async function clickhouseQuery(
).then(formatResults(steps));
}
-const formatResults = (steps: { type: string; value: string }[]) => (results: unknown) => {
- return steps.map((step: { type: string; value: string }, i: number) => {
+const formatResults = (steps: Array<FunnelStep>) => (results: unknown) => {
+ return steps.map((step: FunnelStep, i: number) => {
const visitors = Number(results[i]?.count) || 0;
const previous = Number(results[i - 1]?.count) || 0;
const dropped = previous > 0 ? previous - visitors : 0;