Skip to content

armgit / umami

Public Imported from GitHub
0

Umami is a privacy-first web and product analytics platform. Traffic, campaigns, behavior, conversions, and revenue in one place. No cookies, no surveillance, self-hosted or in the cloud.

Add funnel queries

Brian Caod01aa5cd52df
diff --git a/db/postgresql/migrations/02_report_schema/migration.sql b/db/postgresql/migrations/02_report_schema/migration.sql
new file mode 100644
index 000000000..61f1164d0
--- /dev/null
+++ b/db/postgresql/migrations/02_report_schema/migration.sql
@@ -0,0 +1,18 @@
+-- CreateTable
+CREATE TABLE "report" (
+ "report_id" UUID NOT NULL,
+ "user_id" UUID NOT NULL,
+ "report_name" VARCHAR(200) NOT NULL,
+ "template_name" VARCHAR(200) NOT NULL,
+ "parameters" VARCHAR(6000) NOT NULL,
+ "created_at" TIMESTAMPTZ(6) DEFAULT CURRENT_TIMESTAMP,
+ "updated_at" TIMESTAMPTZ(6),
+
+ CONSTRAINT "report_pkey" PRIMARY KEY ("report_id")
+);
+
+-- CreateIndex
+CREATE UNIQUE INDEX "report_report_id_key" ON "report"("report_id");
+
+-- CreateIndex
+CREATE INDEX "report_user_id_idx" ON "report"("user_id");
diff --git a/db/postgresql/schema.prisma b/db/postgresql/schema.prisma
index b336bce4b..7523ce217 100644
--- a/db/postgresql/schema.prisma
+++ b/db/postgresql/schema.prisma
@@ -14,11 +14,12 @@ model User {
password String @db.VarChar(60)
role String @map("role") @db.VarChar(50)
createdAt DateTime? @default(now()) @map("created_at") @db.Timestamptz(6)
- updatedAt DateTime? @map("updated_at") @updatedAt @db.Timestamptz(6)
+ updatedAt DateTime? @updatedAt @map("updated_at") @db.Timestamptz(6)
deletedAt DateTime? @map("deleted_at") @db.Timestamptz(6)
- website Website[]
- teamUser TeamUser[]
+ website Website[]
+ teamUser TeamUser[]
+ ReportTemplate ReportTemplate[]
@@map("user")
}
@@ -53,7 +54,7 @@ model Website {
resetAt DateTime? @map("reset_at") @db.Timestamptz(6)
userId String? @map("user_id") @db.Uuid
createdAt DateTime? @default(now()) @map("created_at") @db.Timestamptz(6)
- updatedAt DateTime? @map("updated_at") @updatedAt @db.Timestamptz(6)
+ updatedAt DateTime? @updatedAt @map("updated_at") @db.Timestamptz(6)
deletedAt DateTime? @map("deleted_at") @db.Timestamptz(6)
user User? @relation(fields: [userId], references: [id])
@@ -116,7 +117,7 @@ model Team {
name String @db.VarChar(50)
accessCode String? @unique @map("access_code") @db.VarChar(50)
createdAt DateTime? @default(now()) @map("created_at") @db.Timestamptz(6)
- updatedAt DateTime? @map("updated_at") @updatedAt @db.Timestamptz(6)
+ updatedAt DateTime? @updatedAt @map("updated_at") @db.Timestamptz(6)
teamUser TeamUser[]
teamWebsite TeamWebsite[]
@@ -131,7 +132,7 @@ model TeamUser {
userId String @map("user_id") @db.Uuid
role String @map("role") @db.VarChar(50)
createdAt DateTime? @default(now()) @map("created_at") @db.Timestamptz(6)
- updatedAt DateTime? @map("updated_at") @updatedAt @db.Timestamptz(6)
+ updatedAt DateTime? @updatedAt @map("updated_at") @db.Timestamptz(6)
team Team @relation(fields: [teamId], references: [id])
user User @relation(fields: [userId], references: [id])
@@ -154,3 +155,18 @@ model TeamWebsite {
@@index([websiteId])
@@map("team_website")
}
+
+model ReportTemplate {
+ id String @id() @unique() @map("report_id") @db.Uuid
+ userId String @map("user_id") @db.Uuid
+ reportName String @map("report_name") @db.VarChar(200)
+ templateName String @map("template_name") @db.VarChar(200)
+ parameters String @map("parameters") @db.VarChar(6000)
+ createdAt DateTime? @default(now()) @map("created_at") @db.Timestamptz(6)
+ updatedAt DateTime? @updatedAt @map("updated_at") @db.Timestamptz(6)
+
+ user User @relation(fields: [userId], references: [id])
+
+ @@index([userId])
+ @@map("report")
+}
diff --git a/lib/clickhouse.ts b/lib/clickhouse.ts
index 90cf6088a..88922c0f5 100644
--- a/lib/clickhouse.ts
+++ b/lib/clickhouse.ts
@@ -121,6 +121,29 @@ function getFilterQuery(filters = {}, params = {}) {
return query.join('\n');
}
+function getFunnelQuery(urls: string[]): {
+ columnsQuery: string;
+ conditionQuery: string;
+ urlParams: { [key: string]: string };
+} {
+ return urls.reduce(
+ (pv, cv, i) => {
+ pv.columnsQuery += `\n,url_path = {url${i}:String}${
+ i > 0 && urls[i - 1] ? ` AND request_url = {url${i - 1}:String}` : ''
+ },'`;
+ pv.conditionQuery += `${i > 0 ? ',' : ''} {url${i}:String}`;
+ pv.urlParams[`url${i}`] = cv;
+
+ return pv;
+ },
+ {
+ columnsQuery: '',
+ conditionQuery: '',
+ urlParams: {},
+ },
+ );
+}
+
function parseFilters(filters: WebsiteMetricFilter = {}, params: any = {}) {
return {
filterQuery: getFilterQuery(filters, params),
@@ -168,6 +191,7 @@ export default {
getDateFormat,
getBetweenDates,
getFilterQuery,
+ getFunnelQuery,
getEventDataFilterQuery,
parseFilters,
findUnique,
diff --git a/lib/prisma.ts b/lib/prisma.ts
index 0a10d9816..ce2238b6a 100644
--- a/lib/prisma.ts
+++ b/lib/prisma.ts
@@ -32,6 +32,18 @@ function toUuid(): string {
}
}
+function getAddMinutesQuery(field: string, minutes: number) {
+ const db = getDatabaseType(process.env.DATABASE_URL);
+
+ if (db === POSTGRESQL) {
+ return `${field} + interval '${minutes} minute'`;
+ }
+
+ if (db === MYSQL) {
+ return `DATE_ADD(${field}, interval ${minutes} minute)`;
+ }
+}
+
function getDateQuery(field: string, unit: string, timezone?: string): string {
const db = getDatabaseType(process.env.DATABASE_URL);
@@ -122,6 +134,48 @@ function getFilterQuery(filters = {}, params = []): string {
return query.join('\n');
}
+function getFunnelQuery(
+ urls: string[],
+ windowMinutes: number,
+ initParamLength = 3,
+): {
+ levelQuery: string;
+ sumQuery: string;
+ urlFilterQuery: string;
+} {
+ return urls.reduce(
+ (pv, cv, i) => {
+ const levelNumber = i + 1;
+ const start = i > 0 ? ',' : '';
+
+ pv.levelQuery += `\n
+ , level${levelNumber} AS (
+ select cl.*,
+ l0.created_at level_${levelNumber}_created_at,
+ l0.url_path as level_${levelNumber}_url
+ from level${i} cl
+ left join level0 l0
+ on cl.session_id = l0.session_id
+ and l0.created_at between cl.level_${levelNumber}_created_at
+ and ${getAddMinutesQuery(`cl.level_${levelNumber}_created_at`, windowMinutes)}
+ and l0.referrer_path = $${i + initParamLength}
+ and l0.url_path = $${i + initParamLength}
+ )`;
+
+ pv.sumQuery += `\n${start}SUM(CASE WHEN l1_url is not null THEN 1 ELSE 0 END) AS level1`;
+
+ pv.urlFilterQuery += `\n${start}$${levelNumber + initParamLength} `;
+
+ return pv;
+ },
+ {
+ levelQuery: '',
+ sumQuery: '',
+ urlFilterQuery: '',
+ },
+ );
+}
+
function parseFilters(
filters: { [key: string]: any } = {},
params = [],
@@ -152,9 +206,11 @@ async function rawQuery(query: string, params: never[] = []): Promise<any> {
export default {
...prisma,
+ getAddMinutesQuery,
getDateQuery,
getTimestampInterval,
getFilterQuery,
+ getFunnelQuery,
getEventDataFilterQuery,
toUuid,
parseFilters,
diff --git a/pages/api/reports/funnel.ts b/pages/api/reports/funnel.ts
new file mode 100644
index 000000000..080130de7
--- /dev/null
+++ b/pages/api/reports/funnel.ts
@@ -0,0 +1,50 @@
+import { canViewWebsite } from 'lib/auth';
+import { useCors, useAuth } from 'lib/middleware';
+import { NextApiRequestQueryBody } from 'lib/types';
+import { NextApiResponse } from 'next';
+import { ok, methodNotAllowed, unauthorized } from 'next-basics';
+import { getPageviewFunnel } from 'queries';
+
+export interface FunnelRequestBody {
+ websiteId: string;
+ urls: string[];
+ window: number;
+ startAt: number;
+ endAt: number;
+}
+
+export interface FunnelResponse {
+ urls: string[];
+ window: number;
+ startAt: number;
+ endAt: number;
+}
+
+export default async (
+ req: NextApiRequestQueryBody<any, FunnelRequestBody>,
+ res: NextApiResponse<FunnelResponse>,
+) => {
+ await useCors(req, res);
+ await useAuth(req, res);
+
+ if (req.method === 'POST') {
+ const { websiteId, urls, window, startAt, endAt } = req.body;
+ if (!(await canViewWebsite(req.auth, websiteId))) {
+ return unauthorized(res);
+ }
+
+ const startDate = new Date(+startAt);
+ const endDate = new Date(+endAt);
+
+ const data = getPageviewFunnel(websiteId, {
+ startDate,
+ endDate,
+ urls,
+ windowMinutes: window,
+ });
+
+ return ok(res);
+ }
+
+ return methodNotAllowed(res);
+};
diff --git a/queries/analytics/pageview/getPageviewFunnel.ts b/queries/analytics/pageview/getPageviewFunnel.ts
new file mode 100644
index 000000000..f77516306
--- /dev/null
+++ b/queries/analytics/pageview/getPageviewFunnel.ts
@@ -0,0 +1,97 @@
+import clickhouse from 'lib/clickhouse';
+import { CLICKHOUSE, PRISMA, runQuery } from 'lib/db';
+import prisma from 'lib/prisma';
+
+export async function getPageviewFunnel(
+ ...args: [
+ websiteId: string,
+ criteria: {
+ windowMinutes: number;
+ startDate: Date;
+ endDate: Date;
+ urls: string[];
+ },
+ ]
+) {
+ return runQuery({
+ [PRISMA]: () => relationalQuery(...args),
+ [CLICKHOUSE]: () => clickhouseQuery(...args),
+ });
+}
+
+async function relationalQuery(
+ websiteId: string,
+ criteria: {
+ windowMinutes: number;
+ startDate: Date;
+ endDate: Date;
+ urls: string[];
+ },
+) {
+ const { windowMinutes, startDate, endDate, urls } = criteria;
+ const { rawQuery, getFunnelQuery, toUuid } = prisma;
+ const { levelQuery, sumQuery, urlFilterQuery } = getFunnelQuery(urls, windowMinutes);
+
+ const params: any = [websiteId, startDate, endDate, ...urls];
+
+ return rawQuery(
+ `WITH level0 AS (
+ select session_id, url_path, created_at
+ from website_event
+ where url_path in (${urlFilterQuery})
+ website_event.website_id = $1${toUuid()}
+ and created_at between $2 and $3
+ ),level1 AS (
+ select session_id, url_path as level1_url, created_at as level1_created_at
+ from level0
+ )${levelQuery}
+
+ SELECT ${sumQuery}
+ from level3;
+ `,
+ params,
+ );
+}
+
+async function clickhouseQuery(
+ websiteId: string,
+ criteria: {
+ windowMinutes: number;
+ startDate: Date;
+ endDate: Date;
+ urls: string[];
+ },
+) {
+ const { windowMinutes, startDate, endDate, urls } = criteria;
+ const { rawQuery, getBetweenDates, getFunnelQuery } = clickhouse;
+ const { columnsQuery, conditionQuery, urlParams } = getFunnelQuery(urls);
+
+ const params = {
+ websiteId,
+ window: windowMinutes * 60,
+ ...urlParams,
+ };
+
+ return rawQuery(
+ `
+ SELECT level,
+ count(*) AS count
+ FROM (
+ SELECT session_id,
+ windowFunnel({window:UInt32}, 'strict_order')
+ (
+ created_at,
+ ${columnsQuery}
+ ) AS level
+ FROM website_event
+ WHERE website_id = {websiteId:UUID}
+ and ${getBetweenDates('created_at', startDate, endDate)}
+ AND (url_path in [${conditionQuery}])
+ GROUP BY 1
+ )
+ GROUP BY level
+ ORDER BY level ASC;
+ `,
+ params,
+ );
+}
diff --git a/queries/index.js b/queries/index.js
index d87d5dd5f..5c295fff0 100644
--- a/queries/index.js
+++ b/queries/index.js
@@ -8,6 +8,7 @@ export * from './analytics/event/getEvents';
export * from './analytics/eventData/getEventData';
export * from './analytics/eventData/getEventDataUsage';
export * from './analytics/event/saveEvent';
+export * from './analytics/pageview/getPageviewFunnel';
export * from './analytics/pageview/getPageviewMetrics';
export * from './analytics/pageview/getPageviewStats';
export * from './analytics/session/createSession';