Umami is a privacy-first web and product analytics platform. Traffic, campaigns, behavior, conversions, and revenue in one place. No cookies, no surveillance, self-hosted or in the cloud.
TypeScript analyticsaudience-segmentationchartscohort-analysisgoogle-analyticsproduct-analyticsstatisticsuser-journeyweb-analytics
build out retention reports
Francis Cao9cde107ddfa8
diff --git a/components/messages.js b/components/messages.jsindex a31e28751..68e3b3d59 100644--- a/components/messages.js+++ b/components/messages.js@@ -161,6 +161,7 @@ export const labels = defineMessages({overview: { id: 'labels.overview', defaultMessage: 'Overview' },totalRecords: { id: 'labels.total-records', defaultMessage: 'Total records' },insights: { id: 'label.insights', defaultMessage: 'Insights' },+ retention: { id: 'label.retention', defaultMessage: 'Retention' },dropoff: { id: 'label.dropoff', defaultMessage: 'Dropoff' },});diff --git a/components/pages/reports/ReportDetails.js b/components/pages/reports/ReportDetails.jsindex c41d12f6a..39cd285d7 100644--- a/components/pages/reports/ReportDetails.js+++ b/components/pages/reports/ReportDetails.js@@ -1,9 +1,11 @@import FunnelReport from './funnel/FunnelReport';import EventDataReport from './event-data/EventDataReport';+import RetentionReport from './retention/RetentionReport';const reports = {funnel: FunnelReport,'event-data': EventDataReport,+ retention: RetentionReport,};export default function ReportDetails({ reportId, reportType }) {diff --git a/components/pages/reports/ReportTemplates.js b/components/pages/reports/ReportTemplates.jsindex 60ae11e79..29c193a86 100644--- a/components/pages/reports/ReportTemplates.js+++ b/components/pages/reports/ReportTemplates.js@@ -47,6 +47,12 @@ export function ReportTemplates() {url: '/reports/funnel',icon: <Funnel />,},+ {+ title: formatMessage(labels.retention),+ description: 'Track your websites user retention',+ url: '/reports/retention',+ icon: <Funnel />,+ },];return (diff --git a/components/pages/reports/funnel/FunnelChart.js b/components/pages/reports/funnel/FunnelChart.jsindex 7253c3fa5..c35afe4e6 100644--- a/components/pages/reports/funnel/FunnelChart.js+++ b/components/pages/reports/funnel/FunnelChart.js@@ -1,5 +1,5 @@import { useCallback, useContext, useMemo } from 'react';-import { Loading } from 'react-basics';+import { Loading, StatusLight } from 'react-basics';import useMessages from 'hooks/useMessages';import useTheme from 'hooks/useTheme';import BarChart from 'components/metrics/BarChart';@@ -22,14 +22,25 @@ export function FunnelChart({ className, loading }) {);const renderTooltipPopup = useCallback((setTooltipPopup, model) => {- const { opacity, dataPoints } = model.tooltip;+ const { opacity, labelColors, dataPoints } = model.tooltip;if (!dataPoints?.length || !opacity) {setTooltipPopup(null);return;}- setTooltipPopup(`${formatLongNumber(dataPoints[0].raw.y)} ${formatMessage(labels.visitors)}`);+ setTooltipPopup(+ <>+ <div>+ {formatLongNumber(dataPoints[0].raw.y)} {formatMessage(labels.visitors)}+ </div>+ <div>+ <StatusLight color={labelColors?.[0]?.backgroundColor}>+ {formatLongNumber(dataPoints[0].raw.z)}% {formatMessage(labels.dropoff)}+ </StatusLight>+ </div>+ </>,+ );}, []);const datasets = useMemo(() => {diff --git a/components/pages/reports/retention/RetentionChart.js b/components/pages/reports/retention/RetentionChart.jsnew file mode 100644index 000000000..5f7361fdb--- /dev/null+++ b/components/pages/reports/retention/RetentionChart.js@@ -0,0 +1,74 @@+import { useCallback, useContext, useMemo } from 'react';+import { Loading, StatusLight } from 'react-basics';+import useMessages from 'hooks/useMessages';+import useTheme from 'hooks/useTheme';+import BarChart from 'components/metrics/BarChart';+import { formatLongNumber } from 'lib/format';+import styles from './RetentionChart.module.css';+import { ReportContext } from '../Report';++export function RetentionChart({ className, loading }) {+ const { report } = useContext(ReportContext);+ const { formatMessage, labels } = useMessages();+ const { colors } = useTheme();++ const { parameters, data } = report || {};++ const renderXLabel = useCallback(+ (label, index) => {+ return parameters.urls[index];+ },+ [parameters],+ );++ const renderTooltipPopup = useCallback((setTooltipPopup, model) => {+ const { opacity, labelColors, dataPoints } = model.tooltip;++ if (!dataPoints?.length || !opacity) {+ setTooltipPopup(null);+ return;+ }++ setTooltipPopup(+ <>+ <div>+ {formatLongNumber(dataPoints[0].raw.y)} {formatMessage(labels.visitors)}+ </div>+ <div>+ <StatusLight color={labelColors?.[0]?.backgroundColor}>+ {formatLongNumber(dataPoints[0].raw.z)}% {formatMessage(labels.dropoff)}+ </StatusLight>+ </div>+ </>,+ );+ }, []);++ const datasets = useMemo(() => {+ return [+ {+ label: formatMessage(labels.uniqueVisitors),+ data: data,+ borderWidth: 1,+ ...colors.chart.visitors,+ },+ ];+ }, [data]);++ if (loading) {+ return <Loading icon="dots" className={styles.loading} />;+ }++ return (+ <BarChart+ className={className}+ datasets={datasets}+ unit="day"+ loading={loading}+ renderXLabel={renderXLabel}+ renderTooltipPopup={renderTooltipPopup}+ XAxisType="category"+ />+ );+}++export default RetentionChart;diff --git a/components/pages/reports/retention/RetentionChart.module.css b/components/pages/reports/retention/RetentionChart.module.cssnew file mode 100644index 000000000..9e1690b31--- /dev/null+++ b/components/pages/reports/retention/RetentionChart.module.css@@ -0,0 +1,3 @@+.loading {+ height: 300px;+}diff --git a/components/pages/reports/retention/RetentionParameters.js b/components/pages/reports/retention/RetentionParameters.jsnew file mode 100644index 000000000..29c0eff2f--- /dev/null+++ b/components/pages/reports/retention/RetentionParameters.js@@ -0,0 +1,44 @@+import { useContext, useRef } from 'react';+import { useMessages } from 'hooks';+import { Form, FormButtons, FormInput, FormRow, SubmitButton, TextField } from 'react-basics';+import { ReportContext } from 'components/pages/reports/Report';+import BaseParameters from '../BaseParameters';++export function RetentionParameters() {+ const { report, runReport, isRunning } = useContext(ReportContext);+ const { formatMessage, labels } = useMessages();+ const ref = useRef(null);++ const { parameters } = report || {};+ const { websiteId, dateRange } = parameters || {};+ const queryDisabled = !websiteId || !dateRange;++ const handleSubmit = (data, e) => {+ e.stopPropagation();+ e.preventDefault();+ if (!queryDisabled) {+ runReport(data);+ }+ };++ return (+ <Form ref={ref} values={parameters} onSubmit={handleSubmit} preventSubmit={true}>+ <BaseParameters />+ <FormRow label={formatMessage(labels.window)}>+ <FormInput+ name="window"+ rules={{ required: formatMessage(labels.required), pattern: /[0-9]+/ }}+ >+ <TextField autoComplete="off" />+ </FormInput>+ </FormRow>+ <FormButtons>+ <SubmitButton variant="primary" disabled={queryDisabled} loading={isRunning}>+ {formatMessage(labels.runQuery)}+ </SubmitButton>+ </FormButtons>+ </Form>+ );+}++export default RetentionParameters;diff --git a/components/pages/reports/retention/RetentionReport.js b/components/pages/reports/retention/RetentionReport.jsnew file mode 100644index 000000000..31d085f7b--- /dev/null+++ b/components/pages/reports/retention/RetentionReport.js@@ -0,0 +1,28 @@+import RetentionChart from './RetentionChart';+import RetentionTable from './RetentionTable';+import RetentionParameters from './RetentionParameters';+import Report from '../Report';+import ReportHeader from '../ReportHeader';+import ReportMenu from '../ReportMenu';+import ReportBody from '../ReportBody';+import Funnel from 'assets/funnel.svg';++const defaultParameters = {+ type: 'Retention',+ parameters: { window: 60, urls: [] },+};++export default function RetentionReport({ reportId }) {+ return (+ <Report reportId={reportId} defaultParameters={defaultParameters}>+ <ReportHeader icon={<Funnel />} />+ <ReportMenu>+ <RetentionParameters />+ </ReportMenu>+ <ReportBody>+ <RetentionChart />+ <RetentionTable />+ </ReportBody>+ </Report>+ );+}diff --git a/components/pages/reports/retention/RetentionReport.module.css b/components/pages/reports/retention/RetentionReport.module.cssnew file mode 100644index 000000000..aed66b74e--- /dev/null+++ b/components/pages/reports/retention/RetentionReport.module.css@@ -0,0 +1,10 @@+.filters {+ display: flex;+ flex-direction: column;+ justify-content: space-between;+ border: 1px solid var(--base400);+ border-radius: var(--border-radius);+ line-height: 32px;+ padding: 10px;+ overflow: hidden;+}diff --git a/components/pages/reports/retention/RetentionTable.js b/components/pages/reports/retention/RetentionTable.jsnew file mode 100644index 000000000..4ef879862--- /dev/null+++ b/components/pages/reports/retention/RetentionTable.js@@ -0,0 +1,19 @@+import { useContext } from 'react';+import DataTable from 'components/metrics/DataTable';+import { useMessages } from 'hooks';+import { ReportContext } from '../Report';++export function RetentionTable() {+ const { report } = useContext(ReportContext);+ const { formatMessage, labels } = useMessages();+ return (+ <DataTable+ data={report?.data}+ title={formatMessage(labels.url)}+ metric={formatMessage(labels.visitors)}+ showPercentage={true}+ />+ );+}++export default RetentionTable;diff --git a/pages/api/reports/retention.ts b/pages/api/reports/retention.tsnew file mode 100644index 000000000..6b8aebcc3--- /dev/null+++ b/pages/api/reports/retention.ts@@ -0,0 +1,55 @@+import { canViewWebsite } from 'lib/auth';+import { useCors, useAuth } from 'lib/middleware';+import { NextApiRequestQueryBody } from 'lib/types';+import { NextApiResponse } from 'next';+import { ok, methodNotAllowed, unauthorized } from 'next-basics';+import { getRetention } from 'queries';++export interface RetentionRequestBody {+ websiteId: string;+ urls: string[];+ window: number;+ dateRange: {+ startDate: string;+ endDate: string;+ };+}++export interface RetentionResponse {+ urls: string[];+ window: number;+ startAt: number;+ endAt: number;+}++export default async (+ req: NextApiRequestQueryBody<any, RetentionRequestBody>,+ res: NextApiResponse<RetentionResponse>,+) => {+ await useCors(req, res);+ await useAuth(req, res);++ if (req.method === 'POST') {+ const {+ websiteId,+ urls,+ window,+ dateRange: { startDate, endDate },+ } = req.body;++ if (!(await canViewWebsite(req.auth, websiteId))) {+ return unauthorized(res);+ }++ const data = await getRetention(websiteId, {+ startDate: new Date(startDate),+ endDate: new Date(endDate),+ urls,+ windowMinutes: +window,+ });++ return ok(res, data);+ }++ return methodNotAllowed(res);+};diff --git a/pages/reports/retention.js b/pages/reports/retention.jsnew file mode 100644index 000000000..b7f0bd0f8--- /dev/null+++ b/pages/reports/retention.js@@ -0,0 +1,13 @@+import AppLayout from 'components/layout/AppLayout';+import RetentionReport from 'components/pages/reports/retention/RetentionReport';+import useMessages from 'hooks/useMessages';++export default function () {+ const { formatMessage, labels } = useMessages();++ return (+ <AppLayout title={`${formatMessage(labels.retention)} - ${formatMessage(labels.reports)}`}>+ <RetentionReport />+ </AppLayout>+ );+}diff --git a/queries/analytics/reports/getRetention.ts b/queries/analytics/reports/getRetention.tsnew file mode 100644index 000000000..b2c478827--- /dev/null+++ b/queries/analytics/reports/getRetention.ts@@ -0,0 +1,209 @@+import clickhouse from 'lib/clickhouse';+import { CLICKHOUSE, PRISMA, runQuery } from 'lib/db';+import prisma from 'lib/prisma';++export async function getRetention(+ ...args: [+ websiteId: string,+ criteria: {+ windowMinutes: number;+ startDate: Date;+ endDate: Date;+ urls: string[];+ },+ ]+) {+ return runQuery({+ [PRISMA]: () => relationalQuery(...args),+ [CLICKHOUSE]: () => clickhouseQuery(...args),+ });+}++async function relationalQuery(+ websiteId: string,+ criteria: {+ windowMinutes: number;+ startDate: Date;+ endDate: Date;+ urls: string[];+ },+): Promise<+ {+ x: string;+ y: number;+ z: number;+ }[]+> {+ const { windowMinutes, startDate, endDate, urls } = criteria;+ const { rawQuery, getAddMinutesQuery } = prisma;+ const { levelQuery, sumQuery } = getRetentionQuery(urls, windowMinutes);++ function getRetentionQuery(+ urls: string[],+ windowMinutes: number,+ ): {+ levelQuery: string;+ sumQuery: string;+ } {+ return urls.reduce(+ (pv, cv, i) => {+ const levelNumber = i + 1;+ const startSum = i > 0 ? 'union ' : '';++ if (levelNumber >= 2) {+ pv.levelQuery += `+ , level${levelNumber} AS (+ select distinct we.session_id, we.created_at+ from level${i} l+ join website_event we+ on l.session_id = we.session_id+ where we.created_at between l.created_at+ and ${getAddMinutesQuery(`l.created_at `, windowMinutes)}+ and we.referrer_path = {{${i - 1}}}+ and we.url_path = {{${i}}}+ and we.created_at <= {{endDate}}+ and we.website_id = {{websiteId::uuid}}+ )`;+ }++ pv.sumQuery += `\n${startSum}select ${levelNumber} as level, count(distinct(session_id)) as count from level${levelNumber}`;++ return pv;+ },+ {+ levelQuery: '',+ sumQuery: '',+ },+ );+ }++ return rawQuery(+ `+ WITH level1 AS (+ select distinct session_id, created_at+ from website_event+ where website_id = {{websiteId::uuid}}+ and created_at between {{startDate}} and {{endDate}}+ and url_path = {{0}}+ )+ ${levelQuery}+ ${sumQuery}+ ORDER BY level;+ `,+ {+ websiteId,+ startDate,+ endDate,+ ...urls,+ },+ ).then(results => {+ return urls.map((a, i) => ({+ x: a,+ y: results[i]?.count || 0,+ z: (1 - Number(results[i]?.count) / Number(results[i - 1]?.count)) * 100 || 0, // drop off+ }));+ });+}++async function clickhouseQuery(+ websiteId: string,+ criteria: {+ windowMinutes: number;+ startDate: Date;+ endDate: Date;+ urls: string[];+ },+): Promise<+ {+ x: string;+ y: number;+ z: number;+ }[]+> {+ const { windowMinutes, startDate, endDate, urls } = criteria;+ const { rawQuery } = clickhouse;+ const { levelQuery, sumQuery, urlFilterQuery, urlParams } = getRetentionQuery(+ urls,+ windowMinutes,+ );++ function getRetentionQuery(+ urls: string[],+ windowMinutes: number,+ ): {+ levelQuery: string;+ sumQuery: string;+ urlFilterQuery: string;+ urlParams: { [key: string]: string };+ } {+ return urls.reduce(+ (pv, cv, i) => {+ const levelNumber = i + 1;+ const startSum = i > 0 ? 'union all ' : '';+ const startFilter = i > 0 ? ', ' : '';++ if (levelNumber >= 2) {+ pv.levelQuery += `\n+ , level${levelNumber} AS (+ select distinct y.session_id as session_id,+ y.url_path as url_path,+ y.referrer_path as referrer_path,+ y.created_at as created_at+ from level${i} x+ join level0 y+ on x.session_id = y.session_id+ where y.created_at between x.created_at and x.created_at + interval ${windowMinutes} minute+ and y.referrer_path = {url${i - 1}:String}+ and y.url_path = {url${i}:String}+ )`;+ }++ pv.sumQuery += `\n${startSum}select ${levelNumber} as level, count(distinct(session_id)) as count from level${levelNumber}`;+ pv.urlFilterQuery += `${startFilter}{url${i}:String} `;+ pv.urlParams[`url${i}`] = cv;++ return pv;+ },+ {+ levelQuery: '',+ sumQuery: '',+ urlFilterQuery: '',+ urlParams: {},+ },+ );+ }++ return rawQuery<{ level: number; count: number }[]>(+ `+ WITH level0 AS (+ select distinct session_id, url_path, referrer_path, created_at+ from umami.website_event+ where url_path in (${urlFilterQuery})+ and website_id = {websiteId:UUID}+ and created_at between {startDate:DateTime64} and {endDate:DateTime64}+ ),+ level1 AS (+ select *+ from level0+ where url_path = {url0:String}+ )+ ${levelQuery}+ select *+ from (+ ${sumQuery}+ ) ORDER BY level;+ `,+ {+ websiteId,+ startDate,+ endDate,+ ...urlParams,+ },+ ).then(results => {+ return urls.map((a, i) => ({+ x: a,+ y: results[i]?.count || 0,+ z: (1 - Number(results[i]?.count) / Number(results[i - 1]?.count)) * 100 || 0, // drop off+ }));+ });+}diff --git a/queries/index.js b/queries/index.jsindex f509e0392..0fb2bf2cf 100644--- a/queries/index.js+++ b/queries/index.js@@ -12,6 +12,7 @@ export * from './analytics/eventData/getEventDataFields';export * from './analytics/eventData/getEventDataUsage';export * from './analytics/events/saveEvent';export * from './analytics/reports/getFunnel';+export * from './analytics/reports/getRetention';export * from './analytics/reports/getInsights';export * from './analytics/pageviews/getPageviewMetrics';export * from './analytics/pageviews/getPageviewStats';