Umami is a privacy-first web and product analytics platform. Traffic, campaigns, behavior, conversions, and revenue in one place. No cookies, no surveillance, self-hosted or in the cloud.
TypeScript analyticsaudience-segmentationchartscohort-analysisgoogle-analyticsproduct-analyticsstatisticsuser-journeyweb-analytics
Insights report filtering.
Mike Cao9436efabc054
diff --git a/components/pages/reports/funnel/FunnelReport.js b/components/pages/reports/funnel/FunnelReport.jsindex 7b4d8eced..d2971fa30 100644--- a/components/pages/reports/funnel/FunnelReport.js+++ b/components/pages/reports/funnel/FunnelReport.js@@ -6,9 +6,10 @@ import ReportHeader from '../ReportHeader';import ReportMenu from '../ReportMenu';import ReportBody from '../ReportBody';import Funnel from 'assets/funnel.svg';+import { REPORT_TYPES } from 'lib/constants';const defaultParameters = {- type: 'funnel',+ type: REPORT_TYPES.funnel,parameters: { window: 60, urls: [] },};diff --git a/components/pages/reports/insights/InsightsParameters.js b/components/pages/reports/insights/InsightsParameters.jsindex 18eeffc3d..6de4b838e 100644--- a/components/pages/reports/insights/InsightsParameters.js+++ b/components/pages/reports/insights/InsightsParameters.js@@ -119,7 +119,7 @@ export function InsightsParameters() {<div className={styles.parameter}>{id === 'fields' && (<>- <div>{label}</div>+ <div>{fieldOptions.find(f => f.name === name)?.label}</div></>)}{id === 'filters' && (diff --git a/components/pages/reports/insights/InsightsReport.js b/components/pages/reports/insights/InsightsReport.jsindex 88f12304c..3d855d9e8 100644--- a/components/pages/reports/insights/InsightsReport.js+++ b/components/pages/reports/insights/InsightsReport.js@@ -5,10 +5,11 @@ import ReportBody from '../ReportBody';import InsightsParameters from './InsightsParameters';import InsightsTable from './InsightsTable';import Lightbulb from 'assets/lightbulb.svg';+import { REPORT_TYPES } from 'lib/constants';const defaultParameters = {- type: 'insights',- parameters: { fields: [], filters: [], groups: [] },+ type: REPORT_TYPES.insights,+ parameters: { fields: [], filters: [] },};export default function InsightsReport({ reportId }) {diff --git a/components/pages/reports/insights/InsightsTable.js b/components/pages/reports/insights/InsightsTable.jsindex f45490018..d5422c9ef 100644--- a/components/pages/reports/insights/InsightsTable.js+++ b/components/pages/reports/insights/InsightsTable.js@@ -31,10 +31,15 @@ export function InsightsTable() {</GridColumn>);})}- <GridColumn name="visitors" label={formatMessage(labels.visitors)} width="100px">+ <GridColumn+ name="visitors"+ label={formatMessage(labels.visitors)}+ width="100px"+ alignment="end"+ >{row => row.visitors.toLocaleString()}</GridColumn>- <GridColumn name="views" label={formatMessage(labels.views)} width="100px">+ <GridColumn name="views" label={formatMessage(labels.views)} width="100px" alignment="end">{row => row.views.toLocaleString()}</GridColumn></GridTable>diff --git a/hooks/useFilters.js b/hooks/useFilters.jsindex 5143fe5b4..089f2ee84 100644--- a/hooks/useFilters.js+++ b/hooks/useFilters.js@@ -1,32 +1,40 @@import { useMessages } from 'hooks';+import { OPERATORS } from 'lib/constants';export function useFilters() {const { formatMessage, labels } = useMessages();const filterLabels = {- eq: formatMessage(labels.is),- neq: formatMessage(labels.isNot),- s: formatMessage(labels.isSet),- ns: formatMessage(labels.isNotSet),- c: formatMessage(labels.contains),- dnc: formatMessage(labels.doesNotContain),- t: formatMessage(labels.true),- f: formatMessage(labels.false),- gt: formatMessage(labels.greaterThan),- lt: formatMessage(labels.lessThan),- gte: formatMessage(labels.greaterThanEquals),- lte: formatMessage(labels.lessThanEquals),- be: formatMessage(labels.before),- af: formatMessage(labels.after),+ [OPERATORS.equals]: formatMessage(labels.is),+ [OPERATORS.notEquals]: formatMessage(labels.isNot),+ [OPERATORS.set]: formatMessage(labels.isSet),+ [OPERATORS.notSet]: formatMessage(labels.isNotSet),+ [OPERATORS.contains]: formatMessage(labels.contains),+ [OPERATORS.doesNotContain]: formatMessage(labels.doesNotContain),+ [OPERATORS.true]: formatMessage(labels.true),+ [OPERATORS.false]: formatMessage(labels.false),+ [OPERATORS.greaterThan]: formatMessage(labels.greaterThan),+ [OPERATORS.lessThan]: formatMessage(labels.lessThan),+ [OPERATORS.greaterThanEquals]: formatMessage(labels.greaterThanEquals),+ [OPERATORS.lessThanEquals]: formatMessage(labels.lessThanEquals),+ [OPERATORS.before]: formatMessage(labels.before),+ [OPERATORS.after]: formatMessage(labels.after),};const typeFilters = {- string: ['eq', 'neq'],- array: ['c', 'dnc'],- boolean: ['t', 'f'],- number: ['eq', 'neq', 'gt', 'lt', 'gte', 'lte'],- date: ['be', 'af'],- uuid: ['eq'],+ string: [OPERATORS.equals, OPERATORS.notEquals],+ array: [OPERATORS.contains, OPERATORS.doesNotContain],+ boolean: [OPERATORS.true, OPERATORS.false],+ number: [+ OPERATORS.equals,+ OPERATORS.notEquals,+ OPERATORS.greaterThan,+ OPERATORS.lessThan,+ OPERATORS.greaterThanEquals,+ OPERATORS.lessThanEquals,+ ],+ date: [OPERATORS.before, OPERATORS.after],+ uuid: [OPERATORS.equals],};const getFilters = type => {diff --git a/lib/clickhouse.ts b/lib/clickhouse.tsindex f7abd94f2..75786850d 100644--- a/lib/clickhouse.ts+++ b/lib/clickhouse.ts@@ -3,7 +3,7 @@ import dateFormat from 'dateformat';import debug from 'debug';import { CLICKHOUSE } from 'lib/db';import { QueryFilters, QueryOptions } from './types';-import { FILTER_COLUMNS } from './constants';+import { FILTER_COLUMNS, OPERATORS } from './constants';import { loadWebsite } from './load';import { maxDate } from './date';@@ -63,17 +63,29 @@ function getDateFormat(date) {return `'${dateFormat(date, 'UTC:yyyy-mm-dd HH:MM:ss')}'`;}+function mapFilter(column, operator, name) {+ switch (operator) {+ case OPERATORS.equals:+ return `${column} = {${name}:String}`;+ case OPERATORS.notEquals:+ return `${column} != {${name}:String}`;+ default:+ return '';+ }+}+function getFilterQuery(filters: QueryFilters = {}, options: QueryOptions = {}) {- const query = Object.keys(filters).reduce((arr, key) => {- const filter = filters[key];- const column = FILTER_COLUMNS[key] ?? options?.columns?.[key];+ const query = Object.keys(filters).reduce((arr, name) => {+ const value = filters[name];+ const operator = value?.filter ?? OPERATORS.equals;+ const column = FILTER_COLUMNS[name] ?? options?.columns?.[name];- if (filter !== undefined && column) {- arr.push(`and ${column} = {${key}:String}`);- }+ if (value !== undefined && column) {+ arr.push(`and ${mapFilter(column, operator, name)}`);- if (key === 'referrer') {- arr.push('and referrer_domain != {websiteDomain:String}');+ if (name === 'referrer') {+ arr.push('and referrer_domain != {websiteDomain:String}');+ }}return arr;@@ -82,11 +94,7 @@ function getFilterQuery(filters: QueryFilters = {}, options: QueryOptions = {})return query.join('\n');}-async function parseFilters(- websiteId: string,- filters: QueryFilters & { [key: string]: any } = {},- options?: QueryOptions,-) {+async function parseFilters(websiteId: string, filters: QueryFilters = {}, options?: QueryOptions) {const website = await loadWebsite(websiteId);return {diff --git a/lib/constants.ts b/lib/constants.tsindex 887f90a96..8972f81fc 100644--- a/lib/constants.ts+++ b/lib/constants.ts@@ -76,6 +76,23 @@ export const DATA_TYPE = {array: 5,} as const;+export const OPERATORS = {+ equals: 'eq',+ notEquals: 'neq',+ set: 's',+ notSet: 'ns',+ contains: 'c',+ doesNotContain: 'dnc',+ true: 't',+ false: 'f',+ greaterThan: 'gt',+ lessThan: 'lt',+ greaterThanEquals: 'gte',+ lessThanEquals: 'lte',+ before: 'bf',+ after: 'af',+} as const;+export const DATA_TYPES = {[DATA_TYPE.string]: 'string',[DATA_TYPE.number]: 'number',@@ -84,6 +101,12 @@ export const DATA_TYPES = {[DATA_TYPE.array]: 'array',};+export const REPORT_TYPES = {+ funnel: 'funnel',+ insights: 'insights',+ retention: 'retention',+} as const;+export const REPORT_PARAMETERS = {fields: 'fields',filters: 'filters',diff --git a/lib/prisma.ts b/lib/prisma.tsindex 753f1ae4a..a49932869 100644--- a/lib/prisma.ts+++ b/lib/prisma.ts@@ -1,7 +1,7 @@import prisma from '@umami/prisma-client';import moment from 'moment-timezone';import { MYSQL, POSTGRESQL, getDatabaseType } from 'lib/db';-import { FILTER_COLUMNS, SESSION_COLUMNS } from './constants';+import { FILTER_COLUMNS, SESSION_COLUMNS, OPERATORS } from './constants';import { loadWebsite } from './load';import { maxDate } from './date';import { QueryFilters, QueryOptions } from './types';@@ -67,15 +67,27 @@ function getTimestampIntervalQuery(field: string): string {}}+function mapFilter(column, operator, name) {+ switch (operator) {+ case OPERATORS.equals:+ return `${column} = {{${name}}}`;+ case OPERATORS.notEquals:+ return `${column} != {{${name}}}`;+ default:+ return '';+ }+}+function getFilterQuery(filters: QueryFilters = {}, options: QueryOptions = {}): string {- const query = Object.keys(filters).reduce((arr, key) => {- const filter = filters[key];- const column = FILTER_COLUMNS[key] ?? options?.columns?.[key];+ const query = Object.keys(filters).reduce((arr, name) => {+ const value = filters[name];+ const operator = value?.filter ?? OPERATORS.equals;+ const column = FILTER_COLUMNS[name] ?? options?.columns?.[name];- if (filter !== undefined && column) {- arr.push(`and ${column}={{${key}}}`);+ if (value !== undefined && column) {+ arr.push(`and ${mapFilter(column, operator, name)}`);- if (key === 'referrer') {+ if (name === 'referrer') {arr.push('and (website_event.referrer_domain != {{websiteDomain}} or website_event.referrer_domain is null)',);@@ -88,11 +100,17 @@ function getFilterQuery(filters: QueryFilters = {}, options: QueryOptions = {}):return query.join('\n');}-async function parseFilters(- websiteId,- filters: QueryFilters & { [key: string]: any } = {},- options: QueryOptions = {},-) {+function normalizeFilters(filters = {}) {+ return Object.keys(filters).reduce((obj, key) => {+ const value = filters[key];++ obj[key] = value?.value ?? value;++ return obj;+ }, {});+}++async function parseFilters(websiteId, filters: QueryFilters = {}, options: QueryOptions = {}) {const website = await loadWebsite(websiteId);return {@@ -102,7 +120,7 @@ async function parseFilters(: '',filterQuery: getFilterQuery(filters, options),params: {- ...filters,+ ...normalizeFilters(filters),websiteId,startDate: maxDate(filters.startDate, website.resetAt),websiteDomain: website.domain,diff --git a/package.json b/package.jsonindex 647cdf418..89dc5e97f 100644--- a/package.json+++ b/package.json@@ -94,7 +94,7 @@"node-fetch": "^3.2.8","npm-run-all": "^4.1.5","react": "^18.2.0",- "react-basics": "^0.91.0",+ "react-basics": "^0.92.0","react-beautiful-dnd": "^13.1.0","react-dom": "^18.2.0","react-error-boundary": "^4.0.4",diff --git a/pages/api/reports/insights.ts b/pages/api/reports/insights.tsindex decb1f81f..09a07d2fb 100644--- a/pages/api/reports/insights.ts+++ b/pages/api/reports/insights.ts@@ -16,6 +16,14 @@ export interface InsightsRequestBody {groups: { name: string; type: string }[];}+function convertFilters(filters) {+ return filters.reduce((obj, { name, ...value }) => {+ obj[name] = value;++ return obj;+ }, {});+}+export default async (req: NextApiRequestQueryBody<any, InsightsRequestBody>,res: NextApiResponse,@@ -36,7 +44,7 @@ export default async (}const data = await getInsights(websiteId, fields, {- ...filters,+ ...convertFilters(filters),startDate: new Date(startDate),endDate: new Date(endDate),});diff --git a/queries/analytics/reports/getInsights.ts b/queries/analytics/reports/getInsights.tsindex 9793f258c..fa54488b6 100644--- a/queries/analytics/reports/getInsights.ts+++ b/queries/analytics/reports/getInsights.ts@@ -1,7 +1,7 @@import { CLICKHOUSE, PRISMA, runQuery } from 'lib/db';import prisma from 'lib/prisma';import clickhouse from 'lib/clickhouse';-import { EVENT_TYPE, SESSION_COLUMNS } from 'lib/constants';+import { EVENT_TYPE, FILTER_COLUMNS, SESSION_COLUMNS } from 'lib/constants';import { QueryFilters } from 'lib/types';export async function getInsights(@@ -91,7 +91,7 @@ function parseFields(fields) {(arr, field) => {const { name } = field;- return arr.concat(name);+ return arr.concat(`${FILTER_COLUMNS[name]} as "${name}"`);},['count(*) as views', 'count(distinct website_event.session_id) as visitors'],);@@ -103,5 +103,5 @@ function parseGroupBy(fields) {if (!fields.length) {return '';}- return `group by ${fields.map(({ name }) => name).join(',')}`;+ return `group by ${fields.map(({ name }) => FILTER_COLUMNS[name]).join(',')}`;}diff --git a/yarn.lock b/yarn.lockindex d9224c2a4..115e3cc9c 100644--- a/yarn.lock+++ b/yarn.lock@@ -7557,10 +7557,10 @@ rc@^1.2.7:minimist "^1.2.0"strip-json-comments "~2.0.1"-react-basics@^0.91.0:- version "0.91.0"- resolved "https://registry.yarnpkg.com/react-basics/-/react-basics-0.91.0.tgz#2970529a22a455ec73a1be884eb93a109c9dafc0"- integrity sha512-vP8LYWiFwA+eguMEuHvHct4Jl5R/2GUjWc1tMujDG0CsAAUGhx68tAJr0K3gBrWjmpJrTPVfX8SdBNKSDAjQsw==+react-basics@^0.92.0:+ version "0.92.0"+ resolved "https://registry.yarnpkg.com/react-basics/-/react-basics-0.92.0.tgz#02bc6e88bdaf189c30cc6cbd8bbb1c9d12cd089b"+ integrity sha512-BVUWg5a7R88konA9NedYMBx1hl50d6h/MD7qlKOEO/Cnm8cOC7AYTRKAKhO6kHMWjY4ZpUuvlg0UcF+SJP/uXA==dependencies:classnames "^2.3.1"date-fns "^2.29.3"