Umami is a privacy-first web and product analytics platform. Traffic, campaigns, behavior, conversions, and revenue in one place. No cookies, no surveillance, self-hosted or in the cloud.
TypeScript analyticsaudience-segmentationchartscohort-analysisgoogle-analyticsproduct-analyticsstatisticsuser-journeyweb-analytics
Updated filtering logic.
Mike Cao810b0639c87a
diff --git a/src/app/(main)/reports/[reportId]/FieldFilterEditForm.tsx b/src/app/(main)/reports/[reportId]/FieldFilterEditForm.tsxindex f1ebd30df..5612c5540 100644--- a/src/app/(main)/reports/[reportId]/FieldFilterEditForm.tsx+++ b/src/app/(main)/reports/[reportId]/FieldFilterEditForm.tsx@@ -16,7 +16,7 @@ import {} from 'react-basics';import { useMessages, useFilters, useFormat, useLocale, useWebsiteValues } from 'components/hooks';import { OPERATORS } from 'lib/constants';-import { operatorEquals } from 'lib/params';+import { isEqualsOperator } from 'lib/params';import styles from './FieldFilterEditForm.module.css';export interface FieldFilterFormProps {@@ -50,7 +50,7 @@ export default function FieldFilterEditForm({const [operator, setOperator] = useState(defaultOperator);const [value, setValue] = useState(defaultValue);const [showMenu, setShowMenu] = useState(false);- const isEquals = operatorEquals(operator);+ const isEquals = isEqualsOperator(operator);const [search, setSearch] = useState('');const [selected, setSelected] = useState(isEquals ? value : '');const { filters } = useFilters();diff --git a/src/app/(main)/reports/[reportId]/FilterParameters.tsx b/src/app/(main)/reports/[reportId]/FilterParameters.tsxindex d88f785da..edc0ff4ec 100644--- a/src/app/(main)/reports/[reportId]/FilterParameters.tsx+++ b/src/app/(main)/reports/[reportId]/FilterParameters.tsx@@ -7,7 +7,7 @@ import ParameterList from '../[reportId]/ParameterList';import PopupForm from '../[reportId]/PopupForm';import { ReportContext } from './Report';import FieldFilterEditForm from '../[reportId]/FieldFilterEditForm';-import { operatorEquals } from 'lib/params';+import { isSearchOperator } from 'lib/params';import styles from './FilterParameters.module.css';export function FilterParameters() {@@ -69,7 +69,7 @@ export function FilterParameters() {{filters.map(({ name, operator, value }: { name: string; operator: string; value: string }) => {const label = fields.find(f => f.name === name)?.label;- const isEquals = operatorEquals(operator);+ const isSearch = isSearchOperator(operator);return (<ParameterList.Item key={name} onRemove={() => handleRemove(name)}>@@ -79,7 +79,7 @@ export function FilterParameters() {name={name}label={label}operator={operator}- value={isEquals ? formatValue(value, name) : value}+ value={isSearch ? value : formatValue(value, name)}onChange={handleChange}/></ParameterList.Item>diff --git a/src/app/(main)/websites/[websiteId]/WebsiteExpandedView.tsx b/src/app/(main)/websites/[websiteId]/WebsiteExpandedView.tsxindex 9fdc8fd04..ea037852e 100644--- a/src/app/(main)/websites/[websiteId]/WebsiteExpandedView.tsx+++ b/src/app/(main)/websites/[websiteId]/WebsiteExpandedView.tsx@@ -44,7 +44,6 @@ export default function WebsiteExpandedView({const {router,renderUrl,- pathname,query: { view },} = useNavigation();@@ -122,7 +121,12 @@ export default function WebsiteExpandedView({return (<div className={styles.layout}><div className={styles.menu}>- <LinkButton href={pathname} className={styles.back} variant="quiet" scroll={false}>+ <LinkButton+ href={renderUrl({ view: undefined })}+ className={styles.back}+ variant="quiet"+ scroll={false}+ ><Icon rotate={dir === 'rtl' ? 0 : 180}><Icons.ArrowRight /></Icon>diff --git a/src/components/metrics/FilterTags.tsx b/src/components/metrics/FilterTags.tsxindex 24311d6e0..a302575f6 100644--- a/src/components/metrics/FilterTags.tsx+++ b/src/components/metrics/FilterTags.tsx@@ -11,7 +11,7 @@ import {import PopupForm from 'app/(main)/reports/[reportId]/PopupForm';import FieldFilterEditForm from 'app/(main)/reports/[reportId]/FieldFilterEditForm';import { OPERATOR_PREFIXES } from 'lib/constants';-import { operatorEquals, parseParameterValue } from 'lib/params';+import { isSearchOperator, parseParameterValue } from 'lib/params';import styles from './FilterTags.module.css';export function FilterTags({@@ -66,7 +66,7 @@ export function FilterTags({}const label = fields.find(f => f.name === key)?.label;const { operator, value } = parseParameterValue(params[key]);- const paramValue = operatorEquals(operator) ? formatValue(value, key) : value;+ const paramValue = isSearchOperator(operator) ? formatValue(value, key) : value;return (<PopupTrigger key={key}>diff --git a/src/lib/clickhouse.ts b/src/lib/clickhouse.tsindex 0eabd3d23..5bcf7ee0b 100644--- a/src/lib/clickhouse.ts+++ b/src/lib/clickhouse.ts@@ -3,9 +3,10 @@ import dateFormat from 'dateformat';import debug from 'debug';import { CLICKHOUSE } from 'lib/db';import { QueryFilters, QueryOptions } from './types';-import { FILTER_COLUMNS, OPERATORS } from './constants';+import { OPERATORS } from './constants';import { loadWebsite } from './load';import { maxDate } from './date';+import { filtersToArray } from './params';export const CLICKHOUSE_DATE_FORMATS = {minute: '%Y-%m-%d %H:%M:00',@@ -79,15 +80,11 @@ function mapFilter(column: string, operator: string, name: string, type: string}function getFilterQuery(filters: QueryFilters = {}, options: QueryOptions = {}) {- const query = Object.keys(filters).reduce((arr, key) => {- const filter = filters[key];- const operator = filter?.operator ?? OPERATORS.equals;- const column = filter?.column ?? FILTER_COLUMNS[key] ?? options?.columns?.[key];+ const query = filtersToArray(filters, options).reduce((arr, { name, column, operator }) => {+ if (column) {+ arr.push(`and ${mapFilter(column, operator, name)}`);- if (filter !== undefined && column !== undefined) {- arr.push(`and ${mapFilter(column, operator, key)}`);-- if (key === 'referrer') {+ if (name === 'referrer') {arr.push('and referrer_domain != {websiteDomain:String}');}}@@ -98,11 +95,11 @@ function getFilterQuery(filters: QueryFilters = {}, options: QueryOptions = {})return query.join('\n');}-function normalizeFilters(filters = {}) {- return Object.keys(filters).reduce((obj, key) => {- const value = filters[key];-- obj[key] = value?.value ?? value;+function getFilterParams(filters: QueryFilters = {}) {+ return filtersToArray(filters).reduce((obj, { name, value }) => {+ if (name && value !== undefined) {+ obj[name] = value;+ }return obj;}, {});@@ -114,7 +111,7 @@ async function parseFilters(websiteId: string, filters: QueryFilters = {}, optioreturn {filterQuery: getFilterQuery(filters, options),params: {- ...normalizeFilters(filters),+ ...getFilterParams(filters),websiteId,startDate: maxDate(filters.startDate, new Date(website?.resetAt)),websiteDomain: website.domain,diff --git a/src/lib/params.ts b/src/lib/params.tsindex 10944c962..801c60b20 100644--- a/src/lib/params.ts+++ b/src/lib/params.ts@@ -1,15 +1,46 @@-import { OPERATOR_PREFIXES, OPERATORS } from 'lib/constants';+import { FILTER_COLUMNS, OPERATOR_PREFIXES, OPERATORS } from 'lib/constants';+import { QueryFilters, QueryOptions } from 'lib/types';-export function parseParameterValue(param: string) {- const [, prefix, value] = param.match(/^(!~|!|~)?(.*)$/);+export function parseParameterValue(param: any) {+ if (typeof param === 'string') {+ const [, prefix, value] = param.match(/^(!~|!|~)?(.*)$/);- const operator =- Object.keys(OPERATOR_PREFIXES).find(key => OPERATOR_PREFIXES[key] === prefix) ||- OPERATORS.equals;+ const operator =+ Object.keys(OPERATOR_PREFIXES).find(key => OPERATOR_PREFIXES[key] === prefix) ||+ OPERATORS.equals;- return { operator, value };+ return { operator, value };+ }+ return { operator: OPERATORS.equals, value: param };}-export function operatorEquals(operator: any) {+export function isEqualsOperator(operator: any) {return [OPERATORS.equals, OPERATORS.notEquals].includes(operator);}++export function isSearchOperator(operator: any) {+ return [OPERATORS.contains, OPERATORS.doesNotContain].includes(operator);+}++export function filtersToArray(filters: QueryFilters = {}, options: QueryOptions = {}) {+ return Object.keys(filters).reduce((arr, key) => {+ const filter = filters[key];++ if (filter === undefined || filter === null) {+ return arr;+ }++ if (filter?.name && filter?.value !== undefined) {+ return arr.concat(filter);+ }++ const { operator, value } = parseParameterValue(filter);++ return arr.concat({+ name: key,+ column: options?.columns?.[key] ?? FILTER_COLUMNS[key],+ operator,+ value,+ });+ }, []);+}diff --git a/src/lib/prisma.ts b/src/lib/prisma.tsindex c3dd071b8..c35e0cde3 100644--- a/src/lib/prisma.ts+++ b/src/lib/prisma.ts@@ -2,10 +2,11 @@ import { Prisma } from '@prisma/client';import prisma from '@umami/prisma-client';import moment from 'moment-timezone';import { MYSQL, POSTGRESQL, getDatabaseType } from 'lib/db';-import { FILTER_COLUMNS, SESSION_COLUMNS, OPERATORS, DEFAULT_PAGE_SIZE } from './constants';+import { SESSION_COLUMNS, OPERATORS, DEFAULT_PAGE_SIZE } from './constants';import { loadWebsite } from './load';import { maxDate } from './date';import { QueryFilters, QueryOptions, SearchFilter } from './types';+import { filtersToArray } from './params';const MYSQL_DATE_FORMATS = {minute: '%Y-%m-%d %H:%i:00',@@ -112,15 +113,11 @@ function mapFilter(column: string, operator: string, name: string, type: string}function getFilterQuery(filters: QueryFilters = {}, options: QueryOptions = {}): string {- const query = Object.keys(filters).reduce((arr, key) => {- const filter = filters[key];- const operator = filter?.operator ?? OPERATORS.equals;- const column = filter?.column ?? FILTER_COLUMNS[key] ?? options?.columns?.[key];+ const query = filtersToArray(filters, options).reduce((arr, { name, column, operator }) => {+ if (column) {+ arr.push(`and ${mapFilter(column, operator, name)}`);- if (filter !== undefined && column !== undefined) {- arr.push(`and ${mapFilter(column, operator, key)}`);-- if (key === 'referrer') {+ if (name === 'referrer') {arr.push('and (website_event.referrer_domain != {{websiteDomain}} or website_event.referrer_domain is null)',);@@ -133,12 +130,9 @@ function getFilterQuery(filters: QueryFilters = {}, options: QueryOptions = {}):return query.join('\n');}-function normalizeFilters(filters = {}) {- return Object.keys(filters).reduce((obj, key) => {- const filter = filters[key];- const value = filter?.value ?? filter;-- obj[key] = [OPERATORS.contains, OPERATORS.doesNotContain].includes(filter?.operator)+function getFilterParams(filters: QueryFilters = {}) {+ return filtersToArray(filters).reduce((obj, { name, operator, value }) => {+ obj[name] = [OPERATORS.contains, OPERATORS.doesNotContain].includes(operator)? `%${value}%`: value;@@ -152,15 +146,16 @@ async function parseFilters(options: QueryOptions = {},) {const website = await loadWebsite(websiteId);+ const joinSession = Object.keys(filters).find(key => SESSION_COLUMNS.includes(key));return {joinSession:- options?.joinSession || Object.keys(filters).find(key => SESSION_COLUMNS.includes(key))+ options?.joinSession || joinSession? `inner join session on website_event.session_id = session.session_id`: '',filterQuery: getFilterQuery(filters, options),params: {- ...normalizeFilters(filters),+ ...getFilterParams(filters),websiteId,startDate: maxDate(filters.startDate, website?.resetAt),websiteDomain: website.domain,diff --git a/src/lib/query.ts b/src/lib/request.tssimilarity index 63%rename from src/lib/query.tsrename to src/lib/request.tsindex 8b6bcee6b..5e2be2fe2 100644--- a/src/lib/query.ts+++ b/src/lib/request.ts@@ -1,9 +1,9 @@import { NextApiRequest } from 'next';import { getAllowedUnits, getMinimumUnit } from './date';import { getWebsiteDateRange } from '../queries';-import { FILTER_COLUMNS, OPERATORS, OPERATOR_PREFIXES } from 'lib/constants';+import { FILTER_COLUMNS } from 'lib/constants';-export async function parseDateRangeQuery(req: NextApiRequest) {+export async function getRequestDateRange(req: NextApiRequest) {const { websiteId, startAt, endAt, unit } = req.query;// All-time@@ -31,27 +31,14 @@ export async function parseDateRangeQuery(req: NextApiRequest) {};}-export function getQueryFilters(req: NextApiRequest) {+export function getRequestFilters(req: NextApiRequest) {return Object.keys(FILTER_COLUMNS).reduce((obj, key) => {const value = req.query[key];- if (value) {+ if (value !== undefined) {obj[key] = value;}- if (typeof value === 'string') {- const [, prefix, paramValue] = value.match(/^(!~|!|~)?(.*)$/);-- if (prefix && paramValue) {- obj[key] = {- name: key,- column: FILTER_COLUMNS[key],- operator: OPERATOR_PREFIXES[prefix] || OPERATORS.equals,- value: paramValue,- };- }- }-return obj;}, {});}diff --git a/src/pages/api/send.ts b/src/pages/api/send.tsindex 72fd7a64d..0b43f9201 100644--- a/src/pages/api/send.ts+++ b/src/pages/api/send.ts@@ -98,7 +98,7 @@ export default async (req: NextApiRequestCollect, res: NextApiResponse) => {}const { type, payload } = req.body;- const { url, referrer, name: eventName, data: eventData, title } = payload;+ const { url, referrer, name: eventName, data, title } = payload;const pageTitle = safeDecodeURI(title);await useSession(req, res);@@ -142,7 +142,7 @@ export default async (req: NextApiRequestCollect, res: NextApiResponse) => {referrerDomain,pageTitle,eventName,- eventData,+ eventData: data,...session,sessionId: session.id,visitId: session.visitId,@@ -150,14 +150,14 @@ export default async (req: NextApiRequestCollect, res: NextApiResponse) => {}if (type === COLLECTION_TYPE.identify) {- if (!eventData) {+ if (!data) {return badRequest(res, 'Data required.');}await saveSessionData({websiteId: session.websiteId,sessionId: session.id,- sessionData: eventData,+ sessionData: data,});}diff --git a/src/pages/api/websites/[websiteId]/events.ts b/src/pages/api/websites/[websiteId]/events.tsindex b4d465a4b..d07fd28f9 100644--- a/src/pages/api/websites/[websiteId]/events.ts+++ b/src/pages/api/websites/[websiteId]/events.ts@@ -1,6 +1,6 @@import { canViewWebsite } from 'lib/auth';import { useAuth, useCors, useValidate } from 'lib/middleware';-import { parseDateRangeQuery } from 'lib/query';+import { getRequestDateRange } from 'lib/request';import { NextApiRequestQueryBody, WebsiteMetric } from 'lib/types';import { TimezoneTest, UnitTypeTest } from 'lib/yup';import { NextApiResponse } from 'next';@@ -37,7 +37,7 @@ export default async (await useValidate(schema, req, res);const { websiteId, timezone, url } = req.query;- const { startDate, endDate, unit } = await parseDateRangeQuery(req);+ const { startDate, endDate, unit } = await getRequestDateRange(req);if (req.method === 'GET') {if (!(await canViewWebsite(req.auth, websiteId))) {diff --git a/src/pages/api/websites/[websiteId]/metrics.ts b/src/pages/api/websites/[websiteId]/metrics.tsindex 2387d9c20..d1765b1cb 100644--- a/src/pages/api/websites/[websiteId]/metrics.ts+++ b/src/pages/api/websites/[websiteId]/metrics.ts@@ -5,7 +5,7 @@ import { canViewWebsite } from 'lib/auth';import { useAuth, useCors, useValidate } from 'lib/middleware';import { SESSION_COLUMNS, EVENT_COLUMNS, FILTER_COLUMNS, OPERATORS } from 'lib/constants';import { getPageviewMetrics, getSessionMetrics } from 'queries';-import { getQueryFilters, parseDateRangeQuery } from 'lib/query';+import { getRequestFilters, getRequestDateRange } from 'lib/request';import * as yup from 'yup';export interface WebsiteMetricsRequestQuery {@@ -69,10 +69,10 @@ export default async (return unauthorized(res);}- const { startDate, endDate } = await parseDateRangeQuery(req);+ const { startDate, endDate } = await getRequestDateRange(req);const column = FILTER_COLUMNS[type] || type;const filters = {- ...getQueryFilters(req),+ ...getRequestFilters(req),startDate,endDate,};diff --git a/src/pages/api/websites/[websiteId]/pageviews.ts b/src/pages/api/websites/[websiteId]/pageviews.tsindex 9ac4e8709..196710643 100644--- a/src/pages/api/websites/[websiteId]/pageviews.ts+++ b/src/pages/api/websites/[websiteId]/pageviews.ts@@ -1,6 +1,6 @@import { canViewWebsite } from 'lib/auth';import { useAuth, useCors, useValidate } from 'lib/middleware';-import { getQueryFilters, parseDateRangeQuery } from 'lib/query';+import { getRequestFilters, getRequestDateRange } from 'lib/request';import { NextApiRequestQueryBody, WebsitePageviews } from 'lib/types';import { NextApiResponse } from 'next';import { methodNotAllowed, ok, unauthorized } from 'next-basics';@@ -59,10 +59,10 @@ export default async (return unauthorized(res);}- const { startDate, endDate, unit } = await parseDateRangeQuery(req);+ const { startDate, endDate, unit } = await getRequestDateRange(req);const filters = {- ...getQueryFilters(req),+ ...getRequestFilters(req),startDate,endDate,timezone,diff --git a/src/pages/api/websites/[websiteId]/stats.ts b/src/pages/api/websites/[websiteId]/stats.tsindex dfc3df936..81a6d8354 100644--- a/src/pages/api/websites/[websiteId]/stats.ts+++ b/src/pages/api/websites/[websiteId]/stats.ts@@ -4,7 +4,7 @@ import { methodNotAllowed, ok, unauthorized } from 'next-basics';import { canViewWebsite } from 'lib/auth';import { useAuth, useCors, useValidate } from 'lib/middleware';import { NextApiRequestQueryBody, WebsiteStats } from 'lib/types';-import { getQueryFilters, parseDateRangeQuery } from 'lib/query';+import { getRequestFilters, getRequestDateRange } from 'lib/request';import { getWebsiteStats } from 'queries';export interface WebsiteStatsRequestQuery {@@ -59,12 +59,12 @@ export default async (return unauthorized(res);}- const { startDate, endDate } = await parseDateRangeQuery(req);+ const { startDate, endDate } = await getRequestDateRange(req);const diff = differenceInMinutes(endDate, startDate);const prevStartDate = subMinutes(startDate, diff);const prevEndDate = subMinutes(endDate, diff);- const filters = getQueryFilters(req);+ const filters = getRequestFilters(req);const metrics = await getWebsiteStats(websiteId, { ...filters, startDate, endDate });diff --git a/src/pages/api/websites/[websiteId]/values.ts b/src/pages/api/websites/[websiteId]/values.tsindex 36ca7948e..364261d9d 100644--- a/src/pages/api/websites/[websiteId]/values.ts+++ b/src/pages/api/websites/[websiteId]/values.ts@@ -11,7 +11,7 @@ import {} from 'next-basics';import { EVENT_COLUMNS, FILTER_COLUMNS, SESSION_COLUMNS } from 'lib/constants';import { getValues } from 'queries';-import { parseDateRangeQuery } from 'lib/query';+import { getRequestDateRange } from 'lib/request';import * as yup from 'yup';export interface ValuesRequestQuery {@@ -38,7 +38,7 @@ export default async (req: NextApiRequestQueryBody<ValuesRequestQuery>, res: Nexawait useValidate(schema, req, res);const { websiteId, type, search } = req.query;- const { startDate, endDate } = await parseDateRangeQuery(req);+ const { startDate, endDate } = await getRequestDateRange(req);if (req.method === 'GET') {if (!SESSION_COLUMNS.includes(type as string) && !EVENT_COLUMNS.includes(type as string)) {