Umami is a privacy-first web and product analytics platform. Traffic, campaigns, behavior, conversions, and revenue in one place. No cookies, no surveillance, self-hosted or in the cloud.
TypeScript analyticsaudience-segmentationchartscohort-analysisgoogle-analyticsproduct-analyticsstatisticsuser-journeyweb-analytics
Updated handling of date values for event data.
Mike Cao772357621268
diff --git a/src/lib/constants.ts b/src/lib/constants.tsindex 0773f4399..f728837a0 100644--- a/src/lib/constants.ts+++ b/src/lib/constants.ts@@ -216,6 +216,8 @@ export const UUID_REGEX =export const HOSTNAME_REGEX =/^(([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9-]*[a-zA-Z0-9])\.)*([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9-]*[A-Za-z0-9])$/;export const IP_REGEX = /^((25[0-5]|(2[0-4]|1[0-9]|[1-9]|)[0-9])(\.(?!$)|$)){4}$/;+export const DATETIME_REGEX =+ /^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}(\.[0-9]{3}(Z|\+[0-9]{2}:[0-9]{2})?)?$/;export const DESKTOP_SCREEN_WIDTH = 1920;export const LAPTOP_SCREEN_WIDTH = 1024;diff --git a/src/lib/data.ts b/src/lib/data.tsindex 5b502c5d3..24c76dd04 100644--- a/src/lib/data.ts+++ b/src/lib/data.ts@@ -1,19 +1,18 @@-import { isValid } from 'date-fns';-import { DATA_TYPE } from './constants';+import { DATA_TYPE, DATETIME_REGEX } from './constants';import { DynamicDataType } from './types';export function flattenJSON(eventData: { [key: string]: any },- keyValues: { key: string; value: any; dynamicDataType: DynamicDataType }[] = [],+ keyValues: { key: string; value: any; dataType: DynamicDataType }[] = [],parentKey = '',-): { key: string; value: any; dynamicDataType: DynamicDataType }[] {+): { key: string; value: any; dataType: DynamicDataType }[] {return Object.keys(eventData).reduce((acc, key) => {const value = eventData[key];const type = typeof eventData[key];// nested object- if (value && type === 'object' && !Array.isArray(value) && !isValid(value)) {+ if (value && type === 'object' && !Array.isArray(value) && !isValidDateValue(value)) {flattenJSON(value, acc.keyValues, getKeyName(key, parentKey));} else {createKey(getKeyName(key, parentKey), value, acc);@@ -25,48 +24,64 @@ export function flattenJSON().keyValues;}+export function isValidDateValue(value: string) {+ return typeof value === 'string' && DATETIME_REGEX.test(value);+}+export function getDataType(value: any): string {let type: string = typeof value;- if ((type === 'string' && isValid(value)) || isValid(new Date(value))) {+ if (isValidDateValue(value)) {type = 'date';}return type;}-function createKey(key, value, acc: { keyValues: any[]; parentKey: string }) {+export function getStringValue(value: string, dataType: number) {+ if (dataType === DATA_TYPE.number) {+ return parseFloat(value).toFixed(4);+ }++ if (dataType === DATA_TYPE.date) {+ return new Date(value).toISOString();+ }++ return value;+}++function createKey(key: string, value: string, acc: { keyValues: any[]; parentKey: string }) {const type = getDataType(value);- let dynamicDataType = null;+ let dataType = null;switch (type) {case 'number':- dynamicDataType = DATA_TYPE.number;+ dataType = DATA_TYPE.number;break;case 'string':- dynamicDataType = DATA_TYPE.string;+ dataType = DATA_TYPE.string;break;case 'boolean':- dynamicDataType = DATA_TYPE.boolean;+ dataType = DATA_TYPE.boolean;value = value ? 'true' : 'false';break;case 'date':- dynamicDataType = DATA_TYPE.date;+ dataType = DATA_TYPE.date;break;case 'object':- dynamicDataType = DATA_TYPE.array;+ dataType = DATA_TYPE.array;value = JSON.stringify(value);break;default:- dynamicDataType = DATA_TYPE.string;+ dataType = DATA_TYPE.string;break;}- acc.keyValues.push({ key, value, dynamicDataType });+ acc.keyValues.push({ key, value, dataType });}-function getKeyName(key, parentKey) {+function getKeyName(key: string, parentKey: string) {if (!parentKey) {return key;}diff --git a/src/queries/analytics/eventData/saveEventData.ts b/src/queries/analytics/eventData/saveEventData.tsindex 0f1ddb375..345409ce7 100644--- a/src/queries/analytics/eventData/saveEventData.ts+++ b/src/queries/analytics/eventData/saveEventData.ts@@ -2,7 +2,7 @@ import { Prisma } from '@prisma/client';import { DATA_TYPE } from 'lib/constants';import { uuid } from 'lib/crypto';import { CLICKHOUSE, PRISMA, runQuery } from 'lib/db';-import { flattenJSON } from 'lib/data';+import { flattenJSON, getStringValue } from 'lib/data';import kafka from 'lib/kafka';import prisma from 'lib/prisma';import { DynamicData } from 'lib/types';@@ -37,15 +37,10 @@ async function relationalQuery(data: {websiteEventId: eventId,websiteId,eventKey: a.key,- stringValue:- a.dynamicDataType === DATA_TYPE.number- ? parseFloat(a.value).toFixed(4)- : a.dynamicDataType === DATA_TYPE.date- ? a.value.split('.')[0] + 'Z'- : a.value.toString(),- numberValue: a.dynamicDataType === DATA_TYPE.number ? a.value : null,- dateValue: a.dynamicDataType === DATA_TYPE.date ? new Date(a.value) : null,- dataType: a.dynamicDataType,+ stringValue: getStringValue(a.value, a.dataType),+ numberValue: a.dataType === DATA_TYPE.number ? a.value : null,+ dateValue: a.dataType === DATA_TYPE.date ? new Date(a.value) : null,+ dataType: a.dataType,}));return prisma.client.eventData.createMany({@@ -75,13 +70,10 @@ async function clickhouseQuery(data: {url_path: urlPath,event_name: eventName,event_key: a.key,- string_value:- a.dynamicDataType === DATA_TYPE.date- ? getDateFormat(a.value, 'isoUtcDateTime')- : a.value.toString(),- number_value: a.dynamicDataType === DATA_TYPE.number ? a.value : null,- date_value: a.dynamicDataType === DATA_TYPE.date ? getDateFormat(a.value) : null,- data_type: a.dynamicDataType,+ string_value: getStringValue(a.value, a.dataType),+ number_value: a.dataType === DATA_TYPE.number ? a.value : null,+ date_value: a.dataType === DATA_TYPE.date ? getDateFormat(a.value) : null,+ data_type: a.dataType,created_at: createdAt,}));diff --git a/src/queries/analytics/sessions/saveSessionData.ts b/src/queries/analytics/sessions/saveSessionData.tsindex ef32bcfb5..b1c5759b0 100644--- a/src/queries/analytics/sessions/saveSessionData.ts+++ b/src/queries/analytics/sessions/saveSessionData.ts@@ -1,6 +1,6 @@import { DATA_TYPE } from 'lib/constants';import { uuid } from 'lib/crypto';-import { flattenJSON } from 'lib/data';+import { flattenJSON, getStringValue } from 'lib/data';import prisma from 'lib/prisma';import { DynamicData } from 'lib/types';@@ -19,15 +19,10 @@ export async function saveSessionData(data: {websiteId,sessionId,key: a.key,- stringValue:- a.dynamicDataType === DATA_TYPE.number- ? parseFloat(a.value).toFixed(4)- : a.dynamicDataType === DATA_TYPE.date- ? a.value.split('.')[0] + 'Z'- : a.value.toString(),- numberValue: a.dynamicDataType === DATA_TYPE.number ? a.value : null,- dateValue: a.dynamicDataType === DATA_TYPE.date ? new Date(a.value) : null,- dataType: a.dynamicDataType,+ stringValue: getStringValue(a.value, a.dataType),+ numberValue: a.dataType === DATA_TYPE.number ? a.value : null,+ dateValue: a.dataType === DATA_TYPE.date ? new Date(a.value) : null,+ dataType: a.dataType,}));return transaction([