Skip to content

armgit / umami

Public Imported from GitHub
0

Umami is a privacy-first web and product analytics platform. Traffic, campaigns, behavior, conversions, and revenue in one place. No cookies, no surveillance, self-hosted or in the cloud.

Updated handling of date values for event data.

Mike Cao772357621268
diff --git a/src/lib/constants.ts b/src/lib/constants.ts
index 0773f4399..f728837a0 100644
--- a/src/lib/constants.ts
+++ b/src/lib/constants.ts
@@ -216,6 +216,8 @@ export const UUID_REGEX =
export const HOSTNAME_REGEX =
/^(([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9-]*[a-zA-Z0-9])\.)*([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9-]*[A-Za-z0-9])$/;
export const IP_REGEX = /^((25[0-5]|(2[0-4]|1[0-9]|[1-9]|)[0-9])(\.(?!$)|$)){4}$/;
+export const DATETIME_REGEX =
+ /^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}(\.[0-9]{3}(Z|\+[0-9]{2}:[0-9]{2})?)?$/;
export const DESKTOP_SCREEN_WIDTH = 1920;
export const LAPTOP_SCREEN_WIDTH = 1024;
diff --git a/src/lib/data.ts b/src/lib/data.ts
index 5b502c5d3..24c76dd04 100644
--- a/src/lib/data.ts
+++ b/src/lib/data.ts
@@ -1,19 +1,18 @@
-import { isValid } from 'date-fns';
-import { DATA_TYPE } from './constants';
+import { DATA_TYPE, DATETIME_REGEX } from './constants';
import { DynamicDataType } from './types';
export function flattenJSON(
eventData: { [key: string]: any },
- keyValues: { key: string; value: any; dynamicDataType: DynamicDataType }[] = [],
+ keyValues: { key: string; value: any; dataType: DynamicDataType }[] = [],
parentKey = '',
-): { key: string; value: any; dynamicDataType: DynamicDataType }[] {
+): { key: string; value: any; dataType: DynamicDataType }[] {
return Object.keys(eventData).reduce(
(acc, key) => {
const value = eventData[key];
const type = typeof eventData[key];
// nested object
- if (value && type === 'object' && !Array.isArray(value) && !isValid(value)) {
+ if (value && type === 'object' && !Array.isArray(value) && !isValidDateValue(value)) {
flattenJSON(value, acc.keyValues, getKeyName(key, parentKey));
} else {
createKey(getKeyName(key, parentKey), value, acc);
@@ -25,48 +24,64 @@ export function flattenJSON(
).keyValues;
}
+export function isValidDateValue(value: string) {
+ return typeof value === 'string' && DATETIME_REGEX.test(value);
+}
+
export function getDataType(value: any): string {
let type: string = typeof value;
- if ((type === 'string' && isValid(value)) || isValid(new Date(value))) {
+ if (isValidDateValue(value)) {
type = 'date';
}
return type;
}
-function createKey(key, value, acc: { keyValues: any[]; parentKey: string }) {
+export function getStringValue(value: string, dataType: number) {
+ if (dataType === DATA_TYPE.number) {
+ return parseFloat(value).toFixed(4);
+ }
+
+ if (dataType === DATA_TYPE.date) {
+ return new Date(value).toISOString();
+ }
+
+ return value;
+}
+
+function createKey(key: string, value: string, acc: { keyValues: any[]; parentKey: string }) {
const type = getDataType(value);
- let dynamicDataType = null;
+ let dataType = null;
switch (type) {
case 'number':
- dynamicDataType = DATA_TYPE.number;
+ dataType = DATA_TYPE.number;
break;
case 'string':
- dynamicDataType = DATA_TYPE.string;
+ dataType = DATA_TYPE.string;
break;
case 'boolean':
- dynamicDataType = DATA_TYPE.boolean;
+ dataType = DATA_TYPE.boolean;
value = value ? 'true' : 'false';
break;
case 'date':
- dynamicDataType = DATA_TYPE.date;
+ dataType = DATA_TYPE.date;
break;
case 'object':
- dynamicDataType = DATA_TYPE.array;
+ dataType = DATA_TYPE.array;
value = JSON.stringify(value);
break;
default:
- dynamicDataType = DATA_TYPE.string;
+ dataType = DATA_TYPE.string;
break;
}
- acc.keyValues.push({ key, value, dynamicDataType });
+ acc.keyValues.push({ key, value, dataType });
}
-function getKeyName(key, parentKey) {
+function getKeyName(key: string, parentKey: string) {
if (!parentKey) {
return key;
}
diff --git a/src/queries/analytics/eventData/saveEventData.ts b/src/queries/analytics/eventData/saveEventData.ts
index 0f1ddb375..345409ce7 100644
--- a/src/queries/analytics/eventData/saveEventData.ts
+++ b/src/queries/analytics/eventData/saveEventData.ts
@@ -2,7 +2,7 @@ import { Prisma } from '@prisma/client';
import { DATA_TYPE } from 'lib/constants';
import { uuid } from 'lib/crypto';
import { CLICKHOUSE, PRISMA, runQuery } from 'lib/db';
-import { flattenJSON } from 'lib/data';
+import { flattenJSON, getStringValue } from 'lib/data';
import kafka from 'lib/kafka';
import prisma from 'lib/prisma';
import { DynamicData } from 'lib/types';
@@ -37,15 +37,10 @@ async function relationalQuery(data: {
websiteEventId: eventId,
websiteId,
eventKey: a.key,
- stringValue:
- a.dynamicDataType === DATA_TYPE.number
- ? parseFloat(a.value).toFixed(4)
- : a.dynamicDataType === DATA_TYPE.date
- ? a.value.split('.')[0] + 'Z'
- : a.value.toString(),
- numberValue: a.dynamicDataType === DATA_TYPE.number ? a.value : null,
- dateValue: a.dynamicDataType === DATA_TYPE.date ? new Date(a.value) : null,
- dataType: a.dynamicDataType,
+ stringValue: getStringValue(a.value, a.dataType),
+ numberValue: a.dataType === DATA_TYPE.number ? a.value : null,
+ dateValue: a.dataType === DATA_TYPE.date ? new Date(a.value) : null,
+ dataType: a.dataType,
}));
return prisma.client.eventData.createMany({
@@ -75,13 +70,10 @@ async function clickhouseQuery(data: {
url_path: urlPath,
event_name: eventName,
event_key: a.key,
- string_value:
- a.dynamicDataType === DATA_TYPE.date
- ? getDateFormat(a.value, 'isoUtcDateTime')
- : a.value.toString(),
- number_value: a.dynamicDataType === DATA_TYPE.number ? a.value : null,
- date_value: a.dynamicDataType === DATA_TYPE.date ? getDateFormat(a.value) : null,
- data_type: a.dynamicDataType,
+ string_value: getStringValue(a.value, a.dataType),
+ number_value: a.dataType === DATA_TYPE.number ? a.value : null,
+ date_value: a.dataType === DATA_TYPE.date ? getDateFormat(a.value) : null,
+ data_type: a.dataType,
created_at: createdAt,
}));
diff --git a/src/queries/analytics/sessions/saveSessionData.ts b/src/queries/analytics/sessions/saveSessionData.ts
index ef32bcfb5..b1c5759b0 100644
--- a/src/queries/analytics/sessions/saveSessionData.ts
+++ b/src/queries/analytics/sessions/saveSessionData.ts
@@ -1,6 +1,6 @@
import { DATA_TYPE } from 'lib/constants';
import { uuid } from 'lib/crypto';
-import { flattenJSON } from 'lib/data';
+import { flattenJSON, getStringValue } from 'lib/data';
import prisma from 'lib/prisma';
import { DynamicData } from 'lib/types';
@@ -19,15 +19,10 @@ export async function saveSessionData(data: {
websiteId,
sessionId,
key: a.key,
- stringValue:
- a.dynamicDataType === DATA_TYPE.number
- ? parseFloat(a.value).toFixed(4)
- : a.dynamicDataType === DATA_TYPE.date
- ? a.value.split('.')[0] + 'Z'
- : a.value.toString(),
- numberValue: a.dynamicDataType === DATA_TYPE.number ? a.value : null,
- dateValue: a.dynamicDataType === DATA_TYPE.date ? new Date(a.value) : null,
- dataType: a.dynamicDataType,
+ stringValue: getStringValue(a.value, a.dataType),
+ numberValue: a.dataType === DATA_TYPE.number ? a.value : null,
+ dateValue: a.dataType === DATA_TYPE.date ? new Date(a.value) : null,
+ dataType: a.dataType,
}));
return transaction([