Umami is a privacy-first web and product analytics platform. Traffic, campaigns, behavior, conversions, and revenue in one place. No cookies, no surveillance, self-hosted or in the cloud.
TypeScript analyticsaudience-segmentationchartscohort-analysisgoogle-analyticsproduct-analyticsstatisticsuser-journeyweb-analytics
Feat/um 305 unique session ch (#2065)
Brian Caob48428652327
diff --git a/components/pages/console/TestConsole.js b/components/pages/console/TestConsole.jsindex 6c14c2c19..eda93f0b5 100644--- a/components/pages/console/TestConsole.js+++ b/components/pages/console/TestConsole.js@@ -28,22 +28,41 @@ export function TestConsole() {window.umami.track({ url: '/page-view', referrer: 'https://www.google.com' });window.umami.track('track-event-no-data');window.umami.track('track-event-with-data', {- data: {+ test: 'test-data',+ boolean: true,+ booleanError: 'true',+ time: new Date(),+ number: 1,+ time2: new Date().toISOString(),+ nested: {test: 'test-data',- boolean: true,- booleanError: 'true',- time: new Date(),number: 1,- time2: new Date().toISOString(),- nested: {+ object: {test: 'test-data',- number: 1,- object: {- test: 'test-data',- },},- array: [1, 2, 3],},+ array: [1, 2, 3],+ });+ }++ function handleIdentifyClick() {+ window.umami.identify({+ userId: 123,+ name: 'brian',+ number: Math.random() * 100,+ test: 'test-data',+ boolean: true,+ booleanError: 'true',+ time: new Date(),+ time2: new Date().toISOString(),+ nested: {+ test: 'test-data',+ number: 1,+ object: {+ test: 'test-data',+ },+ },+ array: [1, 2, 3],});}@@ -116,6 +135,10 @@ export function TestConsole() {<Button id="manual-button" variant="action" onClick={handleClick}>Run script</Button>+ <p />+ <Button id="manual-button" variant="action" onClick={handleIdentifyClick}>+ Run identify+ </Button></Column></Row><Row>diff --git a/db/clickhouse/migrations/01_edit_keys.sql b/db/clickhouse/migrations/01_edit_keys.sqlnew file mode 100644index 000000000..b948ff387--- /dev/null+++ b/db/clickhouse/migrations/01_edit_keys.sql@@ -0,0 +1,4 @@+ALTER TABLE "event_data" RENAME COLUMN "event_date_value" TO "date_value";+ALTER TABLE "event_data" RENAME COLUMN "event_numeric_value" TO "numeric_value";+ALTER TABLE "event_data" RENAME COLUMN "event_string_value" TO "string_value";+ALTER TABLE "event_data" RENAME COLUMN "event_data_type" TO "data_type";\ No newline at end of filediff --git a/db/clickhouse/schema.sql b/db/clickhouse/schema.sqlindex 8f48b4346..de1082ecb 100644--- a/db/clickhouse/schema.sql+++ b/db/clickhouse/schema.sql@@ -117,10 +117,10 @@ CREATE TABLE umami.event_dataurl_path String,event_name String,event_key String,- event_string_value Nullable(String),- event_numeric_value Nullable(Decimal64(4)), --922337203685477.5625- event_date_value Nullable(DateTime('UTC')),- event_data_type UInt32,+ string_value Nullable(String),+ numeric_value Nullable(Decimal64(4)), --922337203685477.5625+ date_value Nullable(DateTime('UTC')),+ data_type UInt32,created_at DateTime('UTC'))engine = MergeTree@@ -134,10 +134,10 @@ CREATE TABLE umami.event_data_queue (url_path String,event_name String,event_key String,- event_string_value Nullable(String),- event_numeric_value Nullable(Decimal64(4)), --922337203685477.5625- event_date_value Nullable(DateTime('UTC')),- event_data_type UInt32,+ string_value Nullable(String),+ numeric_value Nullable(Decimal64(4)), --922337203685477.5625+ date_value Nullable(DateTime('UTC')),+ data_type UInt32,created_at DateTime('UTC'),--virtual columns_error String,@@ -158,10 +158,10 @@ SELECT website_id,url_path,event_name,event_key,- event_string_value,- event_numeric_value,- event_date_value,- event_data_type,+ string_value,+ numeric_value,+ date_value,+ data_type,created_atFROM umami.event_data_queue;diff --git a/db/mysql/migrations/03_session_data/migration.sql b/db/mysql/migrations/03_session_data/migration.sqlnew file mode 100644index 000000000..3618080d2--- /dev/null+++ b/db/mysql/migrations/03_session_data/migration.sql@@ -0,0 +1,37 @@+/*+ Warnings:++ - The primary key for the `event_data` table will be changed. If it partially fails, the table could be left without primary key constraint.+ - You are about to drop the column `event_data_type` on the `event_data` table. All the data in the column will be lost.+ - You are about to drop the column `event_date_value` on the `event_data` table. All the data in the column will be lost.+ - You are about to drop the column `event_id` on the `event_data` table. All the data in the column will be lost.+ - You are about to drop the column `event_numeric_value` on the `event_data` table. All the data in the column will be lost.+ - You are about to drop the column `event_string_value` on the `event_data` table. All the data in the column will be lost.+ - Added the required column `data_type` to the `event_data` table without a default value. This is not possible if the table is not empty.+ - Added the required column `event_data_id` to the `event_data` table without a default value. This is not possible if the table is not empty.++*/+-- AlterTable+ALTER TABLE `event_data` RENAME COLUMN `event_data_type` TO `data_type`;+ALTER TABLE `event_data` RENAME COLUMN `event_date_value` TO `date_value`;+ALTER TABLE `event_data` RENAME COLUMN `event_id` TO `event_data_id`;+ALTER TABLE `event_data` RENAME COLUMN `event_numeric_value` TO `numeric_value`;+ALTER TABLE `event_data` RENAME COLUMN `event_string_value` TO `string_value`;++-- CreateTable+CREATE TABLE `session_data` (+ `session_data_id` VARCHAR(36) NOT NULL,+ `website_id` VARCHAR(36) NOT NULL,+ `session_id` VARCHAR(36) NOT NULL,+ `event_key` VARCHAR(500) NOT NULL,+ `event_string_value` VARCHAR(500) NULL,+ `event_numeric_value` DECIMAL(19, 4) NULL,+ `event_date_value` TIMESTAMP(0) NULL,+ `event_data_type` INTEGER UNSIGNED NOT NULL,+ `created_at` TIMESTAMP(0) NULL DEFAULT CURRENT_TIMESTAMP(0),++ INDEX `session_data_created_at_idx`(`created_at`),+ INDEX `session_data_website_id_idx`(`website_id`),+ INDEX `session_data_session_id_idx`(`session_id`),+ PRIMARY KEY (`session_data_id`)+) DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;diff --git a/db/mysql/schema.prisma b/db/mysql/schema.prismaindex 0752f4183..88d5ef804 100644--- a/db/mysql/schema.prisma+++ b/db/mysql/schema.prisma@@ -14,12 +14,12 @@ model User {password String @db.VarChar(60)role String @map("role") @db.VarChar(50)createdAt DateTime? @default(now()) @map("created_at") @db.Timestamp(0)- updatedAt DateTime? @map("updated_at") @updatedAt @db.Timestamp(0)+ updatedAt DateTime? @updatedAt @map("updated_at") @db.Timestamp(0)deletedAt DateTime? @map("deleted_at") @db.Timestamp(0)website Website[]teamUser TeamUser[]- Report Report[]+ report Report[]@@map("user")}@@ -40,6 +40,7 @@ model Session {createdAt DateTime? @default(now()) @map("created_at") @db.Timestamp(0)websiteEvent WebsiteEvent[]+ sessionData SessionData[]@@index([createdAt])@@index([websiteId])@@ -54,13 +55,14 @@ model Website {resetAt DateTime? @map("reset_at") @db.Timestamp(0)userId String? @map("user_id") @db.VarChar(36)createdAt DateTime? @default(now()) @map("created_at") @db.Timestamp(0)- updatedAt DateTime? @map("updated_at") @updatedAt @db.Timestamp(0)+ updatedAt DateTime? @updatedAt @map("updated_at") @db.Timestamp(0)deletedAt DateTime? @map("deleted_at") @db.Timestamp(0)user User? @relation(fields: [userId], references: [id])teamWebsite TeamWebsite[]eventData EventData[]- Report Report[]+ report Report[]+ sessionData SessionData[]@@index([userId])@@index([createdAt])@@ -94,9 +96,30 @@ model WebsiteEvent {}model EventData {- id String @id() @map("event_id") @db.VarChar(36)- websiteEventId String @map("website_event_id") @db.VarChar(36)+ id String @id() @map("event_data_id") @db.VarChar(36)+ websiteEventId String @map("website_event_id") @db.VarChar(36)+ websiteId String @map("website_id") @db.VarChar(36)+ eventKey String @map("event_key") @db.VarChar(500)+ stringValue String? @map("string_value") @db.VarChar(500)+ numericValue Decimal? @map("numeric_value") @db.Decimal(19, 4)+ dateValue DateTime? @map("date_value") @db.Timestamp(0)+ dataType Int @map("data_type") @db.UnsignedInt+ createdAt DateTime? @default(now()) @map("created_at") @db.Timestamp(0)++ website Website @relation(fields: [websiteId], references: [id])+ websiteEvent WebsiteEvent @relation(fields: [websiteEventId], references: [id])++ @@index([createdAt])+ @@index([websiteId])+ @@index([websiteEventId])+ @@index([websiteId, websiteEventId, createdAt])+ @@map("event_data")+}++model SessionData {+ id String @id() @map("session_data_id") @db.VarChar(36)websiteId String @map("website_id") @db.VarChar(36)+ sessionId String @map("session_id") @db.VarChar(36)eventKey String @map("event_key") @db.VarChar(500)eventStringValue String? @map("event_string_value") @db.VarChar(500)eventNumericValue Decimal? @map("event_numeric_value") @db.Decimal(19, 4)@@ -104,14 +127,13 @@ model EventData {eventDataType Int @map("event_data_type") @db.UnsignedIntcreatedAt DateTime? @default(now()) @map("created_at") @db.Timestamp(0)- website Website @relation(fields: [websiteId], references: [id])- websiteEvent WebsiteEvent @relation(fields: [websiteEventId], references: [id])+ website Website @relation(fields: [websiteId], references: [id])+ session Session @relation(fields: [sessionId], references: [id])@@index([createdAt])@@index([websiteId])- @@index([websiteEventId])- @@index([websiteId, websiteEventId, createdAt])- @@map("event_data")+ @@index([sessionId])+ @@map("session_data")}model Team {@@ -119,7 +141,7 @@ model Team {name String @db.VarChar(50)accessCode String? @unique @map("access_code") @db.VarChar(50)createdAt DateTime? @default(now()) @map("created_at") @db.Timestamp(0)- updatedAt DateTime? @map("updated_at") @updatedAt @db.Timestamp(0)+ updatedAt DateTime? @updatedAt @map("updated_at") @db.Timestamp(0)teamUser TeamUser[]teamWebsite TeamWebsite[]@@ -134,7 +156,7 @@ model TeamUser {userId String @map("user_id") @db.VarChar(36)role String @map("role") @db.VarChar(50)createdAt DateTime? @default(now()) @map("created_at") @db.Timestamp(0)- updatedAt DateTime? @map("updated_at") @updatedAt @db.Timestamp(0)+ updatedAt DateTime? @updatedAt @map("updated_at") @db.Timestamp(0)team Team @relation(fields: [teamId], references: [id])user User @relation(fields: [userId], references: [id])@@ -177,4 +199,4 @@ model Report {@@index([type])@@index([name])@@map("report")-}\ No newline at end of file+}diff --git a/db/postgresql/migrations/03_session_data/migration.sql b/db/postgresql/migrations/03_session_data/migration.sqlnew file mode 100644index 000000000..233c37935--- /dev/null+++ b/db/postgresql/migrations/03_session_data/migration.sql@@ -0,0 +1,31 @@+-- AlterTable+ALTER TABLE "event_data" RENAME COLUMN "event_data_type" TO "data_type";+ALTER TABLE "event_data" RENAME COLUMN "event_date_value" TO "date_value";+ALTER TABLE "event_data" RENAME COLUMN "event_id" TO "event_data_id";+ALTER TABLE "event_data" RENAME COLUMN "event_numeric_value" TO "numeric_value";+ALTER TABLE "event_data" RENAME COLUMN "event_string_value" TO "string_value";++-- CreateTable+CREATE TABLE "session_data" (+ "session_data_id" UUID NOT NULL,+ "website_id" UUID NOT NULL,+ "session_id" UUID NOT NULL,+ "session_key" VARCHAR(500) NOT NULL,+ "string_value" VARCHAR(500),+ "numeric_value" DECIMAL(19,4),+ "date_value" TIMESTAMPTZ(6),+ "data_type" INTEGER NOT NULL,+ "created_at" TIMESTAMPTZ(6) DEFAULT CURRENT_TIMESTAMP,+ "deleted_at" TIMESTAMPTZ(6) DEFAULT CURRENT_TIMESTAMP,++ CONSTRAINT "session_data_pkey" PRIMARY KEY ("session_data_id")+);++-- CreateIndex+CREATE INDEX "session_data_created_at_idx" ON "session_data"("created_at");++-- CreateIndex+CREATE INDEX "session_data_website_id_idx" ON "session_data"("website_id");++-- CreateIndex+CREATE INDEX "session_data_session_id_idx" ON "session_data"("session_id");diff --git a/db/postgresql/schema.prisma b/db/postgresql/schema.prismaindex 7c4cb94f9..bfd71fd12 100644--- a/db/postgresql/schema.prisma+++ b/db/postgresql/schema.prisma@@ -19,7 +19,7 @@ model User {website Website[]teamUser TeamUser[]- Report Report[]+ report Report[]@@map("user")}@@ -40,6 +40,7 @@ model Session {createdAt DateTime? @default(now()) @map("created_at") @db.Timestamptz(6)websiteEvent WebsiteEvent[]+ sessionData SessionData[]@@index([createdAt])@@index([websiteId])@@ -60,7 +61,8 @@ model Website {user User? @relation(fields: [userId], references: [id])teamWebsite TeamWebsite[]eventData EventData[]- Report Report[]+ report Report[]+ sessionData SessionData[]@@index([userId])@@index([createdAt])@@ -94,15 +96,15 @@ model WebsiteEvent {}model EventData {- id String @id() @map("event_id") @db.Uuid- websiteId String @map("website_id") @db.Uuid- websiteEventId String @map("website_event_id") @db.Uuid- eventKey String @map("event_key") @db.VarChar(500)- eventStringValue String? @map("event_string_value") @db.VarChar(500)- eventNumericValue Decimal? @map("event_numeric_value") @db.Decimal(19, 4)- eventDateValue DateTime? @map("event_date_value") @db.Timestamptz(6)- eventDataType Int @map("event_data_type") @db.Integer- createdAt DateTime? @default(now()) @map("created_at") @db.Timestamptz(6)+ id String @id() @map("event_data_id") @db.Uuid+ websiteId String @map("website_id") @db.Uuid+ websiteEventId String @map("website_event_id") @db.Uuid+ eventKey String @map("event_key") @db.VarChar(500)+ stringValue String? @map("string_value") @db.VarChar(500)+ numericValue Decimal? @map("numeric_value") @db.Decimal(19, 4)+ dateValue DateTime? @map("date_value") @db.Timestamptz(6)+ dataType Int @map("data_type") @db.Integer+ createdAt DateTime? @default(now()) @map("created_at") @db.Timestamptz(6)website Website @relation(fields: [websiteId], references: [id])websiteEvent WebsiteEvent @relation(fields: [websiteEventId], references: [id])@@ -113,6 +115,27 @@ model EventData {@@map("event_data")}+model SessionData {+ id String @id() @map("session_data_id") @db.Uuid+ websiteId String @map("website_id") @db.Uuid+ sessionId String @map("session_id") @db.Uuid+ sessionKey String @map("session_key") @db.VarChar(500)+ stringValue String? @map("string_value") @db.VarChar(500)+ numericValue Decimal? @map("numeric_value") @db.Decimal(19, 4)+ dateValue DateTime? @map("date_value") @db.Timestamptz(6)+ dataType Int @map("data_type") @db.Integer+ createdAt DateTime? @default(now()) @map("created_at") @db.Timestamptz(6)+ deletedAt DateTime? @default(now()) @map("deleted_at") @db.Timestamptz(6)++ website Website @relation(fields: [websiteId], references: [id])+ session Session @relation(fields: [sessionId], references: [id])++ @@index([createdAt])+ @@index([websiteId])+ @@index([sessionId])+ @@map("session_data")+}+model Team {id String @id() @unique() @map("team_id") @db.Uuidname String @db.VarChar(50)diff --git a/lib/clickhouse.ts b/lib/clickhouse.tsindex e97be8066..c91cf3da6 100644--- a/lib/clickhouse.ts+++ b/lib/clickhouse.ts@@ -2,7 +2,7 @@ import { ClickHouse } from 'clickhouse';import dateFormat from 'dateformat';import debug from 'debug';import { CLICKHOUSE } from 'lib/db';-import { getEventDataType } from './eventData';+import { getDynamicDataType } from './dynamicData';import { WebsiteMetricFilter } from './types';import { FILTER_COLUMNS } from './constants';@@ -74,7 +74,7 @@ function getEventDataFilterQuery(params: any,) {const query = filters.reduce((ac, cv, i) => {- const type = getEventDataType(cv.eventValue);+ const type = getDynamicDataType(cv.eventValue);let value = cv.eventValue;diff --git a/lib/constants.ts b/lib/constants.tsindex 7eef10b11..9edbe1004 100644--- a/lib/constants.ts+++ b/lib/constants.ts@@ -42,6 +42,11 @@ export const SESSION_COLUMNS = ['city',];+export const COLLECTION_TYPE = {+ event: 'event',+ identify: 'identify',+};+export const FILTER_COLUMNS = {url: 'url_path',referrer: 'referrer_domain',@@ -56,7 +61,7 @@ export const EVENT_TYPE = {customEvent: 2,} as const;-export const EVENT_DATA_TYPE = {+export const DYNAMIC_DATA_TYPE = {string: 1,number: 2,boolean: 3,diff --git a/lib/eventData.ts b/lib/dynamicData.tssimilarity index 63%rename from lib/eventData.tsrename to lib/dynamicData.tsindex aee1f9b43..da8eb8b2c 100644--- a/lib/eventData.ts+++ b/lib/dynamicData.ts@@ -1,12 +1,12 @@import { isValid, parseISO } from 'date-fns';-import { EVENT_DATA_TYPE } from './constants';-import { EventDataTypes } from './types';+import { DYNAMIC_DATA_TYPE } from './constants';+import { DynamicDataType } from './types';export function flattenJSON(eventData: { [key: string]: any },- keyValues: { key: string; value: any; eventDataType: EventDataTypes }[] = [],+ keyValues: { key: string; value: any; dynamicDataType: DynamicDataType }[] = [],parentKey = '',-): { key: string; value: any; eventDataType: EventDataTypes }[] {+): { key: string; value: any; dynamicDataType: DynamicDataType }[] {return Object.keys(eventData).reduce((acc, key) => {const value = eventData[key];@@ -25,7 +25,7 @@ export function flattenJSON().keyValues;}-export function getEventDataType(value: any): string {+export function getDynamicDataType(value: any): string {let type: string = typeof value;if ((type === 'string' && isValid(value)) || isValid(parseISO(value))) {@@ -36,34 +36,34 @@ export function getEventDataType(value: any): string {}function createKey(key, value, acc: { keyValues: any[]; parentKey: string }) {- const type = getEventDataType(value);+ const type = getDynamicDataType(value);- let eventDataType = null;+ let dynamicDataType = null;switch (type) {case 'number':- eventDataType = EVENT_DATA_TYPE.number;+ dynamicDataType = DYNAMIC_DATA_TYPE.number;break;case 'string':- eventDataType = EVENT_DATA_TYPE.string;+ dynamicDataType = DYNAMIC_DATA_TYPE.string;break;case 'boolean':- eventDataType = EVENT_DATA_TYPE.boolean;+ dynamicDataType = DYNAMIC_DATA_TYPE.boolean;value = value ? 'true' : 'false';break;case 'date':- eventDataType = EVENT_DATA_TYPE.date;+ dynamicDataType = DYNAMIC_DATA_TYPE.date;break;case 'object':- eventDataType = EVENT_DATA_TYPE.array;+ dynamicDataType = DYNAMIC_DATA_TYPE.array;value = JSON.stringify(value);break;default:- eventDataType = EVENT_DATA_TYPE.string;+ dynamicDataType = DYNAMIC_DATA_TYPE.string;break;}- acc.keyValues.push({ key, value, eventDataType });+ acc.keyValues.push({ key, value, dynamicDataType });}function getKeyName(key, parentKey) {diff --git a/lib/prisma.ts b/lib/prisma.tsindex fdd8a58dd..51707049d 100644--- a/lib/prisma.ts+++ b/lib/prisma.ts@@ -1,7 +1,7 @@import prisma from '@umami/prisma-client';import moment from 'moment-timezone';import { MYSQL, POSTGRESQL, getDatabaseType } from 'lib/db';-import { getEventDataType } from './eventData';+import { getDynamicDataType } from './dynamicData';import { FILTER_COLUMNS } from './constants';const MYSQL_DATE_FORMATS = {@@ -85,7 +85,7 @@ function getEventDataFilterQuery(params: any[],) {const query = filters.reduce((ac, cv) => {- const type = getEventDataType(cv.eventValue);+ const type = getDynamicDataType(cv.eventValue);let value = cv.eventValue;diff --git a/lib/session.ts b/lib/session.tsindex 1fedb91b8..04cbc9b0e 100644--- a/lib/session.ts+++ b/lib/session.ts@@ -1,12 +1,11 @@-import clickhouse from 'lib/clickhouse';import { secret, uuid } from 'lib/crypto';import { getClientInfo, getJsonBody } from 'lib/detect';import { parseToken } from 'next-basics';import { CollectRequestBody, NextApiRequestCollect } from 'pages/api/send';import { createSession } from 'queries';import { validate } from 'uuid';-import { loadSession, loadWebsite } from './query';import cache from './cache';+import { loadSession, loadWebsite } from './query';export async function findSession(req: NextApiRequestCollect) {const { payload } = getJsonBody<CollectRequestBody>(req);@@ -46,26 +45,8 @@ export async function findSession(req: NextApiRequestCollect) {const { userAgent, browser, os, ip, country, subdivision1, subdivision2, city, device } =await getClientInfo(req, payload);- const sessionId = uuid(websiteId, hostname, ip, userAgent);- // Clickhouse does not require session lookup- if (clickhouse.enabled) {- return {- id: sessionId,- websiteId,- hostname,- browser,- os,- device,- screen,- language,- country,- subdivision1,- subdivision2,- city,- ownerId: website.userId,- };- }+ const sessionId = uuid(websiteId, hostname, ip, userAgent);// Find sessionlet session = await loadSession(sessionId);diff --git a/lib/types.ts b/lib/types.tsindex 37c1ffdc9..05c091207 100644--- a/lib/types.ts+++ b/lib/types.ts@@ -1,18 +1,20 @@import { NextApiRequest } from 'next';-import { EVENT_DATA_TYPE, EVENT_TYPE, KAFKA_TOPIC, ROLES } from './constants';+import { COLLECTION_TYPE, DYNAMIC_DATA_TYPE, EVENT_TYPE, KAFKA_TOPIC, ROLES } from './constants';type ObjectValues<T> = T[keyof T];-export type Roles = ObjectValues<typeof ROLES>;+export type CollectionType = ObjectValues<typeof COLLECTION_TYPE>;-export type EventTypes = ObjectValues<typeof EVENT_TYPE>;+export type Role = ObjectValues<typeof ROLES>;-export type EventDataTypes = ObjectValues<typeof EVENT_DATA_TYPE>;+export type EventType = ObjectValues<typeof EVENT_TYPE>;-export type KafkaTopics = ObjectValues<typeof KAFKA_TOPIC>;+export type DynamicDataType = ObjectValues<typeof DYNAMIC_DATA_TYPE>;-export interface EventData {- [key: string]: number | string | EventData | number[] | string[] | EventData[];+export type KafkaTopic = ObjectValues<typeof KAFKA_TOPIC>;++export interface DynamicData {+ [key: string]: number | string | DynamicData | number[] | string[] | DynamicData[];}export interface Auth {diff --git a/pages/api/send.ts b/pages/api/send.tsindex df7ceb6ee..24264fa3c 100644--- a/pages/api/send.ts+++ b/pages/api/send.ts@@ -7,6 +7,9 @@ import { getJsonBody, getIpAddress } from 'lib/detect';import { secret } from 'lib/crypto';import { NextApiRequest, NextApiResponse } from 'next';import { Resolver } from 'dns/promises';+import { CollectionType } from 'lib/types';+import { COLLECTION_TYPE } from 'lib/constants';+import { saveSessionData } from 'queries/analytics/session/saveSessionData';export interface CollectRequestBody {payload: {@@ -20,7 +23,7 @@ export interface CollectRequestBody {website: string;name: string;};- type: string;+ type: CollectionType;}export interface NextApiRequestCollect extends NextApiRequest {@@ -52,17 +55,81 @@ export default async (req: NextApiRequestCollect, res: NextApiResponse) => {const { type, payload } = getJsonBody<CollectRequestBody>(req);- if (type !== 'event') {- return badRequest(res, 'Wrong payload type.');+ validateBody(res, { type, payload });++ if (await hasBlockedIp(req)) {+ return forbidden(res);}- const { url, referrer, name: eventName, data: eventData, title: pageTitle } = payload;+ const { url, referrer, name: eventName, data: dynamicData, title: pageTitle } = payload;++ await useSession(req, res);++ const session = req.session;++ if (type === COLLECTION_TYPE.event) {+ // eslint-disable-next-line prefer-const+ let [urlPath, urlQuery] = url?.split('?') || [];+ let [referrerPath, referrerQuery] = referrer?.split('?') || [];+ let referrerDomain;++ if (!urlPath) {+ urlPath = '/';+ }++ if (referrerPath?.startsWith('http')) {+ const refUrl = new URL(referrer);+ referrerPath = refUrl.pathname;+ referrerQuery = refUrl.search.substring(1);+ referrerDomain = refUrl.hostname.replace(/www\./, '');+ }++ if (process.env.REMOVE_TRAILING_SLASH) {+ urlPath = urlPath.replace(/.+\/$/, '');+ }++ await saveEvent({+ urlPath,+ urlQuery,+ referrerPath,+ referrerQuery,+ referrerDomain,+ pageTitle,+ eventName,+ eventData: dynamicData,+ ...session,+ sessionId: session.id,+ });+ }++ if (type === COLLECTION_TYPE.identify) {+ if (!dynamicData) {+ return badRequest(res, 'Data required.');+ }++ await saveSessionData({ ...session, sessionData: dynamicData, sessionId: session.id });+ }++ const token = createToken(session, secret());++ return send(res, token);+};++function validateBody(res: NextApiResponse, { type, payload }: CollectRequestBody) {+ const { data } = payload;++ // Validate type+ if (type !== COLLECTION_TYPE.event && type !== COLLECTION_TYPE.identify) {+ return badRequest(res, 'Wrong payload type.');+ }// Validate eventData is JSON- if (eventData && !(typeof eventData === 'object' && !Array.isArray(eventData))) {+ if (data && !(typeof data === 'object' && !Array.isArray(data))) {return badRequest(res, 'Invalid event data.');}+}+async function hasBlockedIp(req: NextApiRequestCollect) {const ignoreIps = process.env.IGNORE_IP;const ignoreHostnames = process.env.IGNORE_HOSTNAME;@@ -100,49 +167,6 @@ export default async (req: NextApiRequestCollect, res: NextApiResponse) => {return false;});- if (blocked) {- return forbidden(res);- }- }-- await useSession(req, res);-- const session = req.session;-- // eslint-disable-next-line prefer-const- let [urlPath, urlQuery] = url?.split('?') || [];- let [referrerPath, referrerQuery] = referrer?.split('?') || [];- let referrerDomain;-- if (!urlPath) {- urlPath = '/';- }-- if (referrerPath?.startsWith('http')) {- const refUrl = new URL(referrer);- referrerPath = refUrl.pathname;- referrerQuery = refUrl.search.substring(1);- referrerDomain = refUrl.hostname.replace(/www\./, '');+ return blocked;}-- if (process.env.REMOVE_TRAILING_SLASH) {- urlPath = urlPath.replace(/.+\/$/, '');- }-- await saveEvent({- urlPath,- urlQuery,- referrerPath,- referrerQuery,- referrerDomain,- pageTitle,- eventName,- eventData,- ...session,- sessionId: session.id,- });-- const token = createToken(session, secret());-- return send(res, token);-};+}diff --git a/pages/api/users/[id]/index.ts b/pages/api/users/[id]/index.tsindex de4642cb9..a4ab05fff 100644--- a/pages/api/users/[id]/index.ts+++ b/pages/api/users/[id]/index.ts@@ -1,4 +1,4 @@-import { NextApiRequestQueryBody, Roles, User } from 'lib/types';+import { NextApiRequestQueryBody, Role, User } from 'lib/types';import { canDeleteUser, canUpdateUser, canViewUser } from 'lib/auth';import { useAuth } from 'lib/middleware';import { NextApiResponse } from 'next';@@ -12,7 +12,7 @@ export interface UserRequestQuery {export interface UserRequestBody {username: string;password: string;- role: Roles;+ role: Role;}export default async (diff --git a/pages/api/users/index.ts b/pages/api/users/index.tsindex 4d35d8567..c6103c35b 100644--- a/pages/api/users/index.ts+++ b/pages/api/users/index.ts@@ -2,7 +2,7 @@ import { canCreateUser, canViewUsers } from 'lib/auth';import { ROLES } from 'lib/constants';import { uuid } from 'lib/crypto';import { useAuth } from 'lib/middleware';-import { NextApiRequestQueryBody, Roles, User } from 'lib/types';+import { NextApiRequestQueryBody, Role, User } from 'lib/types';import { NextApiResponse } from 'next';import { badRequest, hashPassword, methodNotAllowed, ok, unauthorized } from 'next-basics';import { createUser, getUser, getUsers } from 'queries';@@ -11,7 +11,7 @@ export interface UsersRequestBody {username: string;password: string;id: string;- role?: Roles;+ role?: Role;}export default async (diff --git a/queries/admin/user.ts b/queries/admin/user.tsindex 3ba2654b9..ec23559f3 100644--- a/queries/admin/user.ts+++ b/queries/admin/user.ts@@ -3,7 +3,7 @@ import { getRandomChars } from 'next-basics';import cache from 'lib/cache';import { ROLES } from 'lib/constants';import prisma from 'lib/prisma';-import { Website, User, Roles } from 'lib/types';+import { Website, User, Role } from 'lib/types';export async function getUser(where: Prisma.UserWhereInput | Prisma.UserWhereUniqueInput,@@ -91,7 +91,7 @@ export async function createUser(data: {id: string;username: string;password: string;- role: Roles;+ role: Role;}): Promise<{id: string;username: string;diff --git a/queries/analytics/event/saveEvent.ts b/queries/analytics/event/saveEvent.tsindex 9a7db00d1..51087a59d 100644--- a/queries/analytics/event/saveEvent.ts+++ b/queries/analytics/event/saveEvent.ts@@ -133,9 +133,10 @@ async function clickhouseQuery(data: {const createdAt = getDateFormat(new Date());const message = {+ ...args,website_id: websiteId,session_id: sessionId,- event_id: eventId,+ event_id: uuid(),country: country ? country : null,subdivision1: country && subdivision1 ? `${country}-${subdivision1}` : null,subdivision2: subdivision2 ? subdivision2 : null,@@ -149,7 +150,6 @@ async function clickhouseQuery(data: {event_type: eventName ? EVENT_TYPE.customEvent : EVENT_TYPE.pageView,event_name: eventName ? eventName?.substring(0, EVENT_NAME_LENGTH) : null,created_at: createdAt,- ...args,};await sendMessage(message, 'event');diff --git a/queries/analytics/eventData/saveEventData.ts b/queries/analytics/eventData/saveEventData.tsindex 90e635656..96ea88312 100644--- a/queries/analytics/eventData/saveEventData.ts+++ b/queries/analytics/eventData/saveEventData.ts@@ -1,11 +1,11 @@import { Prisma } from '@prisma/client';-import { EVENT_DATA_TYPE } from 'lib/constants';+import { DYNAMIC_DATA_TYPE } from 'lib/constants';import { uuid } from 'lib/crypto';import { CLICKHOUSE, PRISMA, runQuery } from 'lib/db';-import { flattenJSON } from 'lib/eventData';+import { flattenJSON } from 'lib/dynamicData';import kafka from 'lib/kafka';import prisma from 'lib/prisma';-import { EventData } from 'lib/types';+import { DynamicData } from 'lib/types';export async function saveEventData(args: {websiteId: string;@@ -13,7 +13,7 @@ export async function saveEventData(args: {sessionId?: string;urlPath?: string;eventName?: string;- eventData: EventData;+ eventData: DynamicData;createdAt?: string;}) {return runQuery({@@ -25,7 +25,7 @@ export async function saveEventData(args: {async function relationalQuery(data: {websiteId: string;eventId: string;- eventData: EventData;+ eventData: DynamicData;}): Promise<Prisma.BatchPayload> {const { websiteId, eventId, eventData } = data;@@ -36,16 +36,16 @@ async function relationalQuery(data: {id: uuid(),websiteEventId: eventId,websiteId,- eventKey: a.key,- eventStringValue:- a.eventDataType === EVENT_DATA_TYPE.string ||- a.eventDataType === EVENT_DATA_TYPE.boolean ||- a.eventDataType === EVENT_DATA_TYPE.array+ key: a.key,+ stringValue:+ a.dynamicDataType === DYNAMIC_DATA_TYPE.string ||+ a.dynamicDataType === DYNAMIC_DATA_TYPE.boolean ||+ a.dynamicDataType === DYNAMIC_DATA_TYPE.array? a.value: null,- eventNumericValue: a.eventDataType === EVENT_DATA_TYPE.number ? a.value : null,- eventDateValue: a.eventDataType === EVENT_DATA_TYPE.date ? new Date(a.value) : null,- eventDataType: a.eventDataType,+ numericValue: a.dynamicDataType === DYNAMIC_DATA_TYPE.number ? a.value : null,+ dateValue: a.dynamicDataType === DYNAMIC_DATA_TYPE.date ? new Date(a.value) : null,+ dataType: a.dynamicDataType,}));return prisma.client.eventData.createMany({@@ -59,7 +59,7 @@ async function clickhouseQuery(data: {sessionId?: string;urlPath?: string;eventName?: string;- eventData: EventData;+ eventData: DynamicData;createdAt?: string;}) {const { websiteId, sessionId, eventId, urlPath, eventName, eventData, createdAt } = data;@@ -75,15 +75,15 @@ async function clickhouseQuery(data: {url_path: urlPath,event_name: eventName,event_key: a.key,- event_string_value:- a.eventDataType === EVENT_DATA_TYPE.string ||- a.eventDataType === EVENT_DATA_TYPE.boolean ||- a.eventDataType === EVENT_DATA_TYPE.array+ string_value:+ a.dynamicDataType === DYNAMIC_DATA_TYPE.string ||+ a.dynamicDataType === DYNAMIC_DATA_TYPE.boolean ||+ a.dynamicDataType === DYNAMIC_DATA_TYPE.array? a.value: null,- event_numeric_value: a.eventDataType === EVENT_DATA_TYPE.number ? a.value : null,- event_date_value: a.eventDataType === EVENT_DATA_TYPE.date ? getDateFormat(a.value) : null,- event_data_type: a.eventDataType,+ numeric_value: a.dynamicDataType === DYNAMIC_DATA_TYPE.number ? a.value : null,+ date_value: a.dynamicDataType === DYNAMIC_DATA_TYPE.date ? getDateFormat(a.value) : null,+ data_type: a.dynamicDataType,created_at: createdAt,}));diff --git a/queries/analytics/session/createSession.ts b/queries/analytics/session/createSession.tsindex 22f7892f7..4fd36d2e0 100644--- a/queries/analytics/session/createSession.ts+++ b/queries/analytics/session/createSession.ts@@ -1,70 +1,8 @@-import { CLICKHOUSE, PRISMA, runQuery } from 'lib/db';-import kafka from 'lib/kafka';-import prisma from 'lib/prisma';-import cache from 'lib/cache';import { Prisma } from '@prisma/client';+import cache from 'lib/cache';+import prisma from 'lib/prisma';-export async function createSession(args: Prisma.SessionCreateInput) {- return runQuery({- [PRISMA]: () => relationalQuery(args),- [CLICKHOUSE]: () => clickhouseQuery(args),- }).then(async data => {- if (cache.enabled) {- await cache.storeSession(data);- }-- return data;- });-}--async function relationalQuery(data: Prisma.SessionCreateInput) {- const {- id,- websiteId,- hostname,- browser,- os,- device,- screen,- language,- country,- subdivision1,- subdivision2,- city,- } = data;-- return prisma.client.session.create({- data: {- id,- websiteId,- hostname,- browser,- os,- device,- screen,- language,- country,- subdivision1: country && subdivision1 ? `${country}-${subdivision1}` : null,- subdivision2,- city,- },- });-}--async function clickhouseQuery(data: {- id: string;- websiteId: string;- hostname?: string;- browser?: string;- os?: string;- device?: string;- screen?: string;- language?: string;- country?: string;- subdivision1?: string;- subdivision2?: string;- city?: string;-}) {+export async function createSession(data: Prisma.SessionCreateInput) {const {id,websiteId,@@ -79,25 +17,29 @@ async function clickhouseQuery(data: {subdivision2,city,} = data;- const { getDateFormat, sendMessage } = kafka;-- const msg = {- session_id: id,- website_id: websiteId,- hostname,- browser,- os,- device,- screen,- language,- country,- subdivision1,- subdivision2,- city,- created_at: getDateFormat(new Date()),- };-- await sendMessage(msg, 'event');- return data;+ return prisma.client.session+ .create({+ data: {+ id,+ websiteId,+ hostname,+ browser,+ os,+ device,+ screen,+ language,+ country,+ subdivision1: country && subdivision1 ? `${country}-${subdivision1}` : null,+ subdivision2,+ city,+ },+ })+ .then(async data => {+ if (cache.enabled) {+ await cache.storeSession(data);+ }++ return data;+ });}diff --git a/queries/analytics/session/getSession.ts b/queries/analytics/session/getSession.tsindex d226e8326..2fd8d18f6 100644--- a/queries/analytics/session/getSession.ts+++ b/queries/analytics/session/getSession.ts@@ -1,43 +1,8 @@-import clickhouse from 'lib/clickhouse';-import { CLICKHOUSE, PRISMA, runQuery } from 'lib/db';-import prisma from 'lib/prisma';import { Prisma } from '@prisma/client';+import prisma from 'lib/prisma';-export async function getSession(args: { id: string }) {- return runQuery({- [PRISMA]: () => relationalQuery(args),- [CLICKHOUSE]: () => clickhouseQuery(args),- });-}--async function relationalQuery(where: Prisma.SessionWhereUniqueInput) {+export async function getSession(where: Prisma.SessionWhereUniqueInput) {return prisma.client.session.findUnique({where,});}--async function clickhouseQuery({ id: sessionId }: { id: string }) {- const { rawQuery, findFirst } = clickhouse;- const params = { sessionId };-- return rawQuery(- `select- session_id,- website_id,- created_at,- hostname,- browser,- os,- device,- screen,- language,- country,- subdivision1,- subdivision2,- city- from website_event- where session_id = {sessionId:UUID}- limit 1`,- params,- ).then(result => findFirst(result));-}diff --git a/queries/analytics/session/saveSessionData.ts b/queries/analytics/session/saveSessionData.tsnew file mode 100644index 000000000..76842b4f6--- /dev/null+++ b/queries/analytics/session/saveSessionData.ts@@ -0,0 +1,43 @@+import { DYNAMIC_DATA_TYPE } from 'lib/constants';+import { uuid } from 'lib/crypto';+import { flattenJSON } from 'lib/dynamicData';+import prisma from 'lib/prisma';+import { DynamicData } from 'lib/types';++export async function saveSessionData(data: {+ websiteId: string;+ sessionId: string;+ sessionData: DynamicData;+}) {+ const { client, transaction } = prisma;+ const { websiteId, sessionId, sessionData } = data;++ const jsonKeys = flattenJSON(sessionData);++ const flattendData = jsonKeys.map(a => ({+ id: uuid(),+ websiteId,+ sessionId,+ key: a.key,+ stringValue:+ a.dynamicDataType === DYNAMIC_DATA_TYPE.string ||+ a.dynamicDataType === DYNAMIC_DATA_TYPE.boolean ||+ a.dynamicDataType === DYNAMIC_DATA_TYPE.array+ ? a.value+ : null,+ numericValue: a.dynamicDataType === DYNAMIC_DATA_TYPE.number ? a.value : null,+ dateValue: a.dynamicDataType === DYNAMIC_DATA_TYPE.date ? new Date(a.value) : null,+ dataType: a.dynamicDataType,+ }));++ return transaction([+ client.sessionData.deleteMany({+ where: {+ sessionId,+ },+ }),+ client.sessionData.createMany({+ data: flattendData,+ }),+ ]);+}diff --git a/tracker/index.js b/tracker/index.jsindex 1c40036e9..6127ed191 100644--- a/tracker/index.js+++ b/tracker/index.js@@ -173,7 +173,7 @@}};- const send = payload => {+ const send = (payload, type = 'event') => {if (trackingDisabled()) return;const headers = {'Content-Type': 'application/json',@@ -183,7 +183,7 @@}return fetch(endpoint, {method: 'POST',- body: JSON.stringify({ type: 'event', payload }),+ body: JSON.stringify({ type, payload }),headers,}).then(res => res.text())@@ -205,11 +205,14 @@return send(getPayload());};+ const identify = data => send({ ...getPayload(), data }, 'identify');+/* Start */if (!window.umami) {window.umami = {track,+ identify,};}