Umami is a privacy-first web and product analytics platform. Traffic, campaigns, behavior, conversions, and revenue in one place. No cookies, no surveillance, self-hosted or in the cloud.
TypeScript analyticsaudience-segmentationchartscohort-analysisgoogle-analyticsproduct-analyticsstatisticsuser-journeyweb-analytics
Database refactoring.
Mike Cao467c7f289f29
diff --git a/lib/clickhouse.js b/lib/clickhouse.jsindex 7ab06b9e2..1548b77ba 100644--- a/lib/clickhouse.js+++ b/lib/clickhouse.js@@ -1,39 +1,50 @@import { ClickHouse } from 'clickhouse';import dateFormat from 'dateformat';+import debug from 'debug';import { FILTER_IGNORED } from 'lib/constants';-import { CLICKHOUSE_DATE_FORMATS } from './constants';+import { CLICKHOUSE } from 'lib/db';++export const CLICKHOUSE_DATE_FORMATS = {+ minute: '%Y-%m-%d %H:%M:00',+ hour: '%Y-%m-%d %H:00:00',+ day: '%Y-%m-%d',+ month: '%Y-%m-01',+ year: '%Y-01-01',+};++const log = debug('clickhouse');function getClient() {- if (!process.env.ANALYTICS_URL) {+ if (!process.env.CLICKHOUSE_URL) {return null;}- const url = new URL(process.env.ANALYTICS_URL);- const database = url.pathname.replace('/', '');-- return new ClickHouse({- url: url.hostname,- port: Number(url.port),- basicAuth: url.password- ? {- username: url.username || 'default',- password: url.password,- }- : null,+ const {+ hostname,+ port,+ pathname,+ username = 'default',+ password,+ } = new URL(process.env.CLICKHOUSE_URL);++ const client = new ClickHouse({+ url: hostname,+ port: Number(port),format: 'json',config: {- database,+ database: pathname.replace('/', ''),},+ basicAuth: password ? { username, password } : null,});-}-const clickhouse = global.clickhouse || getClient();+ if (process.env.NODE_ENV !== 'production') {+ global[CLICKHOUSE] = clickhouse;+ }-if (process.env.NODE_ENV !== 'production') {- global.clickhouse = clickhouse;-}+ log('Clickhouse initialized');-export { clickhouse };+ return client;+}function getDateStringQuery(data, unit) {return `formatDateTime(${data}, '${CLICKHOUSE_DATE_FORMATS[unit]}')`;@@ -176,7 +187,12 @@ async function findFirst(data) {return data[0] ?? null;}+// Initialization+const clickhouse = global[CLICKHOUSE] || getClient();+export default {+ client: clickhouse,+ log,getDateStringQuery,getDateQuery,getDateFormat,diff --git a/lib/constants.js b/lib/constants.jsindex e20a74e67..feef540df 100644--- a/lib/constants.js+++ b/lib/constants.js@@ -67,36 +67,6 @@ export const EVENT_COLORS = ['#ffec16',];-export const RELATIONAL = 'relational';-export const POSTGRESQL = 'postgresql';-export const MYSQL = 'mysql';-export const CLICKHOUSE = 'clickhouse';-export const KAFKA = 'kafka';--export const MYSQL_DATE_FORMATS = {- minute: '%Y-%m-%d %H:%i:00',- hour: '%Y-%m-%d %H:00:00',- day: '%Y-%m-%d',- month: '%Y-%m-01',- year: '%Y-01-01',-};--export const POSTGRESQL_DATE_FORMATS = {- minute: 'YYYY-MM-DD HH24:MI:00',- hour: 'YYYY-MM-DD HH24:00:00',- day: 'YYYY-MM-DD',- month: 'YYYY-MM-01',- year: 'YYYY-01-01',-};--export const CLICKHOUSE_DATE_FORMATS = {- minute: '%Y-%m-%d %H:%M:00',- hour: '%Y-%m-%d %H:00:00',- day: '%Y-%m-%d',- month: '%Y-%m-01',- year: '%Y-01-01',-};-export const FILTER_IGNORED = Symbol.for('filter-ignored');export const DOMAIN_REGEX =@@ -107,6 +77,7 @@ export const LAPTOP_SCREEN_WIDTH = 1024;export const MOBILE_SCREEN_WIDTH = 479;export const URL_LENGTH = 500;+export const EVENT_NAME_LENGTH = 50;export const DESKTOP_OS = ['Windows 3.11',diff --git a/lib/db.js b/lib/db.jsindex 13139cd58..9db00c924 100644--- a/lib/db.js+++ b/lib/db.js@@ -1,5 +1,11 @@-import { POSTGRESQL, RELATIONAL, MYSQL, KAFKA, CLICKHOUSE } from 'lib/constants';-+export const PRISMA = 'prisma';+export const POSTGRESQL = 'postgresql';+export const MYSQL = 'mysql';+export const CLICKHOUSE = 'clickhouse';+export const KAFKA = 'kafka';+export const KAFKA_PRODUCER = 'kafka-producer';++// Fixes issue with converting bigint valuesBigInt.prototype.toJSON = function () {return Number(this);};@@ -14,11 +20,11 @@ export function getDatabaseType(url = process.env.DATABASE_URL) {return type;}-export async function runAnalyticsQuery(queries) {- const db = getDatabaseType(process.env.ANALYTICS_URL || process.env.DATABASE_URL);+export async function runQuery(queries) {+ const db = getDatabaseType(process.env.CLICKHOUSE_URL || process.env.DATABASE_URL);if (db === POSTGRESQL || db === MYSQL) {- return queries[RELATIONAL]();+ return queries[PRISMA]();}if (db === CLICKHOUSE) {diff --git a/lib/format.js b/lib/format.jsindex a336c1c44..a85c3ef76 100644--- a/lib/format.js+++ b/lib/format.js@@ -74,7 +74,7 @@ export function stringToColor(str) {let color = '#';for (let i = 0; i < 3; i++) {let value = (hash >> (i * 8)) & 0xff;- color += ('00' + value.toString(16)).substr(-2);+ color += ('00' + value.toString(16)).substring(-2);}return color;}diff --git a/lib/kafka.js b/lib/kafka.jsindex 393207285..01b69603c 100644--- a/lib/kafka.js+++ b/lib/kafka.js@@ -1,57 +1,53 @@import { Kafka, logLevel } from 'kafkajs';import dateFormat from 'dateformat';+import debug from 'debug';+import { KAFKA, KAFKA_PRODUCER } from 'lib/db';-export function getClient() {- if (!process.env.KAFKA_URL) {+const log = debug('kafka');++function getClient() {+ if (!process.env.KAFKA_URL || !process.env.KAFKA_BROKER) {return null;}- const url = new URL(process.env.KAFKA_URL);+ const { username, password } = new URL(process.env.KAFKA_URL);const brokers = process.env.KAFKA_BROKER.split(',');- if (url.username.length === 0 && url.password.length === 0) {- return new Kafka({- clientId: 'umami',- brokers: brokers,- connectionTimeout: 3000,- logLevel: logLevel.ERROR,- });- } else {- return new Kafka({- clientId: 'umami',- brokers: brokers,- connectionTimeout: 3000,- ssl: true,- sasl: {- mechanism: 'plain',- username: url.username,- password: url.password,- },- });- }-}-const kafka = global.kafka || getClient();-let kafkaProducer = null;+ const ssl =+ username && password+ ? {+ ssl: true,+ sasl: {+ mechanism: 'plain',+ username,+ password,+ },+ }+ : {};-(async () => {- if (kafka) {- kafkaProducer = global.kakfaProducer || (await getProducer());- }+ const client = new Kafka({+ clientId: 'umami',+ brokers: brokers,+ connectionTimeout: 3000,+ logLevel: logLevel.ERROR,+ ...ssl,+ });if (process.env.NODE_ENV !== 'production') {- global.kafka = kafka;- if (kafka) {- global.kakfaProducer = kafkaProducer;- }+ global[KAFKA] = client;}-})();-export { kafka, kafkaProducer };+ return client;+}async function getProducer() {const producer = kafka.producer();await producer.connect();+ if (process.env.NODE_ENV !== 'production') {+ global[KAFKA_PRODUCER] = producer;+ }+return producer;}@@ -60,7 +56,7 @@ function getDateFormat(date) {}async function sendMessage(params, topic) {- await kafkaProducer.send({+ await producer.send({topic,messages: [{@@ -72,7 +68,19 @@ async function sendMessage(params, topic) {});}+// Initialization+let kafka;+let producer;++(async () => {+ kafka = global[KAFKA] || getClient();+ producer = global[KAFKA_PRODUCER] || (await getProducer());+})();+export default {+ client: kafka,+ producer: producer,+ log,getDateFormat,sendMessage,};diff --git a/lib/relational.js b/lib/prisma.jssimilarity index 78%rename from lib/relational.jsrename to lib/prisma.jsindex d86f06075..af8759ba2 100644--- a/lib/relational.js+++ b/lib/prisma.js@@ -1,19 +1,30 @@import { PrismaClient } from '@prisma/client';import chalk from 'chalk';-import {- FILTER_IGNORED,- MYSQL,- MYSQL_DATE_FORMATS,- POSTGRESQL,- POSTGRESQL_DATE_FORMATS,-} from 'lib/constants';-import { getDatabaseType } from 'lib/db';import moment from 'moment-timezone';import debug from 'debug';+import { PRISMA, MYSQL, POSTGRESQL } from 'lib/db';+import { FILTER_IGNORED } from 'lib/constants';+import { getDatabaseType } from 'lib/db';++const MYSQL_DATE_FORMATS = {+ minute: '%Y-%m-%d %H:%i:00',+ hour: '%Y-%m-%d %H:00:00',+ day: '%Y-%m-%d',+ month: '%Y-%m-01',+ year: '%Y-01-01',+};++const POSTGRESQL_DATE_FORMATS = {+ minute: 'YYYY-MM-DD HH24:MI:00',+ hour: 'YYYY-MM-DD HH24:00:00',+ day: 'YYYY-MM-DD',+ month: 'YYYY-MM-01',+ year: 'YYYY-01-01',+};const log = debug('prisma');-const options = {+const PRISMA_OPTIONS = {log: [{emit: 'event',@@ -33,16 +44,16 @@ function getClient(options) {prisma.$on('query', logQuery);}- return prisma;-}+ if (process.env.NODE_ENV !== 'production') {+ global[PRISMA] = prisma;+ }-let prisma = global.prisma || getClient(options);+ log('Prisma initialized');-if (process.env.NODE_ENV !== 'production') {- global.prisma = prisma;+ return prisma;}-export function getDateQuery(field, unit, timezone) {+function getDateQuery(field, unit, timezone) {const db = getDatabaseType(process.env.DATABASE_URL);if (db === POSTGRESQL) {@@ -63,7 +74,7 @@ export function getDateQuery(field, unit, timezone) {}}-export function getTimestampInterval(field) {+function getTimestampInterval(field) {const db = getDatabaseType(process.env.DATABASE_URL);if (db === POSTGRESQL) {@@ -75,7 +86,7 @@ export function getTimestampInterval(field) {}}-export function getFilterQuery(table, column, filters = {}, params = []) {+function getFilterQuery(table, column, filters = {}, params = []) {const query = Object.keys(filters).reduce((arr, key) => {const filter = filters[key];@@ -135,7 +146,7 @@ export function getFilterQuery(table, column, filters = {}, params = []) {return query.join('\n');}-export function parseFilters(table, column, filters = {}, params = [], sessionKey = 'session_id') {+function parseFilters(table, column, filters = {}, params = [], sessionKey = 'session_id') {const { domain, url, event_url, referrer, os, browser, device, country, event_name, query } =filters;@@ -158,13 +169,7 @@ export function parseFilters(table, column, filters = {}, params = [], sessionKe};}-export async function runQuery(query) {- return query.catch(e => {- throw e;- });-}--export async function rawQuery(query, params = []) {+async function rawQuery(query, params = []) {const db = getDatabaseType(process.env.DATABASE_URL);if (db !== POSTGRESQL && db !== MYSQL) {@@ -173,17 +178,23 @@ export async function rawQuery(query, params = []) {const sql = db === MYSQL ? query.replace(/\$[0-9]+/g, '?') : query;- return runQuery(prisma.$queryRawUnsafe.apply(prisma, [sql, ...params]));+ return prisma.$queryRawUnsafe.apply(prisma, [sql, ...params]);+}++async function multiQuery(queries) {+ return prisma.$transaction(queries);}-export { prisma };+// Initialization+const prisma = global[PRISMA] || getClient(PRISMA_OPTIONS);export default {- prisma,+ client: prisma,+ log,getDateQuery,getTimestampInterval,getFilterQuery,parseFilters,- runQuery,rawQuery,+ multiQuery,};diff --git a/lib/redis.js b/lib/redis.jsindex 97011bed5..e307729b3 100644--- a/lib/redis.js+++ b/lib/redis.js@@ -1,6 +1,10 @@import { createClient } from 'redis';import { startOfMonth } from 'date-fns';import { getSessions, getAllWebsites } from '/queries';+import debug from 'debug';++const log = debug('db:redis');+const REDIS = Symbol.for('redis');async function getClient() {const redis = new createClient({@@ -10,29 +14,15 @@ async function getClient() {await redis.connect();if (process.env.LOG_QUERY) {- redis.on('error', err => console.log('Redis Client Error', err));+ redis.on('error', err => log(err));}- return redis;-}--let redis = null;--(async () => {- redis = global.redis || (await getClient());-if (process.env.NODE_ENV !== 'production') {- global.redis = redis;+ global[REDIS] = redis;}- const value = await redis.get('initialized');-- if (!value) {- await stageData();- }-})();--export default redis;+ return redis;+}async function stageData() {const sessions = await getSessions([], startOfMonth(new Date()).toUTCString());@@ -57,3 +47,18 @@ async function addRedis(ids) {await redis.set(key, value);}}++// Initialization+let redis = null;++(async () => {+ redis = global[REDIS] || (await getClient());++ const value = await redis.get('initialized');++ if (!value) {+ await stageData();+ }+})();++export default redis;diff --git a/queries/admin/account/createAccount.js b/queries/admin/account/createAccount.jsindex fb3997e87..2c4e1a40b 100644--- a/queries/admin/account/createAccount.js+++ b/queries/admin/account/createAccount.js@@ -1,9 +1,7 @@-import { prisma, runQuery } from 'lib/relational';+import prisma from 'lib/prisma';export async function createAccount(data) {- return runQuery(- prisma.account.create({- data,- }),- );+ return prisma.client.account.create({+ data,+ });}diff --git a/queries/admin/account/deleteAccount.js b/queries/admin/account/deleteAccount.jsindex f03b370ef..209d73a95 100644--- a/queries/admin/account/deleteAccount.js+++ b/queries/admin/account/deleteAccount.js@@ -1,28 +1,28 @@-import { prisma, runQuery } from 'lib/relational';+import prisma from 'lib/prisma';export async function deleteAccount(user_id) {- return runQuery(- prisma.$transaction([- prisma.pageview.deleteMany({- where: { session: { website: { user_id } } },- }),- prisma.event_data.deleteMany({- where: { event: { session: { website: { user_id } } } },- }),- prisma.event.deleteMany({- where: { session: { website: { user_id } } },- }),- prisma.session.deleteMany({- where: { website: { user_id } },- }),- prisma.website.deleteMany({- where: { user_id },- }),- prisma.account.delete({- where: {- user_id,- },- }),- ]),- );+ const { client } = prisma;++ return client.$transaction([+ client.pageview.deleteMany({+ where: { session: { website: { user_id } } },+ }),+ client.event_data.deleteMany({+ where: { event: { session: { website: { user_id } } } },+ }),+ client.event.deleteMany({+ where: { session: { website: { user_id } } },+ }),+ client.session.deleteMany({+ where: { website: { user_id } },+ }),+ client.website.deleteMany({+ where: { user_id },+ }),+ client.account.delete({+ where: {+ user_id,+ },+ }),+ ]);}diff --git a/queries/admin/account/getAccountById.js b/queries/admin/account/getAccountById.jsindex 4c3d647ce..74c4a4a96 100644--- a/queries/admin/account/getAccountById.js+++ b/queries/admin/account/getAccountById.js@@ -1,11 +1,9 @@-import { prisma, runQuery } from 'lib/relational';+import prisma from 'lib/prisma';export async function getAccountById(user_id) {- return runQuery(- prisma.account.findUnique({- where: {- user_id,- },- }),- );+ return prisma.client.account.findUnique({+ where: {+ user_id,+ },+ });}diff --git a/queries/admin/account/getAccountByUsername.js b/queries/admin/account/getAccountByUsername.jsindex 30480a71b..ff64c8cef 100644--- a/queries/admin/account/getAccountByUsername.js+++ b/queries/admin/account/getAccountByUsername.js@@ -1,11 +1,9 @@-import { prisma, runQuery } from 'lib/relational';+import prisma from 'lib/prisma';export async function getAccountByUsername(username) {- return runQuery(- prisma.account.findUnique({- where: {- username,- },- }),- );+ return prisma.client.account.findUnique({+ where: {+ username,+ },+ });}diff --git a/queries/admin/account/getAccounts.js b/queries/admin/account/getAccounts.jsindex 94f70c350..154626d10 100644--- a/queries/admin/account/getAccounts.js+++ b/queries/admin/account/getAccounts.js@@ -1,21 +1,19 @@-import { prisma, runQuery } from 'lib/relational';+import prisma from 'lib/prisma';export async function getAccounts() {- return runQuery(- prisma.account.findMany({- orderBy: [- { is_admin: 'desc' },- {- username: 'asc',- },- ],- select: {- user_id: true,- username: true,- is_admin: true,- created_at: true,- updated_at: true,+ return prisma.client.account.findMany({+ orderBy: [+ { is_admin: 'desc' },+ {+ username: 'asc',},- }),- );+ ],+ select: {+ user_id: true,+ username: true,+ is_admin: true,+ created_at: true,+ updated_at: true,+ },+ });}diff --git a/queries/admin/account/updateAccount.js b/queries/admin/account/updateAccount.jsindex 7a337f59c..80509decb 100644--- a/queries/admin/account/updateAccount.js+++ b/queries/admin/account/updateAccount.js@@ -1,12 +1,10 @@-import { prisma, runQuery } from 'lib/relational';+import prisma from 'lib/prisma';export async function updateAccount(user_id, data) {- return runQuery(- prisma.account.update({- where: {- user_id,- },- data,- }),- );+ return prisma.client.account.update({+ where: {+ user_id,+ },+ data,+ });}diff --git a/queries/admin/website/createWebsite.js b/queries/admin/website/createWebsite.jsindex 7a589db6f..d5eaaf312 100644--- a/queries/admin/website/createWebsite.js+++ b/queries/admin/website/createWebsite.js@@ -1,9 +1,9 @@-import { prisma, runQuery } from 'lib/relational';+import prisma from 'lib/prisma';import redis from 'lib/redis';export async function createWebsite(user_id, data) {- return runQuery(- prisma.website.create({+ return prisma.client.website+ .create({data: {account: {connect: {@@ -12,12 +12,12 @@ export async function createWebsite(user_id, data) {},...data,},- }),- ).then(async res => {- if (process.env.REDIS_URL) {- await redis.set(`website:${res.website_uuid}`, Number(res.website_id));- }+ })+ .then(async res => {+ if (process.env.REDIS_URL) {+ await redis.set(`website:${res.website_uuid}`, Number(res.website_id));+ }- return res;- });+ return res;+ });}diff --git a/queries/admin/website/deleteWebsite.js b/queries/admin/website/deleteWebsite.jsindex 13c3160df..15030444f 100644--- a/queries/admin/website/deleteWebsite.js+++ b/queries/admin/website/deleteWebsite.js@@ -1,30 +1,28 @@-import { prisma, runQuery } from 'lib/relational';+import prisma from 'lib/prisma';import redis from 'lib/redis';export async function deleteWebsite(website_id) {- return runQuery(- prisma- .$transaction([- prisma.pageview.deleteMany({- where: { session: { website: { website_id } } },- }),- prisma.event_data.deleteMany({- where: { event: { session: { website: { website_id } } } },- }),- prisma.event.deleteMany({- where: { session: { website: { website_id } } },- }),- prisma.session.deleteMany({- where: { website: { website_id } },- }),- prisma.website.delete({- where: { website_id },- }),- ])- .then(async res => {- if (process.env.REDIS_URL) {- await redis.del(`website:${res.website_uuid}`);- }- }),- );+ const { client, multiQuery } = prisma;++ return multiQuery([+ client.pageview.deleteMany({+ where: { session: { website: { website_id } } },+ }),+ client.event_data.deleteMany({+ where: { event: { session: { website: { website_id } } } },+ }),+ client.event.deleteMany({+ where: { session: { website: { website_id } } },+ }),+ client.session.deleteMany({+ where: { website: { website_id } },+ }),+ client.website.delete({+ where: { website_id },+ }),+ ]).then(async res => {+ if (process.env.REDIS_URL) {+ await redis.del(`website:${res.website_uuid}`);+ }+ });}diff --git a/queries/admin/website/getAllWebsites.js b/queries/admin/website/getAllWebsites.jsindex f8d2a3e9c..23afd8137 100644--- a/queries/admin/website/getAllWebsites.js+++ b/queries/admin/website/getAllWebsites.js@@ -1,24 +1,23 @@-import { prisma, runQuery } from 'lib/relational';+import prisma from 'lib/prisma';export async function getAllWebsites() {- let data = await runQuery(- prisma.website.findMany({- orderBy: [- {- user_id: 'asc',- },- {- name: 'asc',- },- ],- include: {- account: {- select: {- username: true,- },+ let data = await prisma.client.website.findMany({+ orderBy: [+ {+ user_id: 'asc',+ },+ {+ name: 'asc',+ },+ ],+ include: {+ account: {+ select: {+ username: true,},},- }),- );+ },+ });+return data.map(i => ({ ...i, account: i.account.username }));}diff --git a/queries/admin/website/getUserWebsites.js b/queries/admin/website/getUserWebsites.jsindex 729ce893d..5a9662c06 100644--- a/queries/admin/website/getUserWebsites.js+++ b/queries/admin/website/getUserWebsites.js@@ -1,14 +1,12 @@-import { prisma, runQuery } from 'lib/relational';+import prisma from 'lib/prisma';export async function getUserWebsites(user_id) {- return runQuery(- prisma.website.findMany({- where: {- user_id,- },- orderBy: {- name: 'asc',- },- }),- );+ return prisma.client.website.findMany({+ where: {+ user_id,+ },+ orderBy: {+ name: 'asc',+ },+ });}diff --git a/queries/admin/website/getWebsiteById.js b/queries/admin/website/getWebsiteById.jsindex f4495b934..9b8cfe931 100644--- a/queries/admin/website/getWebsiteById.js+++ b/queries/admin/website/getWebsiteById.js@@ -1,11 +1,9 @@-import { prisma, runQuery } from 'lib/relational';+import prisma from 'lib/prisma';export async function getWebsiteById(website_id) {- return runQuery(- prisma.website.findUnique({- where: {- website_id,- },- }),- );+ return prisma.client.website.findUnique({+ where: {+ website_id,+ },+ });}diff --git a/queries/admin/website/getWebsiteByShareId.js b/queries/admin/website/getWebsiteByShareId.jsindex 5630f89e2..fbdeebc98 100644--- a/queries/admin/website/getWebsiteByShareId.js+++ b/queries/admin/website/getWebsiteByShareId.js@@ -1,11 +1,9 @@-import { prisma, runQuery } from 'lib/relational';+import prisma from 'lib/prisma';export async function getWebsiteByShareId(share_id) {- return runQuery(- prisma.website.findUnique({- where: {- share_id,- },- }),- );+ return prisma.client.website.findUnique({+ where: {+ share_id,+ },+ });}diff --git a/queries/admin/website/getWebsiteByUuid.js b/queries/admin/website/getWebsiteByUuid.jsindex dd0f24eea..1b9aeb4cc 100644--- a/queries/admin/website/getWebsiteByUuid.js+++ b/queries/admin/website/getWebsiteByUuid.js@@ -1,11 +1,9 @@-import { prisma, runQuery } from 'lib/relational';+import prisma from 'lib/prisma';export async function getWebsiteByUuid(website_uuid) {- return runQuery(- prisma.website.findUnique({- where: {- website_uuid,- },- }),- );+ return prisma.client.website.findUnique({+ where: {+ website_uuid,+ },+ });}diff --git a/queries/admin/website/resetWebsite.js b/queries/admin/website/resetWebsite.jsindex 92bb959c9..661cd65cd 100644--- a/queries/admin/website/resetWebsite.js+++ b/queries/admin/website/resetWebsite.js@@ -1,5 +1,25 @@-import { prisma, runQuery } from 'lib/relational';+import prisma from 'lib/prisma';+import redis from 'lib/redis';export async function resetWebsite(website_id) {- return runQuery(prisma.$queryRaw`delete from session where website_id=${website_id}`);+ const { client, multiQuery } = prisma;++ return multiQuery([+ client.pageview.deleteMany({+ where: { session: { website: { website_id } } },+ }),+ client.event_data.deleteMany({+ where: { event: { session: { website: { website_id } } } },+ }),+ client.event.deleteMany({+ where: { session: { website: { website_id } } },+ }),+ client.session.deleteMany({+ where: { website: { website_id } },+ }),+ ]).then(async res => {+ if (process.env.REDIS_URL) {+ await redis.del(`website:${res.website_uuid}`);+ }+ });}diff --git a/queries/admin/website/updateWebsite.js b/queries/admin/website/updateWebsite.jsindex 9de1a803a..54f01f66d 100644--- a/queries/admin/website/updateWebsite.js+++ b/queries/admin/website/updateWebsite.js@@ -1,12 +1,10 @@-import { prisma, runQuery } from 'lib/relational';+import prisma from 'lib/prisma';export async function updateWebsite(website_id, data) {- return runQuery(- prisma.website.update({- where: {- website_id,- },- data,- }),- );+ return prisma.client.website.update({+ where: {+ website_id,+ },+ data,+ });}diff --git a/queries/analytics/event/getEventMetrics.js b/queries/analytics/event/getEventMetrics.jsindex 531701e8b..84c007778 100644--- a/queries/analytics/event/getEventMetrics.js+++ b/queries/analytics/event/getEventMetrics.js@@ -1,11 +1,10 @@-import { CLICKHOUSE, RELATIONAL } from 'lib/constants';+import prisma from 'lib/prisma';import clickhouse from 'lib/clickhouse';-import { getDateQuery, getFilterQuery, rawQuery } from 'lib/relational';-import { runAnalyticsQuery } from 'lib/db';+import { runQuery, CLICKHOUSE, PRISMA } from 'lib/db';export async function getEventMetrics(...args) {- return runAnalyticsQuery({- [RELATIONAL]: () => relationalQuery(...args),+ return runQuery({+ [PRISMA]: () => relationalQuery(...args),[CLICKHOUSE]: () => clickhouseQuery(...args),});}@@ -18,11 +17,11 @@ async function relationalQuery(unit = 'day',filters = {},) {+ const { rawQuery, getDateQuery, getFilterQuery } = prisma;const params = [website_id, start_at, end_at];return rawQuery(- `- select+ `selectevent_name x,${getDateQuery('created_at', unit, timezone)} t,count(*) y@@ -31,8 +30,7 @@ async function relationalQuery(and created_at between $2 and $3${getFilterQuery('event', filters, params)}group by 1, 2- order by 2- `,+ order by 2`,params,);}@@ -45,21 +43,20 @@ async function clickhouseQuery(unit = 'day',filters = {},) {+ const { rawQuery, getDateQuery, getBetweenDates } = prisma;const params = [website_id];- return clickhouse.rawQuery(- `- select+ return rawQuery(+ `selectevent_name x,- ${clickhouse.getDateQuery('created_at', unit, timezone)} t,+ ${getDateQuery('created_at', unit, timezone)} t,count(*) yfrom eventwhere website_id= $1- and ${clickhouse.getBetweenDates('created_at', start_at, end_at)}+ and ${getBetweenDates('created_at', start_at, end_at)}${clickhouse.getFilterQuery('event', filters, params)}group by x, t- order by t- `,+ order by t`,params,);}diff --git a/queries/analytics/event/getEvents.js b/queries/analytics/event/getEvents.jsindex 6bfb6fb8b..317a8f2e7 100644--- a/queries/analytics/event/getEvents.js+++ b/queries/analytics/event/getEvents.js@@ -1,36 +1,34 @@-import { CLICKHOUSE, RELATIONAL } from 'lib/constants';-import { prisma, runQuery } from 'lib/relational';+import prisma from 'lib/prisma';import clickhouse from 'lib/clickhouse';-import { runAnalyticsQuery } from 'lib/db';+import { runQuery, CLICKHOUSE, PRISMA } from 'lib/db';export function getEvents(...args) {- return runAnalyticsQuery({- [RELATIONAL]: () => relationalQuery(...args),+ return runQuery({+ [PRISMA]: () => relationalQuery(...args),[CLICKHOUSE]: () => clickhouseQuery(...args),});}function relationalQuery(websites, start_at) {- return runQuery(- prisma.event.findMany({- where: {- website: {- website_id: {- in: websites,- },- },- created_at: {- gte: start_at,+ return prisma.client.event.findMany({+ where: {+ website: {+ website_id: {+ in: websites,},},- }),- );+ created_at: {+ gte: start_at,+ },+ },+ });}function clickhouseQuery(websites, start_at) {- return clickhouse.rawQuery(- `- select+ const { rawQuery, getDateFormat } = clickhouse;++ return rawQuery(+ `selectevent_id,website_id,session_id,@@ -39,7 +37,6 @@ function clickhouseQuery(websites, start_at) {event_namefrom eventwhere website_id in (${websites.join[',']}- and created_at >= ${clickhouse.getDateFormat(start_at)})- `,+ and created_at >= ${getDateFormat(start_at)})`,);}diff --git a/queries/analytics/event/saveEvent.js b/queries/analytics/event/saveEvent.jsindex b54144edb..10e894aee 100644--- a/queries/analytics/event/saveEvent.js+++ b/queries/analytics/event/saveEvent.js@@ -1,12 +1,12 @@-import { CLICKHOUSE, KAFKA, RELATIONAL, URL_LENGTH } from 'lib/constants';+import prisma from 'lib/prisma';import clickhouse from 'lib/clickhouse';import kafka from 'lib/kafka';-import { prisma, runQuery } from 'lib/relational';-import { runAnalyticsQuery } from 'lib/db';+import { runQuery, CLICKHOUSE, KAFKA, PRISMA } from 'lib/db';+import { URL_LENGTH, EVENT_NAME_LENGTH } from 'lib/constants';export async function saveEvent(...args) {- return runAnalyticsQuery({- [RELATIONAL]: () => relationalQuery(...args),+ return runQuery({+ [PRISMA]: () => relationalQuery(...args),[CLICKHOUSE]: () => clickhouseQuery(...args),[KAFKA]: () => kafkaQuery(...args),});@@ -16,8 +16,8 @@ async function relationalQuery(website_id, { session_id, url, event_name, event_const data = {website_id,session_id,- url: url?.substr(0, URL_LENGTH),- event_name: event_name?.substr(0, 50),+ url: url?.substring(0, URL_LENGTH),+ event_name: event_name?.substring(0, EVENT_NAME_LENGTH),};if (event_data) {@@ -28,39 +28,38 @@ async function relationalQuery(website_id, { session_id, url, event_name, event_};}- return runQuery(- prisma.event.create({- data,- }),- );+ return prisma.client.event.create({+ data,+ });}async function clickhouseQuery(website_id, { event_uuid, session_uuid, url, event_name }) {+ const { rawQuery, getDateFormat } = clickhouse;const params = [website_id,event_uuid,session_uuid,- url?.substr(0, URL_LENGTH),- event_name?.substr(0, 50),+ url?.substring(0, URL_LENGTH),+ event_name?.substring(0, EVENT_NAME_LENGTH),];- return clickhouse.rawQuery(- `- insert into umami.event (created_at, website_id, session_uuid, url, event_name)- values (${clickhouse.getDateFormat(new Date())}, $1, $2, $3, $4);`,+ return rawQuery(+ `insert into umami.event (created_at, website_id, session_uuid, url, event_name)+ values (${getDateFormat(new Date())}, $1, $2, $3, $4);`,params,);}async function kafkaQuery(website_id, { event_uuid, session_uuid, url, event_name }) {+ const { getDateFormat, sendMessage } = kafka;const params = {event_uuid: event_uuid,website_id: website_id,session_uuid: session_uuid,- created_at: kafka.getDateFormat(new Date()),- url: url?.substr(0, URL_LENGTH),- event_name: event_name?.substr(0, 50),+ created_at: getDateFormat(new Date()),+ url: url?.substring(0, URL_LENGTH),+ event_name: event_name?.substring(0, EVENT_NAME_LENGTH),};- await kafka.sendMessage(params, 'event');+ await sendMessage(params, 'event');}diff --git a/queries/analytics/pageview/getPageviewMetrics.js b/queries/analytics/pageview/getPageviewMetrics.jsindex 40af97d0e..d34f4c17e 100644--- a/queries/analytics/pageview/getPageviewMetrics.js+++ b/queries/analytics/pageview/getPageviewMetrics.js@@ -1,16 +1,16 @@-import { CLICKHOUSE, RELATIONAL } from 'lib/constants';+import prisma from 'lib/prisma';import clickhouse from 'lib/clickhouse';-import { parseFilters, rawQuery } from 'lib/relational';-import { runAnalyticsQuery } from 'lib/db';+import { runQuery, CLICKHOUSE, PRISMA } from 'lib/db';export async function getPageviewMetrics(...args) {- return runAnalyticsQuery({- [RELATIONAL]: () => relationalQuery(...args),+ return runQuery({+ [PRISMA]: () => relationalQuery(...args),[CLICKHOUSE]: () => clickhouseQuery(...args),});}async function relationalQuery(website_id, start_at, end_at, column, table, filters = {}) {+ const { rawQuery, parseFilters } = prisma;const params = [website_id, start_at, end_at];const { pageviewQuery, sessionQuery, eventQuery, joinSession } = parseFilters(table,@@ -20,8 +20,7 @@ async function relationalQuery(website_id, start_at, end_at, column, table, filt);return rawQuery(- `- select ${column} x, count(*) y+ `select ${column} x, count(*) yfrom ${table}${joinSession}where ${table}.website_id=$1@@ -30,15 +29,15 @@ async function relationalQuery(website_id, start_at, end_at, column, table, filt${joinSession && sessionQuery}${eventQuery}group by 1- order by 2 desc- `,+ order by 2 desc`,params,);}async function clickhouseQuery(website_id, start_at, end_at, column, table, filters = {}) {+ const { rawQuery, parseFilters, getBetweenDates } = clickhouse;const params = [website_id];- const { pageviewQuery, sessionQuery, eventQuery, joinSession } = clickhouse.parseFilters(+ const { pageviewQuery, sessionQuery, eventQuery, joinSession } = parseFilters(table,column,filters,@@ -46,19 +45,17 @@ async function clickhouseQuery(website_id, start_at, end_at, column, table, filt'session_uuid',);- return clickhouse.rawQuery(- `- select ${column} x, count(*) y+ return rawQuery(+ `select ${column} x, count(*) yfrom ${table}${joinSession}where ${table}.website_id= $1- and ${clickhouse.getBetweenDates(table + '.created_at', start_at, end_at)}+ and ${getBetweenDates(table + '.created_at', start_at, end_at)}${pageviewQuery}${joinSession && sessionQuery}${eventQuery}group by x- order by y desc- `,+ order by y desc`,params,);}diff --git a/queries/analytics/pageview/getPageviewParams.js b/queries/analytics/pageview/getPageviewParams.jsindex be6e5148e..fa445da58 100644--- a/queries/analytics/pageview/getPageviewParams.js+++ b/queries/analytics/pageview/getPageviewParams.js@@ -1,14 +1,15 @@-import { parseFilters, rawQuery, runAnalyticsQuery } from 'lib/relational';-import { CLICKHOUSE, RELATIONAL } from 'lib/constants';+import prisma from 'lib/prisma';+import { runQuery, CLICKHOUSE, PRISMA } from 'lib/db';export async function getPageviewParams(...args) {- return runAnalyticsQuery({- [RELATIONAL]: () => relationalQuery(...args),+ return runQuery({+ [PRISMA]: () => relationalQuery(...args),[CLICKHOUSE]: () => clickhouseQuery(...args),});}async function relationalQuery(website_id, start_at, end_at, column, table, filters = {}) {+ const { parseFilters, rawQuery } = prisma;const params = [website_id, start_at, end_at];const { pageviewQuery, sessionQuery, eventQuery, joinSession } = parseFilters(table,@@ -18,8 +19,7 @@ async function relationalQuery(website_id, start_at, end_at, column, table, filt);return rawQuery(- `- select url x,+ `select url x,count(*) yfrom ${table}${joinSession}@@ -30,8 +30,7 @@ async function relationalQuery(website_id, start_at, end_at, column, table, filt${joinSession && sessionQuery}${eventQuery}group by 1- order by 2 desc- `,+ order by 2 desc`,params,);}diff --git a/queries/analytics/pageview/getPageviewStats.js b/queries/analytics/pageview/getPageviewStats.jsindex 207f109b8..da25ab0c6 100644--- a/queries/analytics/pageview/getPageviewStats.js+++ b/queries/analytics/pageview/getPageviewStats.js@@ -1,11 +1,10 @@-import { CLICKHOUSE, RELATIONAL } from 'lib/constants';-import { getDateQuery, parseFilters, rawQuery } from 'lib/relational';-import { runAnalyticsQuery } from 'lib/db';+import prisma from 'lib/prisma';import clickhouse from 'lib/clickhouse';+import { runQuery, CLICKHOUSE, PRISMA } from 'lib/db';export async function getPageviewStats(...args) {- return runAnalyticsQuery({- [RELATIONAL]: () => relationalQuery(...args),+ return runQuery({+ [PRISMA]: () => relationalQuery(...args),[CLICKHOUSE]: () => clickhouseQuery(...args),});}@@ -20,6 +19,7 @@ async function relationalQuery(filters = {},sessionKey = 'session_id',) {+ const { getDateQuery, parseFilters, rawQuery } = prisma;const params = [website_id, start_at, end_at];const { pageviewQuery, sessionQuery, joinSession } = parseFilters('pageview',@@ -29,8 +29,7 @@ async function relationalQuery();return rawQuery(- `- select ${getDateQuery('pageview.created_at', unit, timezone)} t,+ `select ${getDateQuery('pageview.created_at', unit, timezone)} t,count(${count !== '*' ? `${count}${sessionKey}` : count}) yfrom pageview${joinSession}@@ -38,8 +37,7 @@ async function relationalQuery(and pageview.created_at between $2 and $3${pageviewQuery}${sessionQuery}- group by 1- `,+ group by 1`,params,);}@@ -54,6 +52,7 @@ async function clickhouseQuery(filters = {},sessionKey = 'session_uuid',) {+ const { parseFilters, rawQuery, getDateStringQuery, getDateQuery, getBetweenDates } = clickhouse;const params = [website_id];const { pageviewQuery, sessionQuery, joinSession } = parseFilters('pageview',@@ -63,24 +62,22 @@ async function clickhouseQuery(sessionKey,);- return clickhouse.rawQuery(- `- select- ${clickhouse.getDateStringQuery('g.t', unit)} as t,+ return rawQuery(+ `select+ ${getDateStringQuery('g.t', unit)} as t,g.y as yfrom(select- ${clickhouse.getDateQuery('created_at', unit, timezone)} t,+ ${getDateQuery('created_at', unit, timezone)} t,count(${count !== '*' ? `${count}${sessionKey}` : count}) yfrom pageview${joinSession}where pageview.website_id= $1- and ${clickhouse.getBetweenDates('pageview.created_at', start_at, end_at)}+ and ${getBetweenDates('pageview.created_at', start_at, end_at)}${pageviewQuery}${sessionQuery}group by t) g- order by t- `,+ order by t`,params,);}diff --git a/queries/analytics/pageview/getPageviews.js b/queries/analytics/pageview/getPageviews.jsindex fbbd96f85..8ce704d89 100644--- a/queries/analytics/pageview/getPageviews.js+++ b/queries/analytics/pageview/getPageviews.js@@ -1,36 +1,32 @@-import { CLICKHOUSE, RELATIONAL } from 'lib/constants';-import { prisma, runQuery } from 'lib/relational';-import { runAnalyticsQuery } from 'lib/db';+import prisma from 'lib/prisma';import clickhouse from 'lib/clickhouse';+import { runQuery, CLICKHOUSE, PRISMA } from 'lib/db';export async function getPageviews(...args) {- return runAnalyticsQuery({- [RELATIONAL]: () => relationalQuery(...args),+ return runQuery({+ [PRISMA]: () => relationalQuery(...args),[CLICKHOUSE]: () => clickhouseQuery(...args),});}async function relationalQuery(websites, start_at) {- return runQuery(- prisma.pageview.findMany({- where: {- website: {- website_id: {- in: websites,- },- },- created_at: {- gte: start_at,+ return prisma.client.pageview.findMany({+ where: {+ website: {+ website_id: {+ in: websites,},},- }),- );+ created_at: {+ gte: start_at,+ },+ },+ });}async function clickhouseQuery(websites, start_at) {return clickhouse.rawQuery(- `- select+ `selectview_id,website_id,session_id,@@ -38,7 +34,6 @@ async function clickhouseQuery(websites, start_at) {urlfrom pageviewwhere website_id in (${websites.join[',']}- and created_at >= ${clickhouse.getDateFormat(start_at)})- `,+ and created_at >= ${clickhouse.getDateFormat(start_at)})`,);}diff --git a/queries/analytics/pageview/savePageView.js b/queries/analytics/pageview/savePageView.jsindex c2b9527e1..366981273 100644--- a/queries/analytics/pageview/savePageView.js+++ b/queries/analytics/pageview/savePageView.js@@ -1,54 +1,52 @@-import { CLICKHOUSE, KAFKA, RELATIONAL, URL_LENGTH } from 'lib/constants';+import prisma from 'lib/prisma';import clickhouse from 'lib/clickhouse';-import { runAnalyticsQuery } from 'lib/db';import kafka from 'lib/kafka';-import { prisma, runQuery } from 'lib/relational';+import { runQuery, CLICKHOUSE, KAFKA, PRISMA } from 'lib/db';+import { URL_LENGTH } from 'lib/constants';export async function savePageView(...args) {- return runAnalyticsQuery({- [RELATIONAL]: () => relationalQuery(...args),+ return runQuery({+ [PRISMA]: () => relationalQuery(...args),[CLICKHOUSE]: () => clickhouseQuery(...args),[KAFKA]: () => kafkaQuery(...args),});}async function relationalQuery(website_id, { session_id, url, referrer }) {- return runQuery(- prisma.pageview.create({- data: {- website_id,- session_id,- url: url?.substr(0, URL_LENGTH),- referrer: referrer?.substr(0, URL_LENGTH),- },- }),- );+ return prisma.client.pageview.create({+ data: {+ website_id,+ session_id,+ url: url?.substring(0, URL_LENGTH),+ referrer: referrer?.substring(0, URL_LENGTH),+ },+ });}async function clickhouseQuery(website_id, { session_uuid, url, referrer }) {const params = [website_id,session_uuid,- url?.substr(0, URL_LENGTH),- referrer?.substr(0, URL_LENGTH),+ url?.substring(0, URL_LENGTH),+ referrer?.substring(0, URL_LENGTH),];return clickhouse.rawQuery(- `- insert into umami.pageview (created_at, website_id, session_uuid, url, referrer)+ `insert into umami.pageview (created_at, website_id, session_uuid, url, referrer)values (${clickhouse.getDateFormat(new Date())}, $1, $2, $3, $4);`,params,);}async function kafkaQuery(website_id, { session_uuid, url, referrer }) {+ const { getDateFormat, sendMessage } = kafka;const params = {website_id: website_id,session_uuid: session_uuid,- created_at: kafka.getDateFormat(new Date()),- url: url?.substr(0, URL_LENGTH),- referrer: referrer?.substr(0, URL_LENGTH),+ created_at: getDateFormat(new Date()),+ url: url?.substring(0, URL_LENGTH),+ referrer: referrer?.substring(0, URL_LENGTH),};- await kafka.sendMessage(params, 'pageview');+ await sendMessage(params, 'pageview');}diff --git a/queries/analytics/session/createSession.js b/queries/analytics/session/createSession.jsindex a4419b7e3..994f9ef50 100644--- a/queries/analytics/session/createSession.js+++ b/queries/analytics/session/createSession.js@@ -1,21 +1,20 @@+import prisma from 'lib/prisma';import clickhouse from 'lib/clickhouse';-import { CLICKHOUSE, KAFKA, RELATIONAL } from 'lib/constants';-import { runAnalyticsQuery } from 'lib/db';import kafka from 'lib/kafka';import redis from 'lib/redis';-import { prisma, runQuery } from 'lib/relational';+import { runQuery, CLICKHOUSE, KAFKA, PRISMA } from 'lib/db';export async function createSession(...args) {- return runAnalyticsQuery({- [RELATIONAL]: () => relationalQuery(...args),+ return runQuery({+ [PRISMA]: () => relationalQuery(...args),[CLICKHOUSE]: () => clickhouseQuery(...args),[KAFKA]: () => kafkaQuery(...args),});}async function relationalQuery(website_id, data) {- return runQuery(- prisma.session.create({+ return prisma.client.session+ .create({data: {website_id,...data,@@ -23,14 +22,14 @@ async function relationalQuery(website_id, data) {select: {session_id: true,},- }),- ).then(async res => {- if (process.env.REDIS_URL) {- await redis.set(`session:${res.session_uuid}`, '');- }+ })+ .then(async res => {+ if (process.env.REDIS_URL) {+ await redis.set(`session:${res.session_uuid}`, '');+ }- return res;- });+ return res;+ });}async function clickhouseQuery(@@ -48,10 +47,11 @@ async function clickhouseQuery(language,country ? country : null,];+ const { rawQuery, getDateFormat } = clickhouse;- await clickhouse.rawQuery(+ await rawQuery(`insert into umami.session (created_at, session_uuid, website_id, hostname, browser, os, device, screen, language, country)- values (${clickhouse.getDateFormat(new Date())}, $1, $2, $3, $4, $5, $6, $7, $8, $9);`,+ values (${getDateFormat(new Date())}, $1, $2, $3, $4, $5, $6, $7, $8, $9);`,params,);}@@ -60,10 +60,11 @@ async function kafkaQuery(website_id,{ session_uuid, hostname, browser, os, screen, language, country, device },) {+ const { getDateFormat, sendMessage } = kafka;const params = {session_uuid: session_uuid,website_id: website_id,- created_at: kafka.getDateFormat(new Date()),+ created_at: getDateFormat(new Date()),hostname: hostname,browser: browser,os: os,@@ -73,7 +74,7 @@ async function kafkaQuery(country: country ? country : null,};- await kafka.sendMessage(params, 'session');+ await sendMessage(params, 'session');await redis.set(`session:${session_uuid}`, '');}diff --git a/queries/analytics/session/getSessionByUuid.js b/queries/analytics/session/getSessionByUuid.jsindex 953029921..6e92a2ca6 100644--- a/queries/analytics/session/getSessionByUuid.js+++ b/queries/analytics/session/getSessionByUuid.js@@ -1,32 +1,28 @@-import { CLICKHOUSE, RELATIONAL } from 'lib/constants';-import { prisma, runQuery } from 'lib/relational';+import prisma from 'lib/prisma';import clickhouse from 'lib/clickhouse';-import { runAnalyticsQuery } from 'lib/db';+import { runQuery, CLICKHOUSE, PRISMA } from 'lib/db';export async function getSessionByUuid(...args) {- return runAnalyticsQuery({- [RELATIONAL]: () => relationalQuery(...args),+ return runQuery({+ [PRISMA]: () => relationalQuery(...args),[CLICKHOUSE]: () => clickhouseQuery(...args),});}async function relationalQuery(session_uuid) {- return runQuery(- prisma.session.findUnique({- where: {- session_uuid,- },- }),- );+ return prisma.client.session.findUnique({+ where: {+ session_uuid,+ },+ });}async function clickhouseQuery(session_uuid) {+ const { rawQuery, findFirst } = clickhouse;const params = [session_uuid];- return clickhouse- .rawQuery(- `- select+ return rawQuery(+ `selectsession_uuid,website_id,created_at,@@ -35,12 +31,10 @@ async function clickhouseQuery(session_uuid) {os,device,screen,- "language",+ language,countryfrom session- where session_uuid = $1- `,- params,- )- .then(result => clickhouse.findFirst(result));+ where session_uuid = $1`,+ params,+ ).then(result => findFirst(result));}diff --git a/queries/analytics/session/getSessionMetrics.js b/queries/analytics/session/getSessionMetrics.jsindex e0d5a24ca..36eb75680 100644--- a/queries/analytics/session/getSessionMetrics.js+++ b/queries/analytics/session/getSessionMetrics.js@@ -1,16 +1,16 @@-import { CLICKHOUSE, RELATIONAL } from 'lib/constants';+import prisma from 'lib/prisma';import clickhouse from 'lib/clickhouse';-import { runAnalyticsQuery } from 'lib/db';-import { parseFilters, rawQuery } from 'lib/relational';+import { runQuery, CLICKHOUSE, PRISMA } from 'lib/db';export async function getSessionMetrics(...args) {- return runAnalyticsQuery({- [RELATIONAL]: () => relationalQuery(...args),+ return runQuery({+ [PRISMA]: () => relationalQuery(...args),[CLICKHOUSE]: () => clickhouseQuery(...args),});}async function relationalQuery(website_id, start_at, end_at, field, filters = {}) {+ const { parseFilters, rawQuery } = prisma;const params = [website_id, start_at, end_at];const { pageviewQuery, sessionQuery, joinSession } = parseFilters('pageview',@@ -20,8 +20,7 @@ async function relationalQuery(website_id, start_at, end_at, field, filters = {});return rawQuery(- `- select ${field} x, count(*) y+ `select ${field} x, count(*) yfrom session as xwhere x.session_id in (select pageview.session_id@@ -33,15 +32,15 @@ async function relationalQuery(website_id, start_at, end_at, field, filters = {}${sessionQuery})group by 1- order by 2 desc- `,+ order by 2 desc`,params,);}async function clickhouseQuery(website_id, start_at, end_at, field, filters = {}) {+ const { parseFilters, getBetweenDates, rawQuery } = clickhouse;const params = [website_id];- const { pageviewQuery, sessionQuery, joinSession } = clickhouse.parseFilters(+ const { pageviewQuery, sessionQuery, joinSession } = parseFilters('pageview',null,filters,@@ -49,22 +48,20 @@ async function clickhouseQuery(website_id, start_at, end_at, field, filters = {}'session_uuid',);- return clickhouse.rawQuery(- `- select ${field} x, count(*) y+ return rawQuery(+ `select ${field} x, count(*) yfrom session as xwhere x.session_uuid in (select pageview.session_uuidfrom pageview${joinSession}where pageview.website_id=$1- and ${clickhouse.getBetweenDates('pageview.created_at', start_at, end_at)}+ and ${getBetweenDates('pageview.created_at', start_at, end_at)}${pageviewQuery}${sessionQuery})group by x- order by y desc- `,+ order by y desc`,params,);}diff --git a/queries/analytics/session/getSessions.js b/queries/analytics/session/getSessions.jsindex bcb0f19e4..bf070651a 100644--- a/queries/analytics/session/getSessions.js+++ b/queries/analytics/session/getSessions.js@@ -1,18 +1,17 @@-import { CLICKHOUSE, RELATIONAL } from 'lib/constants';+import prisma from 'lib/prisma';import clickhouse from 'lib/clickhouse';-import { runAnalyticsQuery } from 'lib/db';-import { prisma, runQuery } from 'lib/relational';+import { runQuery, PRISMA, CLICKHOUSE } from 'lib/db';export async function getSessions(...args) {- return runAnalyticsQuery({- [RELATIONAL]: () => relationalQuery(...args),+ return runQuery({+ [PRISMA]: () => relationalQuery(...args),[CLICKHOUSE]: () => clickhouseQuery(...args),});}async function relationalQuery(websites, start_at) {return runQuery(- prisma.session.findMany({+ prisma.client.session.findMany({where: {...(websites && websites.length > 0? {@@ -32,9 +31,10 @@ async function relationalQuery(websites, start_at) {}async function clickhouseQuery(websites, start_at) {- return clickhouse.rawQuery(- `- select+ const { rawQuery, getDateFormat } = clickhouse;++ return rawQuery(+ `selectsession_uuid,website_id,created_at,@@ -43,11 +43,10 @@ async function clickhouseQuery(websites, start_at) {os,device,screen,- "language",+ language,countryfrom sessionwhere ${websites && websites.length > 0 ? `(website_id in (${websites.join[',']})` : '0 = 0'}- and created_at >= ${clickhouse.getDateFormat(start_at)}- `,+ and created_at >= ${getDateFormat(start_at)}`,);}diff --git a/queries/analytics/stats/getActiveVisitors.js b/queries/analytics/stats/getActiveVisitors.jsindex 4e61d3ebd..023c00a4e 100644--- a/queries/analytics/stats/getActiveVisitors.js+++ b/queries/analytics/stats/getActiveVisitors.js@@ -1,12 +1,11 @@import { subMinutes } from 'date-fns';-import { CLICKHOUSE, RELATIONAL } from 'lib/constants';-import { rawQuery } from 'lib/relational';-import { runAnalyticsQuery } from 'lib/db';+import prisma from 'lib/prisma';import clickhouse from 'lib/clickhouse';+import { runQuery, CLICKHOUSE, PRISMA } from 'lib/db';export async function getActiveVisitors(...args) {- return runAnalyticsQuery({- [RELATIONAL]: () => relationalQuery(...args),+ return runQuery({+ [PRISMA]: () => relationalQuery(...args),[CLICKHOUSE]: () => clickhouseQuery(...args),});}@@ -15,27 +14,24 @@ async function relationalQuery(website_id) {const date = subMinutes(new Date(), 5);const params = [website_id, date];- return rawQuery(- `- select count(distinct session_id) x+ return prisma.rawQuery(+ `select count(distinct session_id) xfrom pageviewwhere website_id = $1- and created_at >= $2- `,+ and created_at >= $2`,params,);}async function clickhouseQuery(website_id) {+ const { rawQuery, getDateFormat } = clickhouse;const params = [website_id];- return clickhouse.rawQuery(- `- select count(distinct session_uuid) x+ return rawQuery(+ `select count(distinct session_uuid) xfrom pageviewwhere website_id = $1- and created_at >= ${clickhouse.getDateFormat(subMinutes(new Date(), 5))}- `,+ and created_at >= ${getDateFormat(subMinutes(new Date(), 5))}`,params,);}diff --git a/queries/analytics/stats/getWebsiteStats.js b/queries/analytics/stats/getWebsiteStats.jsindex 081a16acc..5d280563b 100644--- a/queries/analytics/stats/getWebsiteStats.js+++ b/queries/analytics/stats/getWebsiteStats.js@@ -1,16 +1,16 @@-import { CLICKHOUSE, RELATIONAL } from 'lib/constants';-import { getDateQuery, getTimestampInterval, parseFilters, rawQuery } from 'lib/relational';-import { runAnalyticsQuery } from 'lib/db';+import prisma from 'lib/prisma';import clickhouse from 'lib/clickhouse';+import { runQuery, CLICKHOUSE, PRISMA } from 'lib/db';export async function getWebsiteStats(...args) {- return runAnalyticsQuery({- [RELATIONAL]: () => relationalQuery(...args),+ return runQuery({+ [PRISMA]: () => relationalQuery(...args),[CLICKHOUSE]: () => clickhouseQuery(...args),});}async function relationalQuery(website_id, start_at, end_at, filters = {}) {+ const { getDateQuery, getTimestampInterval, parseFilters, rawQuery } = prisma;const params = [website_id, start_at, end_at];const { pageviewQuery, sessionQuery, joinSession } = parseFilters('pageview',@@ -20,32 +20,31 @@ async function relationalQuery(website_id, start_at, end_at, filters = {}) {);return rawQuery(- `- select sum(t.c) as "pageviews",+ `select sum(t.c) as "pageviews",count(distinct t.session_id) as "uniques",sum(case when t.c = 1 then 1 else 0 end) as "bounces",sum(t.time) as "totaltime"from (- select pageview.session_id,- ${getDateQuery('pageview.created_at', 'hour')},- count(*) c,- ${getTimestampInterval('pageview.created_at')} as "time"- from pageview- ${joinSession}- where pageview.website_id=$1- and pageview.created_at between $2 and $3- ${pageviewQuery}- ${sessionQuery}- group by 1, 2- ) t- `,+ select pageview.session_id,+ ${getDateQuery('pageview.created_at', 'hour')},+ count(*) c,+ ${getTimestampInterval('pageview.created_at')} as "time"+ from pageview+ ${joinSession}+ where pageview.website_id=$1+ and pageview.created_at between $2 and $3+ ${pageviewQuery}+ ${sessionQuery}+ group by 1, 2+ ) t`,params,);}async function clickhouseQuery(website_id, start_at, end_at, filters = {}) {+ const { rawQuery, getDateQuery, getBetweenDates, parseFilters } = clickhouse;const params = [website_id];- const { pageviewQuery, sessionQuery, joinSession } = clickhouse.parseFilters(+ const { pageviewQuery, sessionQuery, joinSession } = parseFilters('pageview',null,filters,@@ -53,28 +52,26 @@ async function clickhouseQuery(website_id, start_at, end_at, filters = {}) {'session_uuid',);- return clickhouse.rawQuery(- `- select- sum(t.c) as "pageviews",- count(distinct t.session_uuid) as "uniques",- sum(if(t.c = 1, 1, 0)) as "bounces",- sum(if(max_time < min_time + interval 1 hour, max_time-min_time, 0)) as "totaltime"- from (- select pageview.session_uuid,- ${clickhouse.getDateQuery('pageview.created_at', 'day')} time_series,- count(*) c,- min(created_at) min_time,- max(created_at) max_time- from pageview- ${joinSession}- where pageview.website_id = $1- and ${clickhouse.getBetweenDates('pageview.created_at', start_at, end_at)}- ${pageviewQuery}- ${sessionQuery}- group by pageview.session_uuid, time_series- ) t;- `,+ return rawQuery(+ `select+ sum(t.c) as "pageviews",+ count(distinct t.session_uuid) as "uniques",+ sum(if(t.c = 1, 1, 0)) as "bounces",+ sum(if(max_time < min_time + interval 1 hour, max_time-min_time, 0)) as "totaltime"+ from (+ select pageview.session_uuid,+ ${getDateQuery('pageview.created_at', 'day')} time_series,+ count(*) c,+ min(created_at) min_time,+ max(created_at) max_time+ from pageview+ ${joinSession}+ where pageview.website_id = $1+ and ${getBetweenDates('pageview.created_at', start_at, end_at)}+ ${pageviewQuery}+ ${sessionQuery}+ group by pageview.session_uuid, time_series+ ) t;`,params,);}diff --git a/scripts/copy-db-files.js b/scripts/copy-db-files.jsindex a7897fca3..3e902d45f 100644--- a/scripts/copy-db-files.js+++ b/scripts/copy-db-files.js@@ -3,10 +3,8 @@ const fse = require('fs-extra');const path = require('path');const del = require('del');-function getDatabaseType() {- const type =- process.env.DATABASE_TYPE ||- (process.env.DATABASE_URL && process.env.DATABASE_URL.split(':')[0]);+function getDatabaseType(url = process.env.DATABASE_URL) {+ const type = process.env.DATABASE_TYPE || (url && url.split(':')[0]);if (type === 'postgres') {return 'postgresql';