Skip to content

armgit / umami

Public Imported from GitHub
0

Umami is a privacy-first web and product analytics platform. Traffic, campaigns, behavior, conversions, and revenue in one place. No cookies, no surveillance, self-hosted or in the cloud.

Database refactoring.

Mike Cao467c7f289f29
diff --git a/lib/clickhouse.js b/lib/clickhouse.js
index 7ab06b9e2..1548b77ba 100644
--- a/lib/clickhouse.js
+++ b/lib/clickhouse.js
@@ -1,39 +1,50 @@
import { ClickHouse } from 'clickhouse';
import dateFormat from 'dateformat';
+import debug from 'debug';
import { FILTER_IGNORED } from 'lib/constants';
-import { CLICKHOUSE_DATE_FORMATS } from './constants';
+import { CLICKHOUSE } from 'lib/db';
+
+export const CLICKHOUSE_DATE_FORMATS = {
+ minute: '%Y-%m-%d %H:%M:00',
+ hour: '%Y-%m-%d %H:00:00',
+ day: '%Y-%m-%d',
+ month: '%Y-%m-01',
+ year: '%Y-01-01',
+};
+
+const log = debug('clickhouse');
function getClient() {
- if (!process.env.ANALYTICS_URL) {
+ if (!process.env.CLICKHOUSE_URL) {
return null;
}
- const url = new URL(process.env.ANALYTICS_URL);
- const database = url.pathname.replace('/', '');
-
- return new ClickHouse({
- url: url.hostname,
- port: Number(url.port),
- basicAuth: url.password
- ? {
- username: url.username || 'default',
- password: url.password,
- }
- : null,
+ const {
+ hostname,
+ port,
+ pathname,
+ username = 'default',
+ password,
+ } = new URL(process.env.CLICKHOUSE_URL);
+
+ const client = new ClickHouse({
+ url: hostname,
+ port: Number(port),
format: 'json',
config: {
- database,
+ database: pathname.replace('/', ''),
},
+ basicAuth: password ? { username, password } : null,
});
-}
-const clickhouse = global.clickhouse || getClient();
+ if (process.env.NODE_ENV !== 'production') {
+ global[CLICKHOUSE] = clickhouse;
+ }
-if (process.env.NODE_ENV !== 'production') {
- global.clickhouse = clickhouse;
-}
+ log('Clickhouse initialized');
-export { clickhouse };
+ return client;
+}
function getDateStringQuery(data, unit) {
return `formatDateTime(${data}, '${CLICKHOUSE_DATE_FORMATS[unit]}')`;
@@ -176,7 +187,12 @@ async function findFirst(data) {
return data[0] ?? null;
}
+// Initialization
+const clickhouse = global[CLICKHOUSE] || getClient();
+
export default {
+ client: clickhouse,
+ log,
getDateStringQuery,
getDateQuery,
getDateFormat,
diff --git a/lib/constants.js b/lib/constants.js
index e20a74e67..feef540df 100644
--- a/lib/constants.js
+++ b/lib/constants.js
@@ -67,36 +67,6 @@ export const EVENT_COLORS = [
'#ffec16',
];
-export const RELATIONAL = 'relational';
-export const POSTGRESQL = 'postgresql';
-export const MYSQL = 'mysql';
-export const CLICKHOUSE = 'clickhouse';
-export const KAFKA = 'kafka';
-
-export const MYSQL_DATE_FORMATS = {
- minute: '%Y-%m-%d %H:%i:00',
- hour: '%Y-%m-%d %H:00:00',
- day: '%Y-%m-%d',
- month: '%Y-%m-01',
- year: '%Y-01-01',
-};
-
-export const POSTGRESQL_DATE_FORMATS = {
- minute: 'YYYY-MM-DD HH24:MI:00',
- hour: 'YYYY-MM-DD HH24:00:00',
- day: 'YYYY-MM-DD',
- month: 'YYYY-MM-01',
- year: 'YYYY-01-01',
-};
-
-export const CLICKHOUSE_DATE_FORMATS = {
- minute: '%Y-%m-%d %H:%M:00',
- hour: '%Y-%m-%d %H:00:00',
- day: '%Y-%m-%d',
- month: '%Y-%m-01',
- year: '%Y-01-01',
-};
-
export const FILTER_IGNORED = Symbol.for('filter-ignored');
export const DOMAIN_REGEX =
@@ -107,6 +77,7 @@ export const LAPTOP_SCREEN_WIDTH = 1024;
export const MOBILE_SCREEN_WIDTH = 479;
export const URL_LENGTH = 500;
+export const EVENT_NAME_LENGTH = 50;
export const DESKTOP_OS = [
'Windows 3.11',
diff --git a/lib/db.js b/lib/db.js
index 13139cd58..9db00c924 100644
--- a/lib/db.js
+++ b/lib/db.js
@@ -1,5 +1,11 @@
-import { POSTGRESQL, RELATIONAL, MYSQL, KAFKA, CLICKHOUSE } from 'lib/constants';
-
+export const PRISMA = 'prisma';
+export const POSTGRESQL = 'postgresql';
+export const MYSQL = 'mysql';
+export const CLICKHOUSE = 'clickhouse';
+export const KAFKA = 'kafka';
+export const KAFKA_PRODUCER = 'kafka-producer';
+
+// Fixes issue with converting bigint values
BigInt.prototype.toJSON = function () {
return Number(this);
};
@@ -14,11 +20,11 @@ export function getDatabaseType(url = process.env.DATABASE_URL) {
return type;
}
-export async function runAnalyticsQuery(queries) {
- const db = getDatabaseType(process.env.ANALYTICS_URL || process.env.DATABASE_URL);
+export async function runQuery(queries) {
+ const db = getDatabaseType(process.env.CLICKHOUSE_URL || process.env.DATABASE_URL);
if (db === POSTGRESQL || db === MYSQL) {
- return queries[RELATIONAL]();
+ return queries[PRISMA]();
}
if (db === CLICKHOUSE) {
diff --git a/lib/format.js b/lib/format.js
index a336c1c44..a85c3ef76 100644
--- a/lib/format.js
+++ b/lib/format.js
@@ -74,7 +74,7 @@ export function stringToColor(str) {
let color = '#';
for (let i = 0; i < 3; i++) {
let value = (hash >> (i * 8)) & 0xff;
- color += ('00' + value.toString(16)).substr(-2);
+ color += ('00' + value.toString(16)).substring(-2);
}
return color;
}
diff --git a/lib/kafka.js b/lib/kafka.js
index 393207285..01b69603c 100644
--- a/lib/kafka.js
+++ b/lib/kafka.js
@@ -1,57 +1,53 @@
import { Kafka, logLevel } from 'kafkajs';
import dateFormat from 'dateformat';
+import debug from 'debug';
+import { KAFKA, KAFKA_PRODUCER } from 'lib/db';
-export function getClient() {
- if (!process.env.KAFKA_URL) {
+const log = debug('kafka');
+
+function getClient() {
+ if (!process.env.KAFKA_URL || !process.env.KAFKA_BROKER) {
return null;
}
- const url = new URL(process.env.KAFKA_URL);
+ const { username, password } = new URL(process.env.KAFKA_URL);
const brokers = process.env.KAFKA_BROKER.split(',');
- if (url.username.length === 0 && url.password.length === 0) {
- return new Kafka({
- clientId: 'umami',
- brokers: brokers,
- connectionTimeout: 3000,
- logLevel: logLevel.ERROR,
- });
- } else {
- return new Kafka({
- clientId: 'umami',
- brokers: brokers,
- connectionTimeout: 3000,
- ssl: true,
- sasl: {
- mechanism: 'plain',
- username: url.username,
- password: url.password,
- },
- });
- }
-}
-const kafka = global.kafka || getClient();
-let kafkaProducer = null;
+ const ssl =
+ username && password
+ ? {
+ ssl: true,
+ sasl: {
+ mechanism: 'plain',
+ username,
+ password,
+ },
+ }
+ : {};
-(async () => {
- if (kafka) {
- kafkaProducer = global.kakfaProducer || (await getProducer());
- }
+ const client = new Kafka({
+ clientId: 'umami',
+ brokers: brokers,
+ connectionTimeout: 3000,
+ logLevel: logLevel.ERROR,
+ ...ssl,
+ });
if (process.env.NODE_ENV !== 'production') {
- global.kafka = kafka;
- if (kafka) {
- global.kakfaProducer = kafkaProducer;
- }
+ global[KAFKA] = client;
}
-})();
-export { kafka, kafkaProducer };
+ return client;
+}
async function getProducer() {
const producer = kafka.producer();
await producer.connect();
+ if (process.env.NODE_ENV !== 'production') {
+ global[KAFKA_PRODUCER] = producer;
+ }
+
return producer;
}
@@ -60,7 +56,7 @@ function getDateFormat(date) {
}
async function sendMessage(params, topic) {
- await kafkaProducer.send({
+ await producer.send({
topic,
messages: [
{
@@ -72,7 +68,19 @@ async function sendMessage(params, topic) {
});
}
+// Initialization
+let kafka;
+let producer;
+
+(async () => {
+ kafka = global[KAFKA] || getClient();
+ producer = global[KAFKA_PRODUCER] || (await getProducer());
+})();
+
export default {
+ client: kafka,
+ producer: producer,
+ log,
getDateFormat,
sendMessage,
};
diff --git a/lib/relational.js b/lib/prisma.js
similarity index 78%
rename from lib/relational.js
rename to lib/prisma.js
index d86f06075..af8759ba2 100644
--- a/lib/relational.js
+++ b/lib/prisma.js
@@ -1,19 +1,30 @@
import { PrismaClient } from '@prisma/client';
import chalk from 'chalk';
-import {
- FILTER_IGNORED,
- MYSQL,
- MYSQL_DATE_FORMATS,
- POSTGRESQL,
- POSTGRESQL_DATE_FORMATS,
-} from 'lib/constants';
-import { getDatabaseType } from 'lib/db';
import moment from 'moment-timezone';
import debug from 'debug';
+import { PRISMA, MYSQL, POSTGRESQL } from 'lib/db';
+import { FILTER_IGNORED } from 'lib/constants';
+import { getDatabaseType } from 'lib/db';
+
+const MYSQL_DATE_FORMATS = {
+ minute: '%Y-%m-%d %H:%i:00',
+ hour: '%Y-%m-%d %H:00:00',
+ day: '%Y-%m-%d',
+ month: '%Y-%m-01',
+ year: '%Y-01-01',
+};
+
+const POSTGRESQL_DATE_FORMATS = {
+ minute: 'YYYY-MM-DD HH24:MI:00',
+ hour: 'YYYY-MM-DD HH24:00:00',
+ day: 'YYYY-MM-DD',
+ month: 'YYYY-MM-01',
+ year: 'YYYY-01-01',
+};
const log = debug('prisma');
-const options = {
+const PRISMA_OPTIONS = {
log: [
{
emit: 'event',
@@ -33,16 +44,16 @@ function getClient(options) {
prisma.$on('query', logQuery);
}
- return prisma;
-}
+ if (process.env.NODE_ENV !== 'production') {
+ global[PRISMA] = prisma;
+ }
-let prisma = global.prisma || getClient(options);
+ log('Prisma initialized');
-if (process.env.NODE_ENV !== 'production') {
- global.prisma = prisma;
+ return prisma;
}
-export function getDateQuery(field, unit, timezone) {
+function getDateQuery(field, unit, timezone) {
const db = getDatabaseType(process.env.DATABASE_URL);
if (db === POSTGRESQL) {
@@ -63,7 +74,7 @@ export function getDateQuery(field, unit, timezone) {
}
}
-export function getTimestampInterval(field) {
+function getTimestampInterval(field) {
const db = getDatabaseType(process.env.DATABASE_URL);
if (db === POSTGRESQL) {
@@ -75,7 +86,7 @@ export function getTimestampInterval(field) {
}
}
-export function getFilterQuery(table, column, filters = {}, params = []) {
+function getFilterQuery(table, column, filters = {}, params = []) {
const query = Object.keys(filters).reduce((arr, key) => {
const filter = filters[key];
@@ -135,7 +146,7 @@ export function getFilterQuery(table, column, filters = {}, params = []) {
return query.join('\n');
}
-export function parseFilters(table, column, filters = {}, params = [], sessionKey = 'session_id') {
+function parseFilters(table, column, filters = {}, params = [], sessionKey = 'session_id') {
const { domain, url, event_url, referrer, os, browser, device, country, event_name, query } =
filters;
@@ -158,13 +169,7 @@ export function parseFilters(table, column, filters = {}, params = [], sessionKe
};
}
-export async function runQuery(query) {
- return query.catch(e => {
- throw e;
- });
-}
-
-export async function rawQuery(query, params = []) {
+async function rawQuery(query, params = []) {
const db = getDatabaseType(process.env.DATABASE_URL);
if (db !== POSTGRESQL && db !== MYSQL) {
@@ -173,17 +178,23 @@ export async function rawQuery(query, params = []) {
const sql = db === MYSQL ? query.replace(/\$[0-9]+/g, '?') : query;
- return runQuery(prisma.$queryRawUnsafe.apply(prisma, [sql, ...params]));
+ return prisma.$queryRawUnsafe.apply(prisma, [sql, ...params]);
+}
+
+async function multiQuery(queries) {
+ return prisma.$transaction(queries);
}
-export { prisma };
+// Initialization
+const prisma = global[PRISMA] || getClient(PRISMA_OPTIONS);
export default {
- prisma,
+ client: prisma,
+ log,
getDateQuery,
getTimestampInterval,
getFilterQuery,
parseFilters,
- runQuery,
rawQuery,
+ multiQuery,
};
diff --git a/lib/redis.js b/lib/redis.js
index 97011bed5..e307729b3 100644
--- a/lib/redis.js
+++ b/lib/redis.js
@@ -1,6 +1,10 @@
import { createClient } from 'redis';
import { startOfMonth } from 'date-fns';
import { getSessions, getAllWebsites } from '/queries';
+import debug from 'debug';
+
+const log = debug('db:redis');
+const REDIS = Symbol.for('redis');
async function getClient() {
const redis = new createClient({
@@ -10,29 +14,15 @@ async function getClient() {
await redis.connect();
if (process.env.LOG_QUERY) {
- redis.on('error', err => console.log('Redis Client Error', err));
+ redis.on('error', err => log(err));
}
- return redis;
-}
-
-let redis = null;
-
-(async () => {
- redis = global.redis || (await getClient());
-
if (process.env.NODE_ENV !== 'production') {
- global.redis = redis;
+ global[REDIS] = redis;
}
- const value = await redis.get('initialized');
-
- if (!value) {
- await stageData();
- }
-})();
-
-export default redis;
+ return redis;
+}
async function stageData() {
const sessions = await getSessions([], startOfMonth(new Date()).toUTCString());
@@ -57,3 +47,18 @@ async function addRedis(ids) {
await redis.set(key, value);
}
}
+
+// Initialization
+let redis = null;
+
+(async () => {
+ redis = global[REDIS] || (await getClient());
+
+ const value = await redis.get('initialized');
+
+ if (!value) {
+ await stageData();
+ }
+})();
+
+export default redis;
diff --git a/queries/admin/account/createAccount.js b/queries/admin/account/createAccount.js
index fb3997e87..2c4e1a40b 100644
--- a/queries/admin/account/createAccount.js
+++ b/queries/admin/account/createAccount.js
@@ -1,9 +1,7 @@
-import { prisma, runQuery } from 'lib/relational';
+import prisma from 'lib/prisma';
export async function createAccount(data) {
- return runQuery(
- prisma.account.create({
- data,
- }),
- );
+ return prisma.client.account.create({
+ data,
+ });
}
diff --git a/queries/admin/account/deleteAccount.js b/queries/admin/account/deleteAccount.js
index f03b370ef..209d73a95 100644
--- a/queries/admin/account/deleteAccount.js
+++ b/queries/admin/account/deleteAccount.js
@@ -1,28 +1,28 @@
-import { prisma, runQuery } from 'lib/relational';
+import prisma from 'lib/prisma';
export async function deleteAccount(user_id) {
- return runQuery(
- prisma.$transaction([
- prisma.pageview.deleteMany({
- where: { session: { website: { user_id } } },
- }),
- prisma.event_data.deleteMany({
- where: { event: { session: { website: { user_id } } } },
- }),
- prisma.event.deleteMany({
- where: { session: { website: { user_id } } },
- }),
- prisma.session.deleteMany({
- where: { website: { user_id } },
- }),
- prisma.website.deleteMany({
- where: { user_id },
- }),
- prisma.account.delete({
- where: {
- user_id,
- },
- }),
- ]),
- );
+ const { client } = prisma;
+
+ return client.$transaction([
+ client.pageview.deleteMany({
+ where: { session: { website: { user_id } } },
+ }),
+ client.event_data.deleteMany({
+ where: { event: { session: { website: { user_id } } } },
+ }),
+ client.event.deleteMany({
+ where: { session: { website: { user_id } } },
+ }),
+ client.session.deleteMany({
+ where: { website: { user_id } },
+ }),
+ client.website.deleteMany({
+ where: { user_id },
+ }),
+ client.account.delete({
+ where: {
+ user_id,
+ },
+ }),
+ ]);
}
diff --git a/queries/admin/account/getAccountById.js b/queries/admin/account/getAccountById.js
index 4c3d647ce..74c4a4a96 100644
--- a/queries/admin/account/getAccountById.js
+++ b/queries/admin/account/getAccountById.js
@@ -1,11 +1,9 @@
-import { prisma, runQuery } from 'lib/relational';
+import prisma from 'lib/prisma';
export async function getAccountById(user_id) {
- return runQuery(
- prisma.account.findUnique({
- where: {
- user_id,
- },
- }),
- );
+ return prisma.client.account.findUnique({
+ where: {
+ user_id,
+ },
+ });
}
diff --git a/queries/admin/account/getAccountByUsername.js b/queries/admin/account/getAccountByUsername.js
index 30480a71b..ff64c8cef 100644
--- a/queries/admin/account/getAccountByUsername.js
+++ b/queries/admin/account/getAccountByUsername.js
@@ -1,11 +1,9 @@
-import { prisma, runQuery } from 'lib/relational';
+import prisma from 'lib/prisma';
export async function getAccountByUsername(username) {
- return runQuery(
- prisma.account.findUnique({
- where: {
- username,
- },
- }),
- );
+ return prisma.client.account.findUnique({
+ where: {
+ username,
+ },
+ });
}
diff --git a/queries/admin/account/getAccounts.js b/queries/admin/account/getAccounts.js
index 94f70c350..154626d10 100644
--- a/queries/admin/account/getAccounts.js
+++ b/queries/admin/account/getAccounts.js
@@ -1,21 +1,19 @@
-import { prisma, runQuery } from 'lib/relational';
+import prisma from 'lib/prisma';
export async function getAccounts() {
- return runQuery(
- prisma.account.findMany({
- orderBy: [
- { is_admin: 'desc' },
- {
- username: 'asc',
- },
- ],
- select: {
- user_id: true,
- username: true,
- is_admin: true,
- created_at: true,
- updated_at: true,
+ return prisma.client.account.findMany({
+ orderBy: [
+ { is_admin: 'desc' },
+ {
+ username: 'asc',
},
- }),
- );
+ ],
+ select: {
+ user_id: true,
+ username: true,
+ is_admin: true,
+ created_at: true,
+ updated_at: true,
+ },
+ });
}
diff --git a/queries/admin/account/updateAccount.js b/queries/admin/account/updateAccount.js
index 7a337f59c..80509decb 100644
--- a/queries/admin/account/updateAccount.js
+++ b/queries/admin/account/updateAccount.js
@@ -1,12 +1,10 @@
-import { prisma, runQuery } from 'lib/relational';
+import prisma from 'lib/prisma';
export async function updateAccount(user_id, data) {
- return runQuery(
- prisma.account.update({
- where: {
- user_id,
- },
- data,
- }),
- );
+ return prisma.client.account.update({
+ where: {
+ user_id,
+ },
+ data,
+ });
}
diff --git a/queries/admin/website/createWebsite.js b/queries/admin/website/createWebsite.js
index 7a589db6f..d5eaaf312 100644
--- a/queries/admin/website/createWebsite.js
+++ b/queries/admin/website/createWebsite.js
@@ -1,9 +1,9 @@
-import { prisma, runQuery } from 'lib/relational';
+import prisma from 'lib/prisma';
import redis from 'lib/redis';
export async function createWebsite(user_id, data) {
- return runQuery(
- prisma.website.create({
+ return prisma.client.website
+ .create({
data: {
account: {
connect: {
@@ -12,12 +12,12 @@ export async function createWebsite(user_id, data) {
},
...data,
},
- }),
- ).then(async res => {
- if (process.env.REDIS_URL) {
- await redis.set(`website:${res.website_uuid}`, Number(res.website_id));
- }
+ })
+ .then(async res => {
+ if (process.env.REDIS_URL) {
+ await redis.set(`website:${res.website_uuid}`, Number(res.website_id));
+ }
- return res;
- });
+ return res;
+ });
}
diff --git a/queries/admin/website/deleteWebsite.js b/queries/admin/website/deleteWebsite.js
index 13c3160df..15030444f 100644
--- a/queries/admin/website/deleteWebsite.js
+++ b/queries/admin/website/deleteWebsite.js
@@ -1,30 +1,28 @@
-import { prisma, runQuery } from 'lib/relational';
+import prisma from 'lib/prisma';
import redis from 'lib/redis';
export async function deleteWebsite(website_id) {
- return runQuery(
- prisma
- .$transaction([
- prisma.pageview.deleteMany({
- where: { session: { website: { website_id } } },
- }),
- prisma.event_data.deleteMany({
- where: { event: { session: { website: { website_id } } } },
- }),
- prisma.event.deleteMany({
- where: { session: { website: { website_id } } },
- }),
- prisma.session.deleteMany({
- where: { website: { website_id } },
- }),
- prisma.website.delete({
- where: { website_id },
- }),
- ])
- .then(async res => {
- if (process.env.REDIS_URL) {
- await redis.del(`website:${res.website_uuid}`);
- }
- }),
- );
+ const { client, multiQuery } = prisma;
+
+ return multiQuery([
+ client.pageview.deleteMany({
+ where: { session: { website: { website_id } } },
+ }),
+ client.event_data.deleteMany({
+ where: { event: { session: { website: { website_id } } } },
+ }),
+ client.event.deleteMany({
+ where: { session: { website: { website_id } } },
+ }),
+ client.session.deleteMany({
+ where: { website: { website_id } },
+ }),
+ client.website.delete({
+ where: { website_id },
+ }),
+ ]).then(async res => {
+ if (process.env.REDIS_URL) {
+ await redis.del(`website:${res.website_uuid}`);
+ }
+ });
}
diff --git a/queries/admin/website/getAllWebsites.js b/queries/admin/website/getAllWebsites.js
index f8d2a3e9c..23afd8137 100644
--- a/queries/admin/website/getAllWebsites.js
+++ b/queries/admin/website/getAllWebsites.js
@@ -1,24 +1,23 @@
-import { prisma, runQuery } from 'lib/relational';
+import prisma from 'lib/prisma';
export async function getAllWebsites() {
- let data = await runQuery(
- prisma.website.findMany({
- orderBy: [
- {
- user_id: 'asc',
- },
- {
- name: 'asc',
- },
- ],
- include: {
- account: {
- select: {
- username: true,
- },
+ let data = await prisma.client.website.findMany({
+ orderBy: [
+ {
+ user_id: 'asc',
+ },
+ {
+ name: 'asc',
+ },
+ ],
+ include: {
+ account: {
+ select: {
+ username: true,
},
},
- }),
- );
+ },
+ });
+
return data.map(i => ({ ...i, account: i.account.username }));
}
diff --git a/queries/admin/website/getUserWebsites.js b/queries/admin/website/getUserWebsites.js
index 729ce893d..5a9662c06 100644
--- a/queries/admin/website/getUserWebsites.js
+++ b/queries/admin/website/getUserWebsites.js
@@ -1,14 +1,12 @@
-import { prisma, runQuery } from 'lib/relational';
+import prisma from 'lib/prisma';
export async function getUserWebsites(user_id) {
- return runQuery(
- prisma.website.findMany({
- where: {
- user_id,
- },
- orderBy: {
- name: 'asc',
- },
- }),
- );
+ return prisma.client.website.findMany({
+ where: {
+ user_id,
+ },
+ orderBy: {
+ name: 'asc',
+ },
+ });
}
diff --git a/queries/admin/website/getWebsiteById.js b/queries/admin/website/getWebsiteById.js
index f4495b934..9b8cfe931 100644
--- a/queries/admin/website/getWebsiteById.js
+++ b/queries/admin/website/getWebsiteById.js
@@ -1,11 +1,9 @@
-import { prisma, runQuery } from 'lib/relational';
+import prisma from 'lib/prisma';
export async function getWebsiteById(website_id) {
- return runQuery(
- prisma.website.findUnique({
- where: {
- website_id,
- },
- }),
- );
+ return prisma.client.website.findUnique({
+ where: {
+ website_id,
+ },
+ });
}
diff --git a/queries/admin/website/getWebsiteByShareId.js b/queries/admin/website/getWebsiteByShareId.js
index 5630f89e2..fbdeebc98 100644
--- a/queries/admin/website/getWebsiteByShareId.js
+++ b/queries/admin/website/getWebsiteByShareId.js
@@ -1,11 +1,9 @@
-import { prisma, runQuery } from 'lib/relational';
+import prisma from 'lib/prisma';
export async function getWebsiteByShareId(share_id) {
- return runQuery(
- prisma.website.findUnique({
- where: {
- share_id,
- },
- }),
- );
+ return prisma.client.website.findUnique({
+ where: {
+ share_id,
+ },
+ });
}
diff --git a/queries/admin/website/getWebsiteByUuid.js b/queries/admin/website/getWebsiteByUuid.js
index dd0f24eea..1b9aeb4cc 100644
--- a/queries/admin/website/getWebsiteByUuid.js
+++ b/queries/admin/website/getWebsiteByUuid.js
@@ -1,11 +1,9 @@
-import { prisma, runQuery } from 'lib/relational';
+import prisma from 'lib/prisma';
export async function getWebsiteByUuid(website_uuid) {
- return runQuery(
- prisma.website.findUnique({
- where: {
- website_uuid,
- },
- }),
- );
+ return prisma.client.website.findUnique({
+ where: {
+ website_uuid,
+ },
+ });
}
diff --git a/queries/admin/website/resetWebsite.js b/queries/admin/website/resetWebsite.js
index 92bb959c9..661cd65cd 100644
--- a/queries/admin/website/resetWebsite.js
+++ b/queries/admin/website/resetWebsite.js
@@ -1,5 +1,25 @@
-import { prisma, runQuery } from 'lib/relational';
+import prisma from 'lib/prisma';
+import redis from 'lib/redis';
export async function resetWebsite(website_id) {
- return runQuery(prisma.$queryRaw`delete from session where website_id=${website_id}`);
+ const { client, multiQuery } = prisma;
+
+ return multiQuery([
+ client.pageview.deleteMany({
+ where: { session: { website: { website_id } } },
+ }),
+ client.event_data.deleteMany({
+ where: { event: { session: { website: { website_id } } } },
+ }),
+ client.event.deleteMany({
+ where: { session: { website: { website_id } } },
+ }),
+ client.session.deleteMany({
+ where: { website: { website_id } },
+ }),
+ ]).then(async res => {
+ if (process.env.REDIS_URL) {
+ await redis.del(`website:${res.website_uuid}`);
+ }
+ });
}
diff --git a/queries/admin/website/updateWebsite.js b/queries/admin/website/updateWebsite.js
index 9de1a803a..54f01f66d 100644
--- a/queries/admin/website/updateWebsite.js
+++ b/queries/admin/website/updateWebsite.js
@@ -1,12 +1,10 @@
-import { prisma, runQuery } from 'lib/relational';
+import prisma from 'lib/prisma';
export async function updateWebsite(website_id, data) {
- return runQuery(
- prisma.website.update({
- where: {
- website_id,
- },
- data,
- }),
- );
+ return prisma.client.website.update({
+ where: {
+ website_id,
+ },
+ data,
+ });
}
diff --git a/queries/analytics/event/getEventMetrics.js b/queries/analytics/event/getEventMetrics.js
index 531701e8b..84c007778 100644
--- a/queries/analytics/event/getEventMetrics.js
+++ b/queries/analytics/event/getEventMetrics.js
@@ -1,11 +1,10 @@
-import { CLICKHOUSE, RELATIONAL } from 'lib/constants';
+import prisma from 'lib/prisma';
import clickhouse from 'lib/clickhouse';
-import { getDateQuery, getFilterQuery, rawQuery } from 'lib/relational';
-import { runAnalyticsQuery } from 'lib/db';
+import { runQuery, CLICKHOUSE, PRISMA } from 'lib/db';
export async function getEventMetrics(...args) {
- return runAnalyticsQuery({
- [RELATIONAL]: () => relationalQuery(...args),
+ return runQuery({
+ [PRISMA]: () => relationalQuery(...args),
[CLICKHOUSE]: () => clickhouseQuery(...args),
});
}
@@ -18,11 +17,11 @@ async function relationalQuery(
unit = 'day',
filters = {},
) {
+ const { rawQuery, getDateQuery, getFilterQuery } = prisma;
const params = [website_id, start_at, end_at];
return rawQuery(
- `
- select
+ `select
event_name x,
${getDateQuery('created_at', unit, timezone)} t,
count(*) y
@@ -31,8 +30,7 @@ async function relationalQuery(
and created_at between $2 and $3
${getFilterQuery('event', filters, params)}
group by 1, 2
- order by 2
- `,
+ order by 2`,
params,
);
}
@@ -45,21 +43,20 @@ async function clickhouseQuery(
unit = 'day',
filters = {},
) {
+ const { rawQuery, getDateQuery, getBetweenDates } = prisma;
const params = [website_id];
- return clickhouse.rawQuery(
- `
- select
+ return rawQuery(
+ `select
event_name x,
- ${clickhouse.getDateQuery('created_at', unit, timezone)} t,
+ ${getDateQuery('created_at', unit, timezone)} t,
count(*) y
from event
where website_id= $1
- and ${clickhouse.getBetweenDates('created_at', start_at, end_at)}
+ and ${getBetweenDates('created_at', start_at, end_at)}
${clickhouse.getFilterQuery('event', filters, params)}
group by x, t
- order by t
- `,
+ order by t`,
params,
);
}
diff --git a/queries/analytics/event/getEvents.js b/queries/analytics/event/getEvents.js
index 6bfb6fb8b..317a8f2e7 100644
--- a/queries/analytics/event/getEvents.js
+++ b/queries/analytics/event/getEvents.js
@@ -1,36 +1,34 @@
-import { CLICKHOUSE, RELATIONAL } from 'lib/constants';
-import { prisma, runQuery } from 'lib/relational';
+import prisma from 'lib/prisma';
import clickhouse from 'lib/clickhouse';
-import { runAnalyticsQuery } from 'lib/db';
+import { runQuery, CLICKHOUSE, PRISMA } from 'lib/db';
export function getEvents(...args) {
- return runAnalyticsQuery({
- [RELATIONAL]: () => relationalQuery(...args),
+ return runQuery({
+ [PRISMA]: () => relationalQuery(...args),
[CLICKHOUSE]: () => clickhouseQuery(...args),
});
}
function relationalQuery(websites, start_at) {
- return runQuery(
- prisma.event.findMany({
- where: {
- website: {
- website_id: {
- in: websites,
- },
- },
- created_at: {
- gte: start_at,
+ return prisma.client.event.findMany({
+ where: {
+ website: {
+ website_id: {
+ in: websites,
},
},
- }),
- );
+ created_at: {
+ gte: start_at,
+ },
+ },
+ });
}
function clickhouseQuery(websites, start_at) {
- return clickhouse.rawQuery(
- `
- select
+ const { rawQuery, getDateFormat } = clickhouse;
+
+ return rawQuery(
+ `select
event_id,
website_id,
session_id,
@@ -39,7 +37,6 @@ function clickhouseQuery(websites, start_at) {
event_name
from event
where website_id in (${websites.join[',']}
- and created_at >= ${clickhouse.getDateFormat(start_at)})
- `,
+ and created_at >= ${getDateFormat(start_at)})`,
);
}
diff --git a/queries/analytics/event/saveEvent.js b/queries/analytics/event/saveEvent.js
index b54144edb..10e894aee 100644
--- a/queries/analytics/event/saveEvent.js
+++ b/queries/analytics/event/saveEvent.js
@@ -1,12 +1,12 @@
-import { CLICKHOUSE, KAFKA, RELATIONAL, URL_LENGTH } from 'lib/constants';
+import prisma from 'lib/prisma';
import clickhouse from 'lib/clickhouse';
import kafka from 'lib/kafka';
-import { prisma, runQuery } from 'lib/relational';
-import { runAnalyticsQuery } from 'lib/db';
+import { runQuery, CLICKHOUSE, KAFKA, PRISMA } from 'lib/db';
+import { URL_LENGTH, EVENT_NAME_LENGTH } from 'lib/constants';
export async function saveEvent(...args) {
- return runAnalyticsQuery({
- [RELATIONAL]: () => relationalQuery(...args),
+ return runQuery({
+ [PRISMA]: () => relationalQuery(...args),
[CLICKHOUSE]: () => clickhouseQuery(...args),
[KAFKA]: () => kafkaQuery(...args),
});
@@ -16,8 +16,8 @@ async function relationalQuery(website_id, { session_id, url, event_name, event_
const data = {
website_id,
session_id,
- url: url?.substr(0, URL_LENGTH),
- event_name: event_name?.substr(0, 50),
+ url: url?.substring(0, URL_LENGTH),
+ event_name: event_name?.substring(0, EVENT_NAME_LENGTH),
};
if (event_data) {
@@ -28,39 +28,38 @@ async function relationalQuery(website_id, { session_id, url, event_name, event_
};
}
- return runQuery(
- prisma.event.create({
- data,
- }),
- );
+ return prisma.client.event.create({
+ data,
+ });
}
async function clickhouseQuery(website_id, { event_uuid, session_uuid, url, event_name }) {
+ const { rawQuery, getDateFormat } = clickhouse;
const params = [
website_id,
event_uuid,
session_uuid,
- url?.substr(0, URL_LENGTH),
- event_name?.substr(0, 50),
+ url?.substring(0, URL_LENGTH),
+ event_name?.substring(0, EVENT_NAME_LENGTH),
];
- return clickhouse.rawQuery(
- `
- insert into umami.event (created_at, website_id, session_uuid, url, event_name)
- values (${clickhouse.getDateFormat(new Date())}, $1, $2, $3, $4);`,
+ return rawQuery(
+ `insert into umami.event (created_at, website_id, session_uuid, url, event_name)
+ values (${getDateFormat(new Date())}, $1, $2, $3, $4);`,
params,
);
}
async function kafkaQuery(website_id, { event_uuid, session_uuid, url, event_name }) {
+ const { getDateFormat, sendMessage } = kafka;
const params = {
event_uuid: event_uuid,
website_id: website_id,
session_uuid: session_uuid,
- created_at: kafka.getDateFormat(new Date()),
- url: url?.substr(0, URL_LENGTH),
- event_name: event_name?.substr(0, 50),
+ created_at: getDateFormat(new Date()),
+ url: url?.substring(0, URL_LENGTH),
+ event_name: event_name?.substring(0, EVENT_NAME_LENGTH),
};
- await kafka.sendMessage(params, 'event');
+ await sendMessage(params, 'event');
}
diff --git a/queries/analytics/pageview/getPageviewMetrics.js b/queries/analytics/pageview/getPageviewMetrics.js
index 40af97d0e..d34f4c17e 100644
--- a/queries/analytics/pageview/getPageviewMetrics.js
+++ b/queries/analytics/pageview/getPageviewMetrics.js
@@ -1,16 +1,16 @@
-import { CLICKHOUSE, RELATIONAL } from 'lib/constants';
+import prisma from 'lib/prisma';
import clickhouse from 'lib/clickhouse';
-import { parseFilters, rawQuery } from 'lib/relational';
-import { runAnalyticsQuery } from 'lib/db';
+import { runQuery, CLICKHOUSE, PRISMA } from 'lib/db';
export async function getPageviewMetrics(...args) {
- return runAnalyticsQuery({
- [RELATIONAL]: () => relationalQuery(...args),
+ return runQuery({
+ [PRISMA]: () => relationalQuery(...args),
[CLICKHOUSE]: () => clickhouseQuery(...args),
});
}
async function relationalQuery(website_id, start_at, end_at, column, table, filters = {}) {
+ const { rawQuery, parseFilters } = prisma;
const params = [website_id, start_at, end_at];
const { pageviewQuery, sessionQuery, eventQuery, joinSession } = parseFilters(
table,
@@ -20,8 +20,7 @@ async function relationalQuery(website_id, start_at, end_at, column, table, filt
);
return rawQuery(
- `
- select ${column} x, count(*) y
+ `select ${column} x, count(*) y
from ${table}
${joinSession}
where ${table}.website_id=$1
@@ -30,15 +29,15 @@ async function relationalQuery(website_id, start_at, end_at, column, table, filt
${joinSession && sessionQuery}
${eventQuery}
group by 1
- order by 2 desc
- `,
+ order by 2 desc`,
params,
);
}
async function clickhouseQuery(website_id, start_at, end_at, column, table, filters = {}) {
+ const { rawQuery, parseFilters, getBetweenDates } = clickhouse;
const params = [website_id];
- const { pageviewQuery, sessionQuery, eventQuery, joinSession } = clickhouse.parseFilters(
+ const { pageviewQuery, sessionQuery, eventQuery, joinSession } = parseFilters(
table,
column,
filters,
@@ -46,19 +45,17 @@ async function clickhouseQuery(website_id, start_at, end_at, column, table, filt
'session_uuid',
);
- return clickhouse.rawQuery(
- `
- select ${column} x, count(*) y
+ return rawQuery(
+ `select ${column} x, count(*) y
from ${table}
${joinSession}
where ${table}.website_id= $1
- and ${clickhouse.getBetweenDates(table + '.created_at', start_at, end_at)}
+ and ${getBetweenDates(table + '.created_at', start_at, end_at)}
${pageviewQuery}
${joinSession && sessionQuery}
${eventQuery}
group by x
- order by y desc
- `,
+ order by y desc`,
params,
);
}
diff --git a/queries/analytics/pageview/getPageviewParams.js b/queries/analytics/pageview/getPageviewParams.js
index be6e5148e..fa445da58 100644
--- a/queries/analytics/pageview/getPageviewParams.js
+++ b/queries/analytics/pageview/getPageviewParams.js
@@ -1,14 +1,15 @@
-import { parseFilters, rawQuery, runAnalyticsQuery } from 'lib/relational';
-import { CLICKHOUSE, RELATIONAL } from 'lib/constants';
+import prisma from 'lib/prisma';
+import { runQuery, CLICKHOUSE, PRISMA } from 'lib/db';
export async function getPageviewParams(...args) {
- return runAnalyticsQuery({
- [RELATIONAL]: () => relationalQuery(...args),
+ return runQuery({
+ [PRISMA]: () => relationalQuery(...args),
[CLICKHOUSE]: () => clickhouseQuery(...args),
});
}
async function relationalQuery(website_id, start_at, end_at, column, table, filters = {}) {
+ const { parseFilters, rawQuery } = prisma;
const params = [website_id, start_at, end_at];
const { pageviewQuery, sessionQuery, eventQuery, joinSession } = parseFilters(
table,
@@ -18,8 +19,7 @@ async function relationalQuery(website_id, start_at, end_at, column, table, filt
);
return rawQuery(
- `
- select url x,
+ `select url x,
count(*) y
from ${table}
${joinSession}
@@ -30,8 +30,7 @@ async function relationalQuery(website_id, start_at, end_at, column, table, filt
${joinSession && sessionQuery}
${eventQuery}
group by 1
- order by 2 desc
- `,
+ order by 2 desc`,
params,
);
}
diff --git a/queries/analytics/pageview/getPageviewStats.js b/queries/analytics/pageview/getPageviewStats.js
index 207f109b8..da25ab0c6 100644
--- a/queries/analytics/pageview/getPageviewStats.js
+++ b/queries/analytics/pageview/getPageviewStats.js
@@ -1,11 +1,10 @@
-import { CLICKHOUSE, RELATIONAL } from 'lib/constants';
-import { getDateQuery, parseFilters, rawQuery } from 'lib/relational';
-import { runAnalyticsQuery } from 'lib/db';
+import prisma from 'lib/prisma';
import clickhouse from 'lib/clickhouse';
+import { runQuery, CLICKHOUSE, PRISMA } from 'lib/db';
export async function getPageviewStats(...args) {
- return runAnalyticsQuery({
- [RELATIONAL]: () => relationalQuery(...args),
+ return runQuery({
+ [PRISMA]: () => relationalQuery(...args),
[CLICKHOUSE]: () => clickhouseQuery(...args),
});
}
@@ -20,6 +19,7 @@ async function relationalQuery(
filters = {},
sessionKey = 'session_id',
) {
+ const { getDateQuery, parseFilters, rawQuery } = prisma;
const params = [website_id, start_at, end_at];
const { pageviewQuery, sessionQuery, joinSession } = parseFilters(
'pageview',
@@ -29,8 +29,7 @@ async function relationalQuery(
);
return rawQuery(
- `
- select ${getDateQuery('pageview.created_at', unit, timezone)} t,
+ `select ${getDateQuery('pageview.created_at', unit, timezone)} t,
count(${count !== '*' ? `${count}${sessionKey}` : count}) y
from pageview
${joinSession}
@@ -38,8 +37,7 @@ async function relationalQuery(
and pageview.created_at between $2 and $3
${pageviewQuery}
${sessionQuery}
- group by 1
- `,
+ group by 1`,
params,
);
}
@@ -54,6 +52,7 @@ async function clickhouseQuery(
filters = {},
sessionKey = 'session_uuid',
) {
+ const { parseFilters, rawQuery, getDateStringQuery, getDateQuery, getBetweenDates } = clickhouse;
const params = [website_id];
const { pageviewQuery, sessionQuery, joinSession } = parseFilters(
'pageview',
@@ -63,24 +62,22 @@ async function clickhouseQuery(
sessionKey,
);
- return clickhouse.rawQuery(
- `
- select
- ${clickhouse.getDateStringQuery('g.t', unit)} as t,
+ return rawQuery(
+ `select
+ ${getDateStringQuery('g.t', unit)} as t,
g.y as y
from
(select
- ${clickhouse.getDateQuery('created_at', unit, timezone)} t,
+ ${getDateQuery('created_at', unit, timezone)} t,
count(${count !== '*' ? `${count}${sessionKey}` : count}) y
from pageview
${joinSession}
where pageview.website_id= $1
- and ${clickhouse.getBetweenDates('pageview.created_at', start_at, end_at)}
+ and ${getBetweenDates('pageview.created_at', start_at, end_at)}
${pageviewQuery}
${sessionQuery}
group by t) g
- order by t
- `,
+ order by t`,
params,
);
}
diff --git a/queries/analytics/pageview/getPageviews.js b/queries/analytics/pageview/getPageviews.js
index fbbd96f85..8ce704d89 100644
--- a/queries/analytics/pageview/getPageviews.js
+++ b/queries/analytics/pageview/getPageviews.js
@@ -1,36 +1,32 @@
-import { CLICKHOUSE, RELATIONAL } from 'lib/constants';
-import { prisma, runQuery } from 'lib/relational';
-import { runAnalyticsQuery } from 'lib/db';
+import prisma from 'lib/prisma';
import clickhouse from 'lib/clickhouse';
+import { runQuery, CLICKHOUSE, PRISMA } from 'lib/db';
export async function getPageviews(...args) {
- return runAnalyticsQuery({
- [RELATIONAL]: () => relationalQuery(...args),
+ return runQuery({
+ [PRISMA]: () => relationalQuery(...args),
[CLICKHOUSE]: () => clickhouseQuery(...args),
});
}
async function relationalQuery(websites, start_at) {
- return runQuery(
- prisma.pageview.findMany({
- where: {
- website: {
- website_id: {
- in: websites,
- },
- },
- created_at: {
- gte: start_at,
+ return prisma.client.pageview.findMany({
+ where: {
+ website: {
+ website_id: {
+ in: websites,
},
},
- }),
- );
+ created_at: {
+ gte: start_at,
+ },
+ },
+ });
}
async function clickhouseQuery(websites, start_at) {
return clickhouse.rawQuery(
- `
- select
+ `select
view_id,
website_id,
session_id,
@@ -38,7 +34,6 @@ async function clickhouseQuery(websites, start_at) {
url
from pageview
where website_id in (${websites.join[',']}
- and created_at >= ${clickhouse.getDateFormat(start_at)})
- `,
+ and created_at >= ${clickhouse.getDateFormat(start_at)})`,
);
}
diff --git a/queries/analytics/pageview/savePageView.js b/queries/analytics/pageview/savePageView.js
index c2b9527e1..366981273 100644
--- a/queries/analytics/pageview/savePageView.js
+++ b/queries/analytics/pageview/savePageView.js
@@ -1,54 +1,52 @@
-import { CLICKHOUSE, KAFKA, RELATIONAL, URL_LENGTH } from 'lib/constants';
+import prisma from 'lib/prisma';
import clickhouse from 'lib/clickhouse';
-import { runAnalyticsQuery } from 'lib/db';
import kafka from 'lib/kafka';
-import { prisma, runQuery } from 'lib/relational';
+import { runQuery, CLICKHOUSE, KAFKA, PRISMA } from 'lib/db';
+import { URL_LENGTH } from 'lib/constants';
export async function savePageView(...args) {
- return runAnalyticsQuery({
- [RELATIONAL]: () => relationalQuery(...args),
+ return runQuery({
+ [PRISMA]: () => relationalQuery(...args),
[CLICKHOUSE]: () => clickhouseQuery(...args),
[KAFKA]: () => kafkaQuery(...args),
});
}
async function relationalQuery(website_id, { session_id, url, referrer }) {
- return runQuery(
- prisma.pageview.create({
- data: {
- website_id,
- session_id,
- url: url?.substr(0, URL_LENGTH),
- referrer: referrer?.substr(0, URL_LENGTH),
- },
- }),
- );
+ return prisma.client.pageview.create({
+ data: {
+ website_id,
+ session_id,
+ url: url?.substring(0, URL_LENGTH),
+ referrer: referrer?.substring(0, URL_LENGTH),
+ },
+ });
}
async function clickhouseQuery(website_id, { session_uuid, url, referrer }) {
const params = [
website_id,
session_uuid,
- url?.substr(0, URL_LENGTH),
- referrer?.substr(0, URL_LENGTH),
+ url?.substring(0, URL_LENGTH),
+ referrer?.substring(0, URL_LENGTH),
];
return clickhouse.rawQuery(
- `
- insert into umami.pageview (created_at, website_id, session_uuid, url, referrer)
+ `insert into umami.pageview (created_at, website_id, session_uuid, url, referrer)
values (${clickhouse.getDateFormat(new Date())}, $1, $2, $3, $4);`,
params,
);
}
async function kafkaQuery(website_id, { session_uuid, url, referrer }) {
+ const { getDateFormat, sendMessage } = kafka;
const params = {
website_id: website_id,
session_uuid: session_uuid,
- created_at: kafka.getDateFormat(new Date()),
- url: url?.substr(0, URL_LENGTH),
- referrer: referrer?.substr(0, URL_LENGTH),
+ created_at: getDateFormat(new Date()),
+ url: url?.substring(0, URL_LENGTH),
+ referrer: referrer?.substring(0, URL_LENGTH),
};
- await kafka.sendMessage(params, 'pageview');
+ await sendMessage(params, 'pageview');
}
diff --git a/queries/analytics/session/createSession.js b/queries/analytics/session/createSession.js
index a4419b7e3..994f9ef50 100644
--- a/queries/analytics/session/createSession.js
+++ b/queries/analytics/session/createSession.js
@@ -1,21 +1,20 @@
+import prisma from 'lib/prisma';
import clickhouse from 'lib/clickhouse';
-import { CLICKHOUSE, KAFKA, RELATIONAL } from 'lib/constants';
-import { runAnalyticsQuery } from 'lib/db';
import kafka from 'lib/kafka';
import redis from 'lib/redis';
-import { prisma, runQuery } from 'lib/relational';
+import { runQuery, CLICKHOUSE, KAFKA, PRISMA } from 'lib/db';
export async function createSession(...args) {
- return runAnalyticsQuery({
- [RELATIONAL]: () => relationalQuery(...args),
+ return runQuery({
+ [PRISMA]: () => relationalQuery(...args),
[CLICKHOUSE]: () => clickhouseQuery(...args),
[KAFKA]: () => kafkaQuery(...args),
});
}
async function relationalQuery(website_id, data) {
- return runQuery(
- prisma.session.create({
+ return prisma.client.session
+ .create({
data: {
website_id,
...data,
@@ -23,14 +22,14 @@ async function relationalQuery(website_id, data) {
select: {
session_id: true,
},
- }),
- ).then(async res => {
- if (process.env.REDIS_URL) {
- await redis.set(`session:${res.session_uuid}`, '');
- }
+ })
+ .then(async res => {
+ if (process.env.REDIS_URL) {
+ await redis.set(`session:${res.session_uuid}`, '');
+ }
- return res;
- });
+ return res;
+ });
}
async function clickhouseQuery(
@@ -48,10 +47,11 @@ async function clickhouseQuery(
language,
country ? country : null,
];
+ const { rawQuery, getDateFormat } = clickhouse;
- await clickhouse.rawQuery(
+ await rawQuery(
`insert into umami.session (created_at, session_uuid, website_id, hostname, browser, os, device, screen, language, country)
- values (${clickhouse.getDateFormat(new Date())}, $1, $2, $3, $4, $5, $6, $7, $8, $9);`,
+ values (${getDateFormat(new Date())}, $1, $2, $3, $4, $5, $6, $7, $8, $9);`,
params,
);
}
@@ -60,10 +60,11 @@ async function kafkaQuery(
website_id,
{ session_uuid, hostname, browser, os, screen, language, country, device },
) {
+ const { getDateFormat, sendMessage } = kafka;
const params = {
session_uuid: session_uuid,
website_id: website_id,
- created_at: kafka.getDateFormat(new Date()),
+ created_at: getDateFormat(new Date()),
hostname: hostname,
browser: browser,
os: os,
@@ -73,7 +74,7 @@ async function kafkaQuery(
country: country ? country : null,
};
- await kafka.sendMessage(params, 'session');
+ await sendMessage(params, 'session');
await redis.set(`session:${session_uuid}`, '');
}
diff --git a/queries/analytics/session/getSessionByUuid.js b/queries/analytics/session/getSessionByUuid.js
index 953029921..6e92a2ca6 100644
--- a/queries/analytics/session/getSessionByUuid.js
+++ b/queries/analytics/session/getSessionByUuid.js
@@ -1,32 +1,28 @@
-import { CLICKHOUSE, RELATIONAL } from 'lib/constants';
-import { prisma, runQuery } from 'lib/relational';
+import prisma from 'lib/prisma';
import clickhouse from 'lib/clickhouse';
-import { runAnalyticsQuery } from 'lib/db';
+import { runQuery, CLICKHOUSE, PRISMA } from 'lib/db';
export async function getSessionByUuid(...args) {
- return runAnalyticsQuery({
- [RELATIONAL]: () => relationalQuery(...args),
+ return runQuery({
+ [PRISMA]: () => relationalQuery(...args),
[CLICKHOUSE]: () => clickhouseQuery(...args),
});
}
async function relationalQuery(session_uuid) {
- return runQuery(
- prisma.session.findUnique({
- where: {
- session_uuid,
- },
- }),
- );
+ return prisma.client.session.findUnique({
+ where: {
+ session_uuid,
+ },
+ });
}
async function clickhouseQuery(session_uuid) {
+ const { rawQuery, findFirst } = clickhouse;
const params = [session_uuid];
- return clickhouse
- .rawQuery(
- `
- select
+ return rawQuery(
+ `select
session_uuid,
website_id,
created_at,
@@ -35,12 +31,10 @@ async function clickhouseQuery(session_uuid) {
os,
device,
screen,
- "language",
+ language,
country
from session
- where session_uuid = $1
- `,
- params,
- )
- .then(result => clickhouse.findFirst(result));
+ where session_uuid = $1`,
+ params,
+ ).then(result => findFirst(result));
}
diff --git a/queries/analytics/session/getSessionMetrics.js b/queries/analytics/session/getSessionMetrics.js
index e0d5a24ca..36eb75680 100644
--- a/queries/analytics/session/getSessionMetrics.js
+++ b/queries/analytics/session/getSessionMetrics.js
@@ -1,16 +1,16 @@
-import { CLICKHOUSE, RELATIONAL } from 'lib/constants';
+import prisma from 'lib/prisma';
import clickhouse from 'lib/clickhouse';
-import { runAnalyticsQuery } from 'lib/db';
-import { parseFilters, rawQuery } from 'lib/relational';
+import { runQuery, CLICKHOUSE, PRISMA } from 'lib/db';
export async function getSessionMetrics(...args) {
- return runAnalyticsQuery({
- [RELATIONAL]: () => relationalQuery(...args),
+ return runQuery({
+ [PRISMA]: () => relationalQuery(...args),
[CLICKHOUSE]: () => clickhouseQuery(...args),
});
}
async function relationalQuery(website_id, start_at, end_at, field, filters = {}) {
+ const { parseFilters, rawQuery } = prisma;
const params = [website_id, start_at, end_at];
const { pageviewQuery, sessionQuery, joinSession } = parseFilters(
'pageview',
@@ -20,8 +20,7 @@ async function relationalQuery(website_id, start_at, end_at, field, filters = {}
);
return rawQuery(
- `
- select ${field} x, count(*) y
+ `select ${field} x, count(*) y
from session as x
where x.session_id in (
select pageview.session_id
@@ -33,15 +32,15 @@ async function relationalQuery(website_id, start_at, end_at, field, filters = {}
${sessionQuery}
)
group by 1
- order by 2 desc
- `,
+ order by 2 desc`,
params,
);
}
async function clickhouseQuery(website_id, start_at, end_at, field, filters = {}) {
+ const { parseFilters, getBetweenDates, rawQuery } = clickhouse;
const params = [website_id];
- const { pageviewQuery, sessionQuery, joinSession } = clickhouse.parseFilters(
+ const { pageviewQuery, sessionQuery, joinSession } = parseFilters(
'pageview',
null,
filters,
@@ -49,22 +48,20 @@ async function clickhouseQuery(website_id, start_at, end_at, field, filters = {}
'session_uuid',
);
- return clickhouse.rawQuery(
- `
- select ${field} x, count(*) y
+ return rawQuery(
+ `select ${field} x, count(*) y
from session as x
where x.session_uuid in (
select pageview.session_uuid
from pageview
${joinSession}
where pageview.website_id=$1
- and ${clickhouse.getBetweenDates('pageview.created_at', start_at, end_at)}
+ and ${getBetweenDates('pageview.created_at', start_at, end_at)}
${pageviewQuery}
${sessionQuery}
)
group by x
- order by y desc
- `,
+ order by y desc`,
params,
);
}
diff --git a/queries/analytics/session/getSessions.js b/queries/analytics/session/getSessions.js
index bcb0f19e4..bf070651a 100644
--- a/queries/analytics/session/getSessions.js
+++ b/queries/analytics/session/getSessions.js
@@ -1,18 +1,17 @@
-import { CLICKHOUSE, RELATIONAL } from 'lib/constants';
+import prisma from 'lib/prisma';
import clickhouse from 'lib/clickhouse';
-import { runAnalyticsQuery } from 'lib/db';
-import { prisma, runQuery } from 'lib/relational';
+import { runQuery, PRISMA, CLICKHOUSE } from 'lib/db';
export async function getSessions(...args) {
- return runAnalyticsQuery({
- [RELATIONAL]: () => relationalQuery(...args),
+ return runQuery({
+ [PRISMA]: () => relationalQuery(...args),
[CLICKHOUSE]: () => clickhouseQuery(...args),
});
}
async function relationalQuery(websites, start_at) {
return runQuery(
- prisma.session.findMany({
+ prisma.client.session.findMany({
where: {
...(websites && websites.length > 0
? {
@@ -32,9 +31,10 @@ async function relationalQuery(websites, start_at) {
}
async function clickhouseQuery(websites, start_at) {
- return clickhouse.rawQuery(
- `
- select
+ const { rawQuery, getDateFormat } = clickhouse;
+
+ return rawQuery(
+ `select
session_uuid,
website_id,
created_at,
@@ -43,11 +43,10 @@ async function clickhouseQuery(websites, start_at) {
os,
device,
screen,
- "language",
+ language,
country
from session
where ${websites && websites.length > 0 ? `(website_id in (${websites.join[',']})` : '0 = 0'}
- and created_at >= ${clickhouse.getDateFormat(start_at)}
- `,
+ and created_at >= ${getDateFormat(start_at)}`,
);
}
diff --git a/queries/analytics/stats/getActiveVisitors.js b/queries/analytics/stats/getActiveVisitors.js
index 4e61d3ebd..023c00a4e 100644
--- a/queries/analytics/stats/getActiveVisitors.js
+++ b/queries/analytics/stats/getActiveVisitors.js
@@ -1,12 +1,11 @@
import { subMinutes } from 'date-fns';
-import { CLICKHOUSE, RELATIONAL } from 'lib/constants';
-import { rawQuery } from 'lib/relational';
-import { runAnalyticsQuery } from 'lib/db';
+import prisma from 'lib/prisma';
import clickhouse from 'lib/clickhouse';
+import { runQuery, CLICKHOUSE, PRISMA } from 'lib/db';
export async function getActiveVisitors(...args) {
- return runAnalyticsQuery({
- [RELATIONAL]: () => relationalQuery(...args),
+ return runQuery({
+ [PRISMA]: () => relationalQuery(...args),
[CLICKHOUSE]: () => clickhouseQuery(...args),
});
}
@@ -15,27 +14,24 @@ async function relationalQuery(website_id) {
const date = subMinutes(new Date(), 5);
const params = [website_id, date];
- return rawQuery(
- `
- select count(distinct session_id) x
+ return prisma.rawQuery(
+ `select count(distinct session_id) x
from pageview
where website_id = $1
- and created_at >= $2
- `,
+ and created_at >= $2`,
params,
);
}
async function clickhouseQuery(website_id) {
+ const { rawQuery, getDateFormat } = clickhouse;
const params = [website_id];
- return clickhouse.rawQuery(
- `
- select count(distinct session_uuid) x
+ return rawQuery(
+ `select count(distinct session_uuid) x
from pageview
where website_id = $1
- and created_at >= ${clickhouse.getDateFormat(subMinutes(new Date(), 5))}
- `,
+ and created_at >= ${getDateFormat(subMinutes(new Date(), 5))}`,
params,
);
}
diff --git a/queries/analytics/stats/getWebsiteStats.js b/queries/analytics/stats/getWebsiteStats.js
index 081a16acc..5d280563b 100644
--- a/queries/analytics/stats/getWebsiteStats.js
+++ b/queries/analytics/stats/getWebsiteStats.js
@@ -1,16 +1,16 @@
-import { CLICKHOUSE, RELATIONAL } from 'lib/constants';
-import { getDateQuery, getTimestampInterval, parseFilters, rawQuery } from 'lib/relational';
-import { runAnalyticsQuery } from 'lib/db';
+import prisma from 'lib/prisma';
import clickhouse from 'lib/clickhouse';
+import { runQuery, CLICKHOUSE, PRISMA } from 'lib/db';
export async function getWebsiteStats(...args) {
- return runAnalyticsQuery({
- [RELATIONAL]: () => relationalQuery(...args),
+ return runQuery({
+ [PRISMA]: () => relationalQuery(...args),
[CLICKHOUSE]: () => clickhouseQuery(...args),
});
}
async function relationalQuery(website_id, start_at, end_at, filters = {}) {
+ const { getDateQuery, getTimestampInterval, parseFilters, rawQuery } = prisma;
const params = [website_id, start_at, end_at];
const { pageviewQuery, sessionQuery, joinSession } = parseFilters(
'pageview',
@@ -20,32 +20,31 @@ async function relationalQuery(website_id, start_at, end_at, filters = {}) {
);
return rawQuery(
- `
- select sum(t.c) as "pageviews",
+ `select sum(t.c) as "pageviews",
count(distinct t.session_id) as "uniques",
sum(case when t.c = 1 then 1 else 0 end) as "bounces",
sum(t.time) as "totaltime"
from (
- select pageview.session_id,
- ${getDateQuery('pageview.created_at', 'hour')},
- count(*) c,
- ${getTimestampInterval('pageview.created_at')} as "time"
- from pageview
- ${joinSession}
- where pageview.website_id=$1
- and pageview.created_at between $2 and $3
- ${pageviewQuery}
- ${sessionQuery}
- group by 1, 2
- ) t
- `,
+ select pageview.session_id,
+ ${getDateQuery('pageview.created_at', 'hour')},
+ count(*) c,
+ ${getTimestampInterval('pageview.created_at')} as "time"
+ from pageview
+ ${joinSession}
+ where pageview.website_id=$1
+ and pageview.created_at between $2 and $3
+ ${pageviewQuery}
+ ${sessionQuery}
+ group by 1, 2
+ ) t`,
params,
);
}
async function clickhouseQuery(website_id, start_at, end_at, filters = {}) {
+ const { rawQuery, getDateQuery, getBetweenDates, parseFilters } = clickhouse;
const params = [website_id];
- const { pageviewQuery, sessionQuery, joinSession } = clickhouse.parseFilters(
+ const { pageviewQuery, sessionQuery, joinSession } = parseFilters(
'pageview',
null,
filters,
@@ -53,28 +52,26 @@ async function clickhouseQuery(website_id, start_at, end_at, filters = {}) {
'session_uuid',
);
- return clickhouse.rawQuery(
- `
- select
- sum(t.c) as "pageviews",
- count(distinct t.session_uuid) as "uniques",
- sum(if(t.c = 1, 1, 0)) as "bounces",
- sum(if(max_time < min_time + interval 1 hour, max_time-min_time, 0)) as "totaltime"
- from (
- select pageview.session_uuid,
- ${clickhouse.getDateQuery('pageview.created_at', 'day')} time_series,
- count(*) c,
- min(created_at) min_time,
- max(created_at) max_time
- from pageview
- ${joinSession}
- where pageview.website_id = $1
- and ${clickhouse.getBetweenDates('pageview.created_at', start_at, end_at)}
- ${pageviewQuery}
- ${sessionQuery}
- group by pageview.session_uuid, time_series
- ) t;
- `,
+ return rawQuery(
+ `select
+ sum(t.c) as "pageviews",
+ count(distinct t.session_uuid) as "uniques",
+ sum(if(t.c = 1, 1, 0)) as "bounces",
+ sum(if(max_time < min_time + interval 1 hour, max_time-min_time, 0)) as "totaltime"
+ from (
+ select pageview.session_uuid,
+ ${getDateQuery('pageview.created_at', 'day')} time_series,
+ count(*) c,
+ min(created_at) min_time,
+ max(created_at) max_time
+ from pageview
+ ${joinSession}
+ where pageview.website_id = $1
+ and ${getBetweenDates('pageview.created_at', start_at, end_at)}
+ ${pageviewQuery}
+ ${sessionQuery}
+ group by pageview.session_uuid, time_series
+ ) t;`,
params,
);
}
diff --git a/scripts/copy-db-files.js b/scripts/copy-db-files.js
index a7897fca3..3e902d45f 100644
--- a/scripts/copy-db-files.js
+++ b/scripts/copy-db-files.js
@@ -3,10 +3,8 @@ const fse = require('fs-extra');
const path = require('path');
const del = require('del');
-function getDatabaseType() {
- const type =
- process.env.DATABASE_TYPE ||
- (process.env.DATABASE_URL && process.env.DATABASE_URL.split(':')[0]);
+function getDatabaseType(url = process.env.DATABASE_URL) {
+ const type = process.env.DATABASE_TYPE || (url && url.split(':')[0]);
if (type === 'postgres') {
return 'postgresql';