Skip to content

armgit / umami

Public Imported from GitHub
0

Umami is a privacy-first web and product analytics platform. Traffic, campaigns, behavior, conversions, and revenue in one place. No cookies, no surveillance, self-hosted or in the cloud.

update CH rawquery and type

Francis Caoe2bb2defd447
diff --git a/src/lib/clickhouse.ts b/src/lib/clickhouse.ts
index 186a085d1..c371e7f97 100644
--- a/src/lib/clickhouse.ts
+++ b/src/lib/clickhouse.ts
@@ -1,4 +1,4 @@
-import { ClickHouse } from 'clickhouse';
+import { ClickHouseClient, createClient } from '@clickhouse/client';
import dateFormat from 'dateformat';
import debug from 'debug';
import { CLICKHOUSE } from 'lib/db';
@@ -17,7 +17,7 @@ export const CLICKHOUSE_DATE_FORMATS = {
const log = debug('umami:clickhouse');
-let clickhouse: ClickHouse;
+let clickhouse: ClickHouseClient;
const enabled = Boolean(process.env.CLICKHOUSE_URL);
function getClient() {
@@ -25,18 +25,19 @@ function getClient() {
hostname,
port,
pathname,
+ // protocol,
username = 'default',
password,
} = new URL(process.env.CLICKHOUSE_URL);
- const client = new ClickHouse({
- url: hostname,
- port: Number(port),
- format: 'json',
- config: {
- database: pathname.replace('/', ''),
- },
- basicAuth: password ? { username, password } : null,
+ // const formattedProtocol =
+ // protocol.toLowerCase() === 'clickhouse:' || protocol === 'https:' ? 'https:' : 'http:';
+
+ const client = createClient({
+ host: `http://${hostname}:${port}`,
+ database: pathname.replace('/', ''),
+ username: username,
+ password,
});
if (process.env.NODE_ENV !== 'production') {
@@ -118,7 +119,7 @@ async function parseFilters(websiteId: string, filters: QueryFilters = {}, optio
};
}
-async function rawQuery<T>(query: string, params: object = {}): Promise<T> {
+async function rawQuery(query: string, params: Record<string, unknown> = {}): Promise<unknown> {
if (process.env.LOG_QUERY) {
log('QUERY:\n', query);
log('PARAMETERS:\n', params);
@@ -126,7 +127,15 @@ async function rawQuery<T>(query: string, params: object = {}): Promise<T> {
await connect();
- return clickhouse.query(query, { params }).toPromise() as Promise<T>;
+ const resultSet = await clickhouse.query({
+ query: query,
+ query_params: params,
+ format: 'JSONEachRow',
+ });
+
+ const data = await resultSet.json();
+
+ return data;
}
async function findUnique(data) {
diff --git a/src/queries/analytics/eventData/getEventDataEvents.ts b/src/queries/analytics/eventData/getEventDataEvents.ts
index 2c8cb0e0d..29158503d 100644
--- a/src/queries/analytics/eventData/getEventDataEvents.ts
+++ b/src/queries/analytics/eventData/getEventDataEvents.ts
@@ -59,7 +59,10 @@ async function relationalQuery(websiteId: string, filters: QueryFilters) {
);
}
-async function clickhouseQuery(websiteId: string, filters: QueryFilters) {
+async function clickhouseQuery(
+ websiteId: string,
+ filters: QueryFilters,
+): Promise<{ eventName: string; fieldName: string; dataType: number; total: number }[]> {
const { rawQuery, parseFilters } = clickhouse;
const { event } = filters;
const { params } = await parseFilters(websiteId, filters);
@@ -75,14 +78,23 @@ async function clickhouseQuery(websiteId: string, filters: QueryFilters) {
count(*) as total
from event_data
where website_id = {websiteId:UUID}
- and created_at between {startDate:DateTime} and {endDate:DateTime}
+ and created_at between {startDate:DateTime64} and {endDate:DateTime64}
and event_name = {event:String}
group by event_key, data_type, string_value, event_name
order by 1 asc, 2 asc, 3 asc, 4 desc
limit 100
`,
params,
- );
+ ).then(a => {
+ return Object.values(a).map(a => {
+ return {
+ eventName: a.eventName,
+ fieldName: a.fieldName,
+ dataType: Number(a.dataType),
+ total: Number(a.total),
+ };
+ });
+ });
}
return rawQuery(
@@ -94,11 +106,20 @@ async function clickhouseQuery(websiteId: string, filters: QueryFilters) {
count(*) as total
from event_data
where website_id = {websiteId:UUID}
- and created_at between {startDate:DateTime} and {endDate:DateTime}
+ and created_at between {startDate:DateTime64} and {endDate:DateTime64}
group by event_key, data_type, event_name
order by 1 asc, 2 asc
limit 100
`,
params,
- );
+ ).then(a => {
+ return Object.values(a).map(a => {
+ return {
+ eventName: a.eventName,
+ fieldName: a.fieldName,
+ dataType: Number(a.dataType),
+ total: Number(a.total),
+ };
+ });
+ });
}
diff --git a/src/queries/analytics/eventData/getEventDataFields.ts b/src/queries/analytics/eventData/getEventDataFields.ts
index ac32b188f..df5a8874f 100644
--- a/src/queries/analytics/eventData/getEventDataFields.ts
+++ b/src/queries/analytics/eventData/getEventDataFields.ts
@@ -37,7 +37,10 @@ async function relationalQuery(websiteId: string, filters: QueryFilters & { fiel
);
}
-async function clickhouseQuery(websiteId: string, filters: QueryFilters & { field?: string }) {
+async function clickhouseQuery(
+ websiteId: string,
+ filters: QueryFilters & { field?: string },
+): Promise<{ fieldName: string; dataType: number; fieldValue: string; total: number }[]> {
const { rawQuery, parseFilters } = clickhouse;
const { filterQuery, params } = await parseFilters(websiteId, filters, {
columns: { field: 'event_key' },
@@ -52,12 +55,21 @@ async function clickhouseQuery(websiteId: string, filters: QueryFilters & { fiel
count(*) as total
from event_data
where website_id = {websiteId:UUID}
- and created_at between {startDate:DateTime} and {endDate:DateTime}
+ and created_at between {startDate:DateTime64} and {endDate:DateTime64}
${filterQuery}
group by event_key, data_type, string_value
order by 3 desc, 2 desc, 1 asc
limit 100
`,
params,
- );
+ ).then(a => {
+ return Object.values(a).map(a => {
+ return {
+ fieldName: a.fieldName,
+ dataType: Number(a.dataType),
+ fieldValue: a.fieldValue,
+ total: Number(a.total),
+ };
+ });
+ });
}
diff --git a/src/queries/analytics/eventData/getEventDataStats.ts b/src/queries/analytics/eventData/getEventDataStats.ts
index cf77ff89a..b940e9c44 100644
--- a/src/queries/analytics/eventData/getEventDataStats.ts
+++ b/src/queries/analytics/eventData/getEventDataStats.ts
@@ -42,7 +42,10 @@ async function relationalQuery(websiteId: string, filters: QueryFilters) {
);
}
-async function clickhouseQuery(websiteId: string, filters: QueryFilters) {
+async function clickhouseQuery(
+ websiteId: string,
+ filters: QueryFilters,
+): Promise<{ events: number; fields: number; records: number }> {
const { rawQuery, parseFilters } = clickhouse;
const { filterQuery, params } = await parseFilters(websiteId, filters);
@@ -59,11 +62,19 @@ async function clickhouseQuery(websiteId: string, filters: QueryFilters) {
count(*) as "total"
from event_data
where website_id = {websiteId:UUID}
- and created_at between {startDate:DateTime} and {endDate:DateTime}
+ and created_at between {startDate:DateTime64} and {endDate:DateTime64}
${filterQuery}
group by event_id, event_key
) as t
`,
params,
- );
+ ).then(a => {
+ return Object.values(a).map(a => {
+ return {
+ events: Number(a.events),
+ fields: Number(a.fields),
+ records: Number(a.records),
+ };
+ });
+ });
}
diff --git a/src/queries/analytics/events/getEventMetrics.ts b/src/queries/analytics/events/getEventMetrics.ts
index 778cfee1b..53638fbde 100644
--- a/src/queries/analytics/events/getEventMetrics.ts
+++ b/src/queries/analytics/events/getEventMetrics.ts
@@ -40,7 +40,10 @@ async function relationalQuery(websiteId: string, filters: QueryFilters) {
);
}
-async function clickhouseQuery(websiteId: string, filters: QueryFilters) {
+async function clickhouseQuery(
+ websiteId: string,
+ filters: QueryFilters,
+): Promise<{ x: string; t: string; y: number }[]> {
const { timezone = 'UTC', unit = 'day' } = filters;
const { rawQuery, getDateQuery, parseFilters } = clickhouse;
const { filterQuery, params } = await parseFilters(websiteId, {
@@ -56,12 +59,16 @@ async function clickhouseQuery(websiteId: string, filters: QueryFilters) {
count(*) y
from website_event
where website_id = {websiteId:UUID}
- and created_at between {startDate:DateTime} and {endDate:DateTime}
+ and created_at between {startDate:DateTime64} and {endDate:DateTime64}
and event_type = {eventType:UInt32}
${filterQuery}
group by x, t
order by t
`,
params,
- );
+ ).then(a => {
+ return Object.values(a).map(a => {
+ return { x: a.x, t: a.t, y: Number(a.y) };
+ });
+ });
}
diff --git a/src/queries/analytics/events/getEvents.ts b/src/queries/analytics/events/getEvents.ts
index 17528d660..fe074ec28 100644
--- a/src/queries/analytics/events/getEvents.ts
+++ b/src/queries/analytics/events/getEvents.ts
@@ -37,7 +37,7 @@ function clickhouseQuery(websiteId: string, startDate: Date, eventType: number)
event_name as eventName
from website_event
where website_id = {websiteId:UUID}
- and created_at >= {startDate:DateTime}
+ and created_at >= {startDate:DateTime64}
and event_type = {eventType:UInt32}
`,
{
diff --git a/src/queries/analytics/getActiveVisitors.ts b/src/queries/analytics/getActiveVisitors.ts
index db3583eb6..962bea350 100644
--- a/src/queries/analytics/getActiveVisitors.ts
+++ b/src/queries/analytics/getActiveVisitors.ts
@@ -24,17 +24,23 @@ async function relationalQuery(websiteId: string) {
);
}
-async function clickhouseQuery(websiteId: string) {
+async function clickhouseQuery(websiteId: string): Promise<{ x: number }> {
const { rawQuery } = clickhouse;
- return rawQuery(
+ const result = rawQuery(
`
select
count(distinct session_id) x
from website_event
where website_id = {websiteId:UUID}
- and created_at >= {startAt:DateTime}
+ and created_at >= {startAt:DateTime64}
`,
{ websiteId, startAt: subMinutes(new Date(), 5) },
- );
+ ).then(a => {
+ return Object.values(a).map(a => {
+ return { x: Number(a.x) };
+ });
+ });
+
+ return result[0] ?? null;
}
diff --git a/src/queries/analytics/getWebsiteDateRange.ts b/src/queries/analytics/getWebsiteDateRange.ts
index 4fb24733a..a4daaafc7 100644
--- a/src/queries/analytics/getWebsiteDateRange.ts
+++ b/src/queries/analytics/getWebsiteDateRange.ts
@@ -40,7 +40,7 @@ async function clickhouseQuery(websiteId: string) {
max(created_at) as maxdate
from website_event
where website_id = {websiteId:UUID}
- and created_at >= {startDate:DateTime}
+ and created_at >= {startDate:DateTime64}
`,
params,
);
diff --git a/src/queries/analytics/getWebsiteStats.ts b/src/queries/analytics/getWebsiteStats.ts
index 165195119..654a09a9b 100644
--- a/src/queries/analytics/getWebsiteStats.ts
+++ b/src/queries/analytics/getWebsiteStats.ts
@@ -46,7 +46,10 @@ async function relationalQuery(websiteId: string, filters: QueryFilters) {
);
}
-async function clickhouseQuery(websiteId: string, filters: QueryFilters) {
+async function clickhouseQuery(
+ websiteId: string,
+ filters: QueryFilters,
+): Promise<{ pageviews: number; uniques: number; bounces: number; totaltime: number }[]> {
const { rawQuery, getDateQuery, parseFilters } = clickhouse;
const { filterQuery, params } = await parseFilters(websiteId, {
...filters,
@@ -69,12 +72,21 @@ async function clickhouseQuery(websiteId: string, filters: QueryFilters) {
max(created_at) max_time
from website_event
where website_id = {websiteId:UUID}
- and created_at between {startDate:DateTime} and {endDate:DateTime}
+ and created_at between {startDate:DateTime64} and {endDate:DateTime64}
and event_type = {eventType:UInt32}
${filterQuery}
group by session_id, time_series
) as t;
`,
params,
- );
+ ).then(a => {
+ return Object.values(a).map(a => {
+ return {
+ pageviews: Number(a.pageviews),
+ uniques: Number(a.uniques),
+ bounces: Number(a.bounces),
+ totaltime: Number(a.totaltime),
+ };
+ });
+ });
}
diff --git a/src/queries/analytics/pageviews/getPageviewMetrics.ts b/src/queries/analytics/pageviews/getPageviewMetrics.ts
index 3cf6c122d..4e7e9da22 100644
--- a/src/queries/analytics/pageviews/getPageviewMetrics.ts
+++ b/src/queries/analytics/pageviews/getPageviewMetrics.ts
@@ -48,7 +48,11 @@ async function relationalQuery(websiteId: string, column: string, filters: Query
);
}
-async function clickhouseQuery(websiteId: string, column: string, filters: QueryFilters) {
+async function clickhouseQuery(
+ websiteId: string,
+ column: string,
+ filters: QueryFilters,
+): Promise<{ x: string; y: number }[]> {
const { rawQuery, parseFilters } = clickhouse;
const { filterQuery, params } = await parseFilters(websiteId, {
...filters,
@@ -65,7 +69,7 @@ async function clickhouseQuery(websiteId: string, column: string, filters: Query
select ${column} x, count(*) y
from website_event
where website_id = {websiteId:UUID}
- and created_at between {startDate:DateTime} and {endDate:DateTime}
+ and created_at between {startDate:DateTime64} and {endDate:DateTime64}
and event_type = {eventType:UInt32}
${excludeDomain}
${filterQuery}
@@ -74,5 +78,9 @@ async function clickhouseQuery(websiteId: string, column: string, filters: Query
limit 100
`,
params,
- );
+ ).then(a => {
+ return Object.values(a).map(a => {
+ return { x: a.x, y: Number(a.y) };
+ });
+ });
}
diff --git a/src/queries/analytics/pageviews/getPageviewStats.ts b/src/queries/analytics/pageviews/getPageviewStats.ts
index d6a980ef7..b221e0104 100644
--- a/src/queries/analytics/pageviews/getPageviewStats.ts
+++ b/src/queries/analytics/pageviews/getPageviewStats.ts
@@ -36,7 +36,10 @@ async function relationalQuery(websiteId: string, filters: QueryFilters) {
);
}
-async function clickhouseQuery(websiteId: string, filters: QueryFilters) {
+async function clickhouseQuery(
+ websiteId: string,
+ filters: QueryFilters,
+): Promise<{ x: string; y: number }[]> {
const { timezone = 'UTC', unit = 'day' } = filters;
const { parseFilters, rawQuery, getDateStringQuery, getDateQuery } = clickhouse;
const { filterQuery, params } = await parseFilters(websiteId, {
@@ -55,7 +58,7 @@ async function clickhouseQuery(websiteId: string, filters: QueryFilters) {
count(*) as y
from website_event
where website_id = {websiteId:UUID}
- and created_at between {startDate:DateTime} and {endDate:DateTime}
+ and created_at between {startDate:DateTime64} and {endDate:DateTime64}
and event_type = {eventType:UInt32}
${filterQuery}
group by t
@@ -63,5 +66,9 @@ async function clickhouseQuery(websiteId: string, filters: QueryFilters) {
order by t
`,
params,
- );
+ ).then(a => {
+ return Object.values(a).map(a => {
+ return { x: a.x, y: Number(a.y) };
+ });
+ });
}
diff --git a/src/queries/analytics/reports/getFunnel.ts b/src/queries/analytics/reports/getFunnel.ts
index 1bbbc8782..8dbd8d45c 100644
--- a/src/queries/analytics/reports/getFunnel.ts
+++ b/src/queries/analytics/reports/getFunnel.ts
@@ -172,7 +172,7 @@ async function clickhouseQuery(
);
}
- return rawQuery<{ level: number; count: number }[]>(
+ return rawQuery(
`
WITH level0 AS (
select distinct session_id, url_path, referrer_path, created_at
@@ -201,7 +201,7 @@ async function clickhouseQuery(
).then(results => {
return urls.map((a, i) => ({
x: a,
- y: results[i]?.count || 0,
+ y: Number(results[i]?.count) || 0,
z: (1 - Number(results[i]?.count) / Number(results[i - 1]?.count)) * 100 || 0, // drop off
}));
});
diff --git a/src/queries/analytics/reports/getInsights.ts b/src/queries/analytics/reports/getInsights.ts
index fa54488b6..c5fac48b3 100644
--- a/src/queries/analytics/reports/getInsights.ts
+++ b/src/queries/analytics/reports/getInsights.ts
@@ -75,7 +75,7 @@ async function clickhouseQuery(
${parseFields(fields)}
from website_event
where website_id = {websiteId:UUID}
- and created_at between {startDate:DateTime} and {endDate:DateTime}
+ and created_at between {startDate:DateTime64} and {endDate:DateTime64}
and event_type = {eventType:UInt32}
${filterQuery}
${parseGroupBy(fields)}
diff --git a/src/queries/analytics/reports/getRetention.ts b/src/queries/analytics/reports/getRetention.ts
index 3c384b6e5..1ec65fcb0 100644
--- a/src/queries/analytics/reports/getRetention.ts
+++ b/src/queries/analytics/reports/getRetention.ts
@@ -172,5 +172,15 @@ async function clickhouseQuery(
startDate,
endDate,
},
- );
+ ).then(a => {
+ return Object.values(a).map(a => {
+ return {
+ date: a.date,
+ day: Number(a.day),
+ visitors: Number(a.visitors),
+ returnVisitors: Number(a.returnVisitors),
+ percentage: Number(a.percentage),
+ };
+ });
+ });
}
diff --git a/src/queries/analytics/sessions/getSessionMetrics.ts b/src/queries/analytics/sessions/getSessionMetrics.ts
index 43d9ef5a4..e97501d72 100644
--- a/src/queries/analytics/sessions/getSessionMetrics.ts
+++ b/src/queries/analytics/sessions/getSessionMetrics.ts
@@ -47,7 +47,11 @@ async function relationalQuery(websiteId: string, column: string, filters: Query
);
}
-async function clickhouseQuery(websiteId: string, column: string, filters: QueryFilters) {
+async function clickhouseQuery(
+ websiteId: string,
+ column: string,
+ filters: QueryFilters,
+): Promise<{ x: string; y: number }[]> {
const { parseFilters, rawQuery } = clickhouse;
const { filterQuery, params } = await parseFilters(websiteId, {
...filters,
@@ -63,7 +67,6 @@ async function clickhouseQuery(websiteId: string, column: string, filters: Query
${includeCountry ? ', country' : ''}
from website_event
where website_id = {websiteId:UUID}
- and created_at between {startDate:DateTime} and {endDate:DateTime}
and event_type = {eventType:UInt32}
${filterQuery}
group by x
@@ -72,5 +75,9 @@ async function clickhouseQuery(websiteId: string, column: string, filters: Query
limit 100
`,
params,
- );
+ ).then(a => {
+ return Object.values(a).map(a => {
+ return { x: a.x, y: Number(a.y) };
+ });
+ });
}
diff --git a/src/queries/analytics/sessions/getSessionStats.ts b/src/queries/analytics/sessions/getSessionStats.ts
index 9ed01a594..c977187df 100644
--- a/src/queries/analytics/sessions/getSessionStats.ts
+++ b/src/queries/analytics/sessions/getSessionStats.ts
@@ -36,7 +36,10 @@ async function relationalQuery(websiteId: string, filters: QueryFilters) {
);
}
-async function clickhouseQuery(websiteId: string, filters: QueryFilters) {
+async function clickhouseQuery(
+ websiteId: string,
+ filters: QueryFilters,
+): Promise<{ x: string; y: number }[]> {
const { timezone = 'UTC', unit = 'day' } = filters;
const { parseFilters, rawQuery, getDateStringQuery, getDateQuery } = clickhouse;
const { filterQuery, params } = await parseFilters(websiteId, {
@@ -55,7 +58,7 @@ async function clickhouseQuery(websiteId: string, filters: QueryFilters) {
count(distinct session_id) as y
from website_event
where website_id = {websiteId:UUID}
- and created_at between {startDate:DateTime} and {endDate:DateTime}
+ and created_at between {startDate:DateTime64} and {endDate:DateTime64}
and event_type = {eventType:UInt32}
${filterQuery}
group by t
@@ -63,5 +66,9 @@ async function clickhouseQuery(websiteId: string, filters: QueryFilters) {
order by t
`,
params,
- );
+ ).then(a => {
+ return Object.values(a).map(a => {
+ return { x: a.x, y: Number(a.y) };
+ });
+ });
}
diff --git a/src/queries/analytics/sessions/getSessions.ts b/src/queries/analytics/sessions/getSessions.ts
index 6936f902e..d67edd5ef 100644
--- a/src/queries/analytics/sessions/getSessions.ts
+++ b/src/queries/analytics/sessions/getSessions.ts
@@ -42,7 +42,7 @@ async function clickhouseQuery(websiteId: string, startDate: Date) {
city
from website_event
where website_id = {websiteId:UUID}
- and created_at >= {startDate:DateTime}
+ and created_at >= {startDate:DateTime64}
`,
{
websiteId,