Umami is a privacy-first web and product analytics platform. Traffic, campaigns, behavior, conversions, and revenue in one place. No cookies, no surveillance, self-hosted or in the cloud.
TypeScript analyticsaudience-segmentationchartscohort-analysisgoogle-analyticsproduct-analyticsstatisticsuser-journeyweb-analytics
update CH rawquery and type
Francis Caoe2bb2defd447
diff --git a/src/lib/clickhouse.ts b/src/lib/clickhouse.tsindex 186a085d1..c371e7f97 100644--- a/src/lib/clickhouse.ts+++ b/src/lib/clickhouse.ts@@ -1,4 +1,4 @@-import { ClickHouse } from 'clickhouse';+import { ClickHouseClient, createClient } from '@clickhouse/client';import dateFormat from 'dateformat';import debug from 'debug';import { CLICKHOUSE } from 'lib/db';@@ -17,7 +17,7 @@ export const CLICKHOUSE_DATE_FORMATS = {const log = debug('umami:clickhouse');-let clickhouse: ClickHouse;+let clickhouse: ClickHouseClient;const enabled = Boolean(process.env.CLICKHOUSE_URL);function getClient() {@@ -25,18 +25,19 @@ function getClient() {hostname,port,pathname,+ // protocol,username = 'default',password,} = new URL(process.env.CLICKHOUSE_URL);- const client = new ClickHouse({- url: hostname,- port: Number(port),- format: 'json',- config: {- database: pathname.replace('/', ''),- },- basicAuth: password ? { username, password } : null,+ // const formattedProtocol =+ // protocol.toLowerCase() === 'clickhouse:' || protocol === 'https:' ? 'https:' : 'http:';++ const client = createClient({+ host: `http://${hostname}:${port}`,+ database: pathname.replace('/', ''),+ username: username,+ password,});if (process.env.NODE_ENV !== 'production') {@@ -118,7 +119,7 @@ async function parseFilters(websiteId: string, filters: QueryFilters = {}, optio};}-async function rawQuery<T>(query: string, params: object = {}): Promise<T> {+async function rawQuery(query: string, params: Record<string, unknown> = {}): Promise<unknown> {if (process.env.LOG_QUERY) {log('QUERY:\n', query);log('PARAMETERS:\n', params);@@ -126,7 +127,15 @@ async function rawQuery<T>(query: string, params: object = {}): Promise<T> {await connect();- return clickhouse.query(query, { params }).toPromise() as Promise<T>;+ const resultSet = await clickhouse.query({+ query: query,+ query_params: params,+ format: 'JSONEachRow',+ });++ const data = await resultSet.json();++ return data;}async function findUnique(data) {diff --git a/src/queries/analytics/eventData/getEventDataEvents.ts b/src/queries/analytics/eventData/getEventDataEvents.tsindex 2c8cb0e0d..29158503d 100644--- a/src/queries/analytics/eventData/getEventDataEvents.ts+++ b/src/queries/analytics/eventData/getEventDataEvents.ts@@ -59,7 +59,10 @@ async function relationalQuery(websiteId: string, filters: QueryFilters) {);}-async function clickhouseQuery(websiteId: string, filters: QueryFilters) {+async function clickhouseQuery(+ websiteId: string,+ filters: QueryFilters,+): Promise<{ eventName: string; fieldName: string; dataType: number; total: number }[]> {const { rawQuery, parseFilters } = clickhouse;const { event } = filters;const { params } = await parseFilters(websiteId, filters);@@ -75,14 +78,23 @@ async function clickhouseQuery(websiteId: string, filters: QueryFilters) {count(*) as totalfrom event_datawhere website_id = {websiteId:UUID}- and created_at between {startDate:DateTime} and {endDate:DateTime}+ and created_at between {startDate:DateTime64} and {endDate:DateTime64}and event_name = {event:String}group by event_key, data_type, string_value, event_nameorder by 1 asc, 2 asc, 3 asc, 4 desclimit 100`,params,- );+ ).then(a => {+ return Object.values(a).map(a => {+ return {+ eventName: a.eventName,+ fieldName: a.fieldName,+ dataType: Number(a.dataType),+ total: Number(a.total),+ };+ });+ });}return rawQuery(@@ -94,11 +106,20 @@ async function clickhouseQuery(websiteId: string, filters: QueryFilters) {count(*) as totalfrom event_datawhere website_id = {websiteId:UUID}- and created_at between {startDate:DateTime} and {endDate:DateTime}+ and created_at between {startDate:DateTime64} and {endDate:DateTime64}group by event_key, data_type, event_nameorder by 1 asc, 2 asclimit 100`,params,- );+ ).then(a => {+ return Object.values(a).map(a => {+ return {+ eventName: a.eventName,+ fieldName: a.fieldName,+ dataType: Number(a.dataType),+ total: Number(a.total),+ };+ });+ });}diff --git a/src/queries/analytics/eventData/getEventDataFields.ts b/src/queries/analytics/eventData/getEventDataFields.tsindex ac32b188f..df5a8874f 100644--- a/src/queries/analytics/eventData/getEventDataFields.ts+++ b/src/queries/analytics/eventData/getEventDataFields.ts@@ -37,7 +37,10 @@ async function relationalQuery(websiteId: string, filters: QueryFilters & { fiel);}-async function clickhouseQuery(websiteId: string, filters: QueryFilters & { field?: string }) {+async function clickhouseQuery(+ websiteId: string,+ filters: QueryFilters & { field?: string },+): Promise<{ fieldName: string; dataType: number; fieldValue: string; total: number }[]> {const { rawQuery, parseFilters } = clickhouse;const { filterQuery, params } = await parseFilters(websiteId, filters, {columns: { field: 'event_key' },@@ -52,12 +55,21 @@ async function clickhouseQuery(websiteId: string, filters: QueryFilters & { fielcount(*) as totalfrom event_datawhere website_id = {websiteId:UUID}- and created_at between {startDate:DateTime} and {endDate:DateTime}+ and created_at between {startDate:DateTime64} and {endDate:DateTime64}${filterQuery}group by event_key, data_type, string_valueorder by 3 desc, 2 desc, 1 asclimit 100`,params,- );+ ).then(a => {+ return Object.values(a).map(a => {+ return {+ fieldName: a.fieldName,+ dataType: Number(a.dataType),+ fieldValue: a.fieldValue,+ total: Number(a.total),+ };+ });+ });}diff --git a/src/queries/analytics/eventData/getEventDataStats.ts b/src/queries/analytics/eventData/getEventDataStats.tsindex cf77ff89a..b940e9c44 100644--- a/src/queries/analytics/eventData/getEventDataStats.ts+++ b/src/queries/analytics/eventData/getEventDataStats.ts@@ -42,7 +42,10 @@ async function relationalQuery(websiteId: string, filters: QueryFilters) {);}-async function clickhouseQuery(websiteId: string, filters: QueryFilters) {+async function clickhouseQuery(+ websiteId: string,+ filters: QueryFilters,+): Promise<{ events: number; fields: number; records: number }> {const { rawQuery, parseFilters } = clickhouse;const { filterQuery, params } = await parseFilters(websiteId, filters);@@ -59,11 +62,19 @@ async function clickhouseQuery(websiteId: string, filters: QueryFilters) {count(*) as "total"from event_datawhere website_id = {websiteId:UUID}- and created_at between {startDate:DateTime} and {endDate:DateTime}+ and created_at between {startDate:DateTime64} and {endDate:DateTime64}${filterQuery}group by event_id, event_key) as t`,params,- );+ ).then(a => {+ return Object.values(a).map(a => {+ return {+ events: Number(a.events),+ fields: Number(a.fields),+ records: Number(a.records),+ };+ });+ });}diff --git a/src/queries/analytics/events/getEventMetrics.ts b/src/queries/analytics/events/getEventMetrics.tsindex 778cfee1b..53638fbde 100644--- a/src/queries/analytics/events/getEventMetrics.ts+++ b/src/queries/analytics/events/getEventMetrics.ts@@ -40,7 +40,10 @@ async function relationalQuery(websiteId: string, filters: QueryFilters) {);}-async function clickhouseQuery(websiteId: string, filters: QueryFilters) {+async function clickhouseQuery(+ websiteId: string,+ filters: QueryFilters,+): Promise<{ x: string; t: string; y: number }[]> {const { timezone = 'UTC', unit = 'day' } = filters;const { rawQuery, getDateQuery, parseFilters } = clickhouse;const { filterQuery, params } = await parseFilters(websiteId, {@@ -56,12 +59,16 @@ async function clickhouseQuery(websiteId: string, filters: QueryFilters) {count(*) yfrom website_eventwhere website_id = {websiteId:UUID}- and created_at between {startDate:DateTime} and {endDate:DateTime}+ and created_at between {startDate:DateTime64} and {endDate:DateTime64}and event_type = {eventType:UInt32}${filterQuery}group by x, torder by t`,params,- );+ ).then(a => {+ return Object.values(a).map(a => {+ return { x: a.x, t: a.t, y: Number(a.y) };+ });+ });}diff --git a/src/queries/analytics/events/getEvents.ts b/src/queries/analytics/events/getEvents.tsindex 17528d660..fe074ec28 100644--- a/src/queries/analytics/events/getEvents.ts+++ b/src/queries/analytics/events/getEvents.ts@@ -37,7 +37,7 @@ function clickhouseQuery(websiteId: string, startDate: Date, eventType: number)event_name as eventNamefrom website_eventwhere website_id = {websiteId:UUID}- and created_at >= {startDate:DateTime}+ and created_at >= {startDate:DateTime64}and event_type = {eventType:UInt32}`,{diff --git a/src/queries/analytics/getActiveVisitors.ts b/src/queries/analytics/getActiveVisitors.tsindex db3583eb6..962bea350 100644--- a/src/queries/analytics/getActiveVisitors.ts+++ b/src/queries/analytics/getActiveVisitors.ts@@ -24,17 +24,23 @@ async function relationalQuery(websiteId: string) {);}-async function clickhouseQuery(websiteId: string) {+async function clickhouseQuery(websiteId: string): Promise<{ x: number }> {const { rawQuery } = clickhouse;- return rawQuery(+ const result = rawQuery(`selectcount(distinct session_id) xfrom website_eventwhere website_id = {websiteId:UUID}- and created_at >= {startAt:DateTime}+ and created_at >= {startAt:DateTime64}`,{ websiteId, startAt: subMinutes(new Date(), 5) },- );+ ).then(a => {+ return Object.values(a).map(a => {+ return { x: Number(a.x) };+ });+ });++ return result[0] ?? null;}diff --git a/src/queries/analytics/getWebsiteDateRange.ts b/src/queries/analytics/getWebsiteDateRange.tsindex 4fb24733a..a4daaafc7 100644--- a/src/queries/analytics/getWebsiteDateRange.ts+++ b/src/queries/analytics/getWebsiteDateRange.ts@@ -40,7 +40,7 @@ async function clickhouseQuery(websiteId: string) {max(created_at) as maxdatefrom website_eventwhere website_id = {websiteId:UUID}- and created_at >= {startDate:DateTime}+ and created_at >= {startDate:DateTime64}`,params,);diff --git a/src/queries/analytics/getWebsiteStats.ts b/src/queries/analytics/getWebsiteStats.tsindex 165195119..654a09a9b 100644--- a/src/queries/analytics/getWebsiteStats.ts+++ b/src/queries/analytics/getWebsiteStats.ts@@ -46,7 +46,10 @@ async function relationalQuery(websiteId: string, filters: QueryFilters) {);}-async function clickhouseQuery(websiteId: string, filters: QueryFilters) {+async function clickhouseQuery(+ websiteId: string,+ filters: QueryFilters,+): Promise<{ pageviews: number; uniques: number; bounces: number; totaltime: number }[]> {const { rawQuery, getDateQuery, parseFilters } = clickhouse;const { filterQuery, params } = await parseFilters(websiteId, {...filters,@@ -69,12 +72,21 @@ async function clickhouseQuery(websiteId: string, filters: QueryFilters) {max(created_at) max_timefrom website_eventwhere website_id = {websiteId:UUID}- and created_at between {startDate:DateTime} and {endDate:DateTime}+ and created_at between {startDate:DateTime64} and {endDate:DateTime64}and event_type = {eventType:UInt32}${filterQuery}group by session_id, time_series) as t;`,params,- );+ ).then(a => {+ return Object.values(a).map(a => {+ return {+ pageviews: Number(a.pageviews),+ uniques: Number(a.uniques),+ bounces: Number(a.bounces),+ totaltime: Number(a.totaltime),+ };+ });+ });}diff --git a/src/queries/analytics/pageviews/getPageviewMetrics.ts b/src/queries/analytics/pageviews/getPageviewMetrics.tsindex 3cf6c122d..4e7e9da22 100644--- a/src/queries/analytics/pageviews/getPageviewMetrics.ts+++ b/src/queries/analytics/pageviews/getPageviewMetrics.ts@@ -48,7 +48,11 @@ async function relationalQuery(websiteId: string, column: string, filters: Query);}-async function clickhouseQuery(websiteId: string, column: string, filters: QueryFilters) {+async function clickhouseQuery(+ websiteId: string,+ column: string,+ filters: QueryFilters,+): Promise<{ x: string; y: number }[]> {const { rawQuery, parseFilters } = clickhouse;const { filterQuery, params } = await parseFilters(websiteId, {...filters,@@ -65,7 +69,7 @@ async function clickhouseQuery(websiteId: string, column: string, filters: Queryselect ${column} x, count(*) yfrom website_eventwhere website_id = {websiteId:UUID}- and created_at between {startDate:DateTime} and {endDate:DateTime}+ and created_at between {startDate:DateTime64} and {endDate:DateTime64}and event_type = {eventType:UInt32}${excludeDomain}${filterQuery}@@ -74,5 +78,9 @@ async function clickhouseQuery(websiteId: string, column: string, filters: Querylimit 100`,params,- );+ ).then(a => {+ return Object.values(a).map(a => {+ return { x: a.x, y: Number(a.y) };+ });+ });}diff --git a/src/queries/analytics/pageviews/getPageviewStats.ts b/src/queries/analytics/pageviews/getPageviewStats.tsindex d6a980ef7..b221e0104 100644--- a/src/queries/analytics/pageviews/getPageviewStats.ts+++ b/src/queries/analytics/pageviews/getPageviewStats.ts@@ -36,7 +36,10 @@ async function relationalQuery(websiteId: string, filters: QueryFilters) {);}-async function clickhouseQuery(websiteId: string, filters: QueryFilters) {+async function clickhouseQuery(+ websiteId: string,+ filters: QueryFilters,+): Promise<{ x: string; y: number }[]> {const { timezone = 'UTC', unit = 'day' } = filters;const { parseFilters, rawQuery, getDateStringQuery, getDateQuery } = clickhouse;const { filterQuery, params } = await parseFilters(websiteId, {@@ -55,7 +58,7 @@ async function clickhouseQuery(websiteId: string, filters: QueryFilters) {count(*) as yfrom website_eventwhere website_id = {websiteId:UUID}- and created_at between {startDate:DateTime} and {endDate:DateTime}+ and created_at between {startDate:DateTime64} and {endDate:DateTime64}and event_type = {eventType:UInt32}${filterQuery}group by t@@ -63,5 +66,9 @@ async function clickhouseQuery(websiteId: string, filters: QueryFilters) {order by t`,params,- );+ ).then(a => {+ return Object.values(a).map(a => {+ return { x: a.x, y: Number(a.y) };+ });+ });}diff --git a/src/queries/analytics/reports/getFunnel.ts b/src/queries/analytics/reports/getFunnel.tsindex 1bbbc8782..8dbd8d45c 100644--- a/src/queries/analytics/reports/getFunnel.ts+++ b/src/queries/analytics/reports/getFunnel.ts@@ -172,7 +172,7 @@ async function clickhouseQuery();}- return rawQuery<{ level: number; count: number }[]>(+ return rawQuery(`WITH level0 AS (select distinct session_id, url_path, referrer_path, created_at@@ -201,7 +201,7 @@ async function clickhouseQuery().then(results => {return urls.map((a, i) => ({x: a,- y: results[i]?.count || 0,+ y: Number(results[i]?.count) || 0,z: (1 - Number(results[i]?.count) / Number(results[i - 1]?.count)) * 100 || 0, // drop off}));});diff --git a/src/queries/analytics/reports/getInsights.ts b/src/queries/analytics/reports/getInsights.tsindex fa54488b6..c5fac48b3 100644--- a/src/queries/analytics/reports/getInsights.ts+++ b/src/queries/analytics/reports/getInsights.ts@@ -75,7 +75,7 @@ async function clickhouseQuery(${parseFields(fields)}from website_eventwhere website_id = {websiteId:UUID}- and created_at between {startDate:DateTime} and {endDate:DateTime}+ and created_at between {startDate:DateTime64} and {endDate:DateTime64}and event_type = {eventType:UInt32}${filterQuery}${parseGroupBy(fields)}diff --git a/src/queries/analytics/reports/getRetention.ts b/src/queries/analytics/reports/getRetention.tsindex 3c384b6e5..1ec65fcb0 100644--- a/src/queries/analytics/reports/getRetention.ts+++ b/src/queries/analytics/reports/getRetention.ts@@ -172,5 +172,15 @@ async function clickhouseQuery(startDate,endDate,},- );+ ).then(a => {+ return Object.values(a).map(a => {+ return {+ date: a.date,+ day: Number(a.day),+ visitors: Number(a.visitors),+ returnVisitors: Number(a.returnVisitors),+ percentage: Number(a.percentage),+ };+ });+ });}diff --git a/src/queries/analytics/sessions/getSessionMetrics.ts b/src/queries/analytics/sessions/getSessionMetrics.tsindex 43d9ef5a4..e97501d72 100644--- a/src/queries/analytics/sessions/getSessionMetrics.ts+++ b/src/queries/analytics/sessions/getSessionMetrics.ts@@ -47,7 +47,11 @@ async function relationalQuery(websiteId: string, column: string, filters: Query);}-async function clickhouseQuery(websiteId: string, column: string, filters: QueryFilters) {+async function clickhouseQuery(+ websiteId: string,+ column: string,+ filters: QueryFilters,+): Promise<{ x: string; y: number }[]> {const { parseFilters, rawQuery } = clickhouse;const { filterQuery, params } = await parseFilters(websiteId, {...filters,@@ -63,7 +67,6 @@ async function clickhouseQuery(websiteId: string, column: string, filters: Query${includeCountry ? ', country' : ''}from website_eventwhere website_id = {websiteId:UUID}- and created_at between {startDate:DateTime} and {endDate:DateTime}and event_type = {eventType:UInt32}${filterQuery}group by x@@ -72,5 +75,9 @@ async function clickhouseQuery(websiteId: string, column: string, filters: Querylimit 100`,params,- );+ ).then(a => {+ return Object.values(a).map(a => {+ return { x: a.x, y: Number(a.y) };+ });+ });}diff --git a/src/queries/analytics/sessions/getSessionStats.ts b/src/queries/analytics/sessions/getSessionStats.tsindex 9ed01a594..c977187df 100644--- a/src/queries/analytics/sessions/getSessionStats.ts+++ b/src/queries/analytics/sessions/getSessionStats.ts@@ -36,7 +36,10 @@ async function relationalQuery(websiteId: string, filters: QueryFilters) {);}-async function clickhouseQuery(websiteId: string, filters: QueryFilters) {+async function clickhouseQuery(+ websiteId: string,+ filters: QueryFilters,+): Promise<{ x: string; y: number }[]> {const { timezone = 'UTC', unit = 'day' } = filters;const { parseFilters, rawQuery, getDateStringQuery, getDateQuery } = clickhouse;const { filterQuery, params } = await parseFilters(websiteId, {@@ -55,7 +58,7 @@ async function clickhouseQuery(websiteId: string, filters: QueryFilters) {count(distinct session_id) as yfrom website_eventwhere website_id = {websiteId:UUID}- and created_at between {startDate:DateTime} and {endDate:DateTime}+ and created_at between {startDate:DateTime64} and {endDate:DateTime64}and event_type = {eventType:UInt32}${filterQuery}group by t@@ -63,5 +66,9 @@ async function clickhouseQuery(websiteId: string, filters: QueryFilters) {order by t`,params,- );+ ).then(a => {+ return Object.values(a).map(a => {+ return { x: a.x, y: Number(a.y) };+ });+ });}diff --git a/src/queries/analytics/sessions/getSessions.ts b/src/queries/analytics/sessions/getSessions.tsindex 6936f902e..d67edd5ef 100644--- a/src/queries/analytics/sessions/getSessions.ts+++ b/src/queries/analytics/sessions/getSessions.ts@@ -42,7 +42,7 @@ async function clickhouseQuery(websiteId: string, startDate: Date) {cityfrom website_eventwhere website_id = {websiteId:UUID}- and created_at >= {startDate:DateTime}+ and created_at >= {startDate:DateTime64}`,{websiteId,