Umami is a privacy-first web and product analytics platform. Traffic, campaigns, behavior, conversions, and revenue in one place. No cookies, no surveillance, self-hosted or in the cloud.
TypeScript analyticsaudience-segmentationchartscohort-analysisgoogle-analyticsproduct-analyticsstatisticsuser-journeyweb-analytics
clickhouse inserts
Brian Cao65910c7348ff
diff --git a/lib/db.js b/lib/db.jsindex 9409178ab..93a54badb 100644--- a/lib/db.js+++ b/lib/db.js@@ -72,19 +72,6 @@ function initializeClickhouse() {database,},});-- // return new ClickHouse({- // url: 'http://164.92.95.2',- // port: 8123,- // basicAuth: {- // username: 'default',- // password: 'shhhthisissupersecret!',- // },- // format: 'json',- // config: {- // database: 'umami_dev',- // },- // });}const prisma = initializePrisma(options);@@ -243,7 +230,7 @@ export function getFilterQuery(table, filters = {}, params = []) {return query.join('\n');}-export function parseFilters(table, filters = {}, params = []) {+export function parseFilters(table, filters = {}, params = [], sessionKey = 'session_id') {const { domain, url, event_url, referrer, os, browser, device, country, event_type } = filters;const pageviewFilters = { domain, url, referrer };@@ -257,7 +244,7 @@ export function parseFilters(table, filters = {}, params = []) {event: { event_type },joinSession:os || browser || device || country- ? `inner join session on ${table}.session_id = session.session_id`+ ? `inner join session on ${table}.${sessionKey} = session.${sessionKey}`: '',pageviewQuery: getFilterQuery('pageview', pageviewFilters, params),sessionQuery: getFilterQuery('session', sessionFilters, params),diff --git a/lib/session.js b/lib/session.jsindex 15a349fdd..edb8aeeb3 100644--- a/lib/session.js+++ b/lib/session.js@@ -37,6 +37,8 @@ export async function getSession(req) {let session = await getSessionByUuid(session_uuid);+ session = Array.isArray(session) && session[0] ? session[0] : null;+if (!session) {try {session = await createSession(website_id, {@@ -50,6 +52,8 @@ export async function getSession(req) {device,});+ console.log(session);+if (!session) {return null;}@@ -65,5 +69,6 @@ export async function getSession(req) {return {website_id,session_id,+ session_uuid,};}diff --git a/pages/api/collect.js b/pages/api/collect.jsindex acd031a98..4ff533b98 100644--- a/pages/api/collect.js+++ b/pages/api/collect.js@@ -60,7 +60,7 @@ export default async (req, res) => {await useSession(req, res);const {- session: { website_id, session_id },+ session: { website_id, session_id, session_uuid },} = req;const { type, payload } = getJsonBody(req);@@ -72,9 +72,9 @@ export default async (req, res) => {}if (type === 'pageview') {- await savePageView(website_id, session_id, url, referrer);+ await savePageView(website_id, { session_id, session_uuid, url, referrer });} else if (type === 'event') {- await saveEvent(website_id, session_id, url, event_type, event_value);+ await saveEvent(website_id, { session_id, session_uuid, url, event_type, event_value });} else {return badRequest(res);}diff --git a/pages/api/website/[id]/pageviews.js b/pages/api/website/[id]/pageviews.jsindex 78145e8b3..3bd572d93 100644--- a/pages/api/website/[id]/pageviews.js+++ b/pages/api/website/[id]/pageviews.js@@ -34,7 +34,7 @@ export default async (req, res) => {device,country,}),- getPageviewStats(websiteId, startDate, endDate, tz, unit, 'distinct pageview.session_id', {+ getPageviewStats(websiteId, startDate, endDate, tz, unit, 'distinct pageview.', {url,os,browser,diff --git a/queries/analytics/event/saveEvent.js b/queries/analytics/event/saveEvent.jsindex 173791359..abb32cbbe 100644--- a/queries/analytics/event/saveEvent.js+++ b/queries/analytics/event/saveEvent.js@@ -1,8 +1,20 @@-import { prisma, runQuery } from 'lib/db';+import { CLICKHOUSE, RELATIONAL, URL_LENGTH } from 'lib/constants';+import {+ getDateFormatClickhouse,+ prisma,+ rawQueryClickhouse,+ runAnalyticsQuery,+ runQuery,+} from 'lib/db';-import { URL_LENGTH } from 'lib/constants';+export async function saveEvent(...args) {+ return runAnalyticsQuery({+ [`${RELATIONAL}`]: () => relationalQuery(...args),+ [`${CLICKHOUSE}`]: () => clickhouseQuery(...args),+ });+}-export async function saveEvent(website_id, session_id, url, event_type, event_value) {+async function relationalQuery(website_id, { session_id, url, event_type, event_value }) {return runQuery(prisma.event.create({data: {@@ -15,3 +27,20 @@ export async function saveEvent(website_id, session_id, url, event_type, event_v}),);}++async function clickhouseQuery(website_id, { session_uuid, url, event_type, event_value }) {+ const params = [+ website_id,+ session_uuid,+ url?.substr(0, URL_LENGTH),+ event_type?.substr(0, 50),+ event_value?.substr(0, 50),+ ];++ return rawQueryClickhouse(+ `+ insert into umami_dev.event (created_at, website_id, session_uuid, url, event_type, event_value)+ values (${getDateFormatClickhouse(new Date())}, $1, $2, $3, $4, $5);`,+ params,+ );+}diff --git a/queries/analytics/pageview/getPageviewMetrics.js b/queries/analytics/pageview/getPageviewMetrics.jsindex fb2cf8a59..3b16d7038 100644--- a/queries/analytics/pageview/getPageviewMetrics.js+++ b/queries/analytics/pageview/getPageviewMetrics.js@@ -45,6 +45,7 @@ async function clickhouseQuery(website_id, start_at, end_at, field, table, filtetable,filters,params,+ 'session_uuid',);return rawQueryClickhouse(diff --git a/queries/analytics/pageview/getPageviewStats.js b/queries/analytics/pageview/getPageviewStats.jsindex 8a26280ce..980c6a864 100644--- a/queries/analytics/pageview/getPageviewStats.js+++ b/queries/analytics/pageview/getPageviewStats.js@@ -26,6 +26,7 @@ async function relationalQuery(unit = 'day',count = '*',filters = {},+ sessionKey = 'session_id',) {const params = [website_id, start_at, end_at];const { pageviewQuery, sessionQuery, joinSession } = parseFilters('pageview', filters, params);@@ -37,7 +38,7 @@ async function relationalQuery(g.y as yfrom(select ${getDateQuery('pageview.created_at', unit, timezone)} t,- count(${count}) y+ count(${count != '*' ? `${count}${sessionKey}` : count}) yfrom pageview${joinSession}where pageview.website_id=$1@@ -59,9 +60,15 @@ async function clickhouseQuery(unit = 'day',count = '*',filters = {},+ sessionKey = 'session_uuid',) {const params = [website_id];- const { pageviewQuery, sessionQuery, joinSession } = parseFilters('pageview', filters, params);+ const { pageviewQuery, sessionQuery, joinSession } = parseFilters(+ 'pageview',+ filters,+ params,+ sessionKey,+ );return rawQueryClickhouse(`@@ -71,7 +78,7 @@ async function clickhouseQuery(from(select${getDateQueryClickhouse('created_at', unit, timezone)} t,- count(${count}) y+ count(${count != '*' ? `${count}${sessionKey}` : count}) yfrom pageview${joinSession}where pageview.website_id= $1diff --git a/queries/analytics/pageview/savePageView.js b/queries/analytics/pageview/savePageView.jsindex 2061e3750..72ab35fb0 100644--- a/queries/analytics/pageview/savePageView.js+++ b/queries/analytics/pageview/savePageView.js@@ -1,8 +1,20 @@-import { prisma, runQuery } from 'lib/db';+import { CLICKHOUSE, RELATIONAL, URL_LENGTH } from 'lib/constants';+import {+ getDateFormatClickhouse,+ prisma,+ rawQueryClickhouse,+ runAnalyticsQuery,+ runQuery,+} from 'lib/db';-import { URL_LENGTH } from 'lib/constants';+export async function savePageView(...args) {+ return runAnalyticsQuery({+ [`${RELATIONAL}`]: () => relationalQuery(...args),+ [`${CLICKHOUSE}`]: () => clickhouseQuery(...args),+ });+}-export async function savePageView(website_id, session_id, url, referrer) {+async function relationalQuery(website_id, { session_id, url, referrer }) {return runQuery(prisma.pageview.create({data: {@@ -14,3 +26,19 @@ export async function savePageView(website_id, session_id, url, referrer) {}),);}++async function clickhouseQuery(website_id, { session_uuid, url, referrer }) {+ const params = [+ website_id,+ session_uuid,+ url?.substr(0, URL_LENGTH),+ referrer?.substr(0, URL_LENGTH),+ ];++ return rawQueryClickhouse(+ `+ insert into umami_dev.pageview (created_at, website_id, session_uuid, url, referrer)+ values (${getDateFormatClickhouse(new Date())}, $1, $2, $3, $4);`,+ params,+ );+}diff --git a/queries/analytics/session/createSession.js b/queries/analytics/session/createSession.jsindex f22aa2b0c..22b41ebdb 100644--- a/queries/analytics/session/createSession.js+++ b/queries/analytics/session/createSession.js@@ -1,6 +1,21 @@-import { prisma, runQuery } from 'lib/db';+import { CLICKHOUSE, RELATIONAL } from 'lib/constants';+import {+ getDateFormatClickhouse,+ prisma,+ rawQueryClickhouse,+ runAnalyticsQuery,+ runQuery,+} from 'lib/db';+import { getSessionByUuid } from 'queries';-export async function createSession(website_id, data) {+export async function createSession(...args) {+ return runAnalyticsQuery({+ [`${RELATIONAL}`]: () => relationalQuery(...args),+ [`${CLICKHOUSE}`]: () => clickhouseQuery(...args),+ });+}++async function relationalQuery(website_id, data) {return runQuery(prisma.session.create({data: {@@ -13,3 +28,28 @@ export async function createSession(website_id, data) {}),);}++async function clickhouseQuery(+ website_id,+ { session_uuid, hostname, browser, os, screen, language, country, device },+) {+ const params = [+ session_uuid,+ website_id,+ hostname,+ browser,+ os,+ device,+ screen,+ language,+ country ? country : null,+ ];++ await rawQueryClickhouse(+ `insert into umami_dev.session (created_at, session_uuid, website_id, hostname, browser, os, device, screen, language, country)+ values (${getDateFormatClickhouse(new Date())}, $1, $2, $3, $4, $5, $6, $7, $8, $9);`,+ params,+ );++ return getSessionByUuid(session_uuid);+}diff --git a/queries/analytics/session/getSessionByUuid.js b/queries/analytics/session/getSessionByUuid.jsindex 042b49144..fde70e7f0 100644--- a/queries/analytics/session/getSessionByUuid.js+++ b/queries/analytics/session/getSessionByUuid.js@@ -24,7 +24,6 @@ async function clickhouseQuery(session_uuid) {return rawQueryClickhouse(`select- session_id,session_uuid,website_id,created_at,@@ -36,7 +35,7 @@ async function clickhouseQuery(session_uuid) {"language",countryfrom session- where session_id = $1+ where session_uuid = $1`,params,);diff --git a/queries/analytics/session/getSessionMetrics.js b/queries/analytics/session/getSessionMetrics.jsindex 432b696cd..ee6d1dc0c 100644--- a/queries/analytics/session/getSessionMetrics.js+++ b/queries/analytics/session/getSessionMetrics.js@@ -40,14 +40,19 @@ async function relationalQuery(website_id, start_at, end_at, field, filters = {}async function clickhouseQuery(website_id, start_at, end_at, field, filters = {}) {const params = [website_id];- const { pageviewQuery, sessionQuery, joinSession } = parseFilters('pageview', filters, params);+ const { pageviewQuery, sessionQuery, joinSession } = parseFilters(+ 'pageview',+ filters,+ params,+ 'session_uuid',+ );return rawQueryClickhouse(`select ${field} x, count(*) yfrom session as x- where x.session_id in (- select pageview.session_id+ where x.session_uuid in (+ select pageview.session_uuidfrom pageview${joinSession}where pageview.website_id=$1diff --git a/queries/analytics/stats/getActiveVisitors.js b/queries/analytics/stats/getActiveVisitors.jsindex 2789f769d..048845217 100644--- a/queries/analytics/stats/getActiveVisitors.js+++ b/queries/analytics/stats/getActiveVisitors.js@@ -29,7 +29,7 @@ async function clickhouseQuery(website_id) {return rawQueryClickhouse(`- select count(distinct session_id) x+ select count(distinct session_uuid) xfrom pageviewwhere website_id = $1and created_at >= ${getDateFormatClickhouse(subMinutes(new Date(), 5))}diff --git a/queries/analytics/stats/getWebsiteStats.js b/queries/analytics/stats/getWebsiteStats.jsindex f1ee921a2..574a877db 100644--- a/queries/analytics/stats/getWebsiteStats.js+++ b/queries/analytics/stats/getWebsiteStats.js@@ -53,17 +53,22 @@ async function relationalQuery(website_id, start_at, end_at, filters = {}) {async function clickhouseQuery(website_id, start_at, end_at, filters = {}) {const params = [website_id];- const { pageviewQuery, sessionQuery, joinSession } = parseFilters('pageview', filters, params);+ const { pageviewQuery, sessionQuery, joinSession } = parseFilters(+ 'pageview',+ filters,+ params,+ 'session_uuid',+ );return rawQueryClickhouse(`selectsum(t.c) as "pageviews",- count(distinct t.session_id) as "uniques",+ count(distinct t.session_uuid) as "uniques",sum(if(t.c = 1, 1, 0)) as "bounces",sum(if(max_time < min_time + interval 1 hour, max_time-min_time, 0)) as "totaltime"from (- select pageview.session_id,+ select pageview.session_uuid,${getDateQueryClickhouse('pageview.created_at', 'day')} time_series,count(*) c,min(created_at) min_time,@@ -74,7 +79,7 @@ async function clickhouseQuery(website_id, start_at, end_at, filters = {}) {and ${getBetweenDatesClickhouse('pageview.created_at', start_at, end_at)}${pageviewQuery}${sessionQuery}- group by pageview.session_id, time_series+ group by pageview.session_uuid, time_series) t;`,params,