Umami is a privacy-first web and product analytics platform. Traffic, campaigns, behavior, conversions, and revenue in one place. No cookies, no surveillance, self-hosted or in the cloud.
TypeScript analyticsaudience-segmentationchartscohort-analysisgoogle-analyticsproduct-analyticsstatisticsuser-journeyweb-analytics
Added rev_id column. Updated redis calls.
Mike Cao3485b6268bc5
diff --git a/db/postgresql/schema.prisma b/db/postgresql/schema.prismaindex 9e71377aa..93b18608a 100644--- a/db/postgresql/schema.prisma+++ b/db/postgresql/schema.prisma@@ -45,7 +45,7 @@ model website {name String @db.VarChar(100)domain String? @db.VarChar(500)shareId String? @unique @map("share_id") @db.VarChar(64)- revId Int @default(0) @map("rev_id") @db.Int+ revId Int @default(0) @map("rev_id") @db.IntegercreatedAt DateTime? @default(now()) @map("created_at") @db.Timestamptz(6)isDeleted Boolean @default(false) @map("is_deleted")diff --git a/lib/auth.js b/lib/auth.jsindex f843065c2..ae66ce81a 100644--- a/lib/auth.js+++ b/lib/auth.js@@ -1,4 +1,4 @@-import { parseSecureToken, parseToken } from 'next-basics';+import { getRandomChars, parseSecureToken, parseToken } from 'next-basics';import { getUser, getWebsite } from 'queries';import debug from 'debug';import { SHARE_TOKEN_HEADER, TYPE_USER, TYPE_WEBSITE } from 'lib/constants';@@ -6,10 +6,19 @@ import { secret } from 'lib/crypto';const log = debug('umami:auth');+export function generateAuthToken() {+ return getRandomChars(32);+}++export function getAuthToken(req) {+ const token = req.headers.authorization;++ return token.split(' ')[1];+}+export function parseAuthToken(req) {try {- const token = req.headers.authorization;- return parseSecureToken(token.split(' ')[1], secret());+ return parseSecureToken(getAuthToken(req), secret());} catch (e) {log(e);return null;diff --git a/lib/cache.js b/lib/cache.jsnew file mode 100644index 000000000..e69de29bbdiff --git a/lib/redis.js b/lib/redis.jsindex 16cfce195..6c4e58efb 100644--- a/lib/redis.js+++ b/lib/redis.js@@ -41,6 +41,12 @@ async function set(key, value) {return redis.set(key, value);}+async function del(key) {+ await connect();++ return redis.del(key);+}+async function connect() {if (!redis) {redis = process.env.REDIS_URL && (global[REDIS] || getClient());@@ -49,4 +55,4 @@ async function connect() {return redis;}-export default { enabled, client: redis, log, connect, get, set };+export default { enabled, client: redis, log, connect, get, set, del };diff --git a/pages/api/auth/login.js b/pages/api/auth/login.jsindex fb9b5a1b9..dfdd3ec36 100644--- a/pages/api/auth/login.js+++ b/pages/api/auth/login.js@@ -1,6 +1,8 @@import { ok, unauthorized, badRequest, checkPassword, createSecureToken } from 'next-basics';import { getUser } from 'queries';import { secret } from 'lib/crypto';+import redis from 'lib/redis';+import { generateAuthToken } from 'lib/auth';export default async (req, res) => {const { username, password } = req.body;@@ -13,6 +15,13 @@ export default async (req, res) => {if (user && checkPassword(password, user.password)) {const { id: userId, username, isAdmin } = user;++ if (redis.enabled) {+ const token = `auth:${generateAuthToken()}`;++ return ok(res, { token, user });+ }+const token = createSecureToken({ userId, username, isAdmin }, secret());return ok(res, { token, user });diff --git a/pages/api/auth/logout.js b/pages/api/auth/logout.jsnew file mode 100644index 000000000..88ea81ecf--- /dev/null+++ b/pages/api/auth/logout.js@@ -0,0 +1,18 @@+import { methodNotAllowed, ok } from 'next-basics';+import { useAuth } from 'lib/middleware';+import redis from 'lib/redis';+import { getAuthToken } from 'lib/auth';++export default async (req, res) => {+ await useAuth(req, res);++ if (req.method === 'POST') {+ if (redis.enabled) {+ await redis.del(`auth:${getAuthToken(req)}`);+ }++ return ok(res);+ }++ return methodNotAllowed(res);+};diff --git a/pages/logout.js b/pages/logout.jsindex bcc99a10b..a91205a4c 100644--- a/pages/logout.js+++ b/pages/logout.js@@ -1,14 +1,22 @@import { useEffect } from 'react';import { useRouter } from 'next/router';-import { removeItem } from 'next-basics';+import { removeItem, useApi } from 'next-basics';import { AUTH_TOKEN } from 'lib/constants';import { setUser } from 'store/app';export default function LogoutPage() {const router = useRouter();+ const { post } = useApi();useEffect(() => {+ async function logout() {+ await post('/logout');+ }+removeItem(AUTH_TOKEN);++ logout();+router.push('/login');return () => setUser(null);diff --git a/queries/admin/website/createWebsite.js b/queries/admin/website/createWebsite.jsindex f9cb94bec..02bb2838f 100644--- a/queries/admin/website/createWebsite.js+++ b/queries/admin/website/createWebsite.js@@ -15,7 +15,7 @@ export async function createWebsite(userId, data) {}).then(async res => {if (redis.enabled && res) {- await redis.set(`website:${res.id}`, 1);+ await redis.set(`website:${res.id}`, res);}return res;diff --git a/queries/admin/website/getWebsite.js b/queries/admin/website/getWebsite.jsindex b7a468d29..46ec5114d 100644--- a/queries/admin/website/getWebsite.js+++ b/queries/admin/website/getWebsite.js@@ -8,7 +8,7 @@ export async function getWebsite(where) {}).then(async data => {if (redis.enabled && data) {- await redis.set(`website:${data.id}`, 1);+ await redis.set(`website:${data.id}`, data);}return data;diff --git a/queries/analytics/event/getEventData.js b/queries/analytics/event/getEventData.jsindex 72857af43..df2f7269b 100644--- a/queries/analytics/event/getEventData.js+++ b/queries/analytics/event/getEventData.js@@ -1,11 +1,16 @@import clickhouse from 'lib/clickhouse';import { CLICKHOUSE, PRISMA, runQuery } from 'lib/db';import prisma from 'lib/prisma';+import redis from 'lib/redis';export async function getEventData(...args) {return runQuery({[PRISMA]: () => relationalQuery(...args),[CLICKHOUSE]: () => clickhouseQuery(...args),+ }).then(results => {+ return Object.keys(results[0]).map(a => {+ return { x: a, y: results[0][`${a}`] };+ });});}@@ -21,7 +26,7 @@ async function relationalQuery(websiteId, { startDate, endDate, event_name, coluon event.website_id = website.website_idjoin event_dataon event.event_id = event_data.event_id- where website.website_id='${websiteId}'+ where website.website_id ='${websiteId}'and event.created_at between $1 and $2${event_name ? `and event_name = ${event_name}` : ''}${@@ -30,23 +35,21 @@ async function relationalQuery(websiteId, { startDate, endDate, event_name, colu: ''}`,params,- ).then(results => {- return Object.keys(results[0]).map(a => {- return { x: a, y: results[0][`${a}`] };- });- });+ );}async function clickhouseQuery(websiteId, { startDate, endDate, event_name, columns, filters }) {const { rawQuery, getBetweenDates, getEventDataColumnsQuery, getEventDataFilterQuery } =clickhouse;- const params = [websiteId];+ const website = await redis.get(`website:${websiteId}`);+ const params = [websiteId, website?.revId || 0];return rawQuery(`select${getEventDataColumnsQuery('event_data', columns)}from event- where website_id= $1+ where website_id = $1+ and rev_id = $2${event_name ? `and event_name = ${event_name}` : ''}and ${getBetweenDates('created_at', startDate, endDate)}${@@ -55,9 +58,5 @@ async function clickhouseQuery(websiteId, { startDate, endDate, event_name, colu: ''}`,params,- ).then(results => {- return Object.keys(results[0]).map(a => {- return { x: a, y: results[0][`${a}`] };- });- });+ );}diff --git a/queries/analytics/event/getEventMetrics.js b/queries/analytics/event/getEventMetrics.jsindex 9dcf6d559..8b6c82633 100644--- a/queries/analytics/event/getEventMetrics.js+++ b/queries/analytics/event/getEventMetrics.js@@ -1,6 +1,7 @@import prisma from 'lib/prisma';import clickhouse from 'lib/clickhouse';import { runQuery, CLICKHOUSE, PRISMA } from 'lib/db';+import redis from 'lib/redis';export async function getEventMetrics(...args) {return runQuery({@@ -46,7 +47,8 @@ async function clickhouseQuery(filters = {},) {const { rawQuery, getDateQuery, getBetweenDates, getFilterQuery } = clickhouse;- const params = [websiteId];+ const website = await redis.get(`website:${websiteId}`);+ const params = [websiteId, website?.revId || 0];return rawQuery(`select@@ -55,7 +57,8 @@ async function clickhouseQuery(count(*) yfrom eventwhere event_name != ''- and website_id= $1+ and website_id = $1+ anbd rev_id = $2and ${getBetweenDates('created_at', start_at, end_at)}${getFilterQuery('event', filters, params)}group by x, tdiff --git a/queries/analytics/event/saveEvent.js b/queries/analytics/event/saveEvent.jsindex a6bbe2fad..c6d3c8275 100644--- a/queries/analytics/event/saveEvent.js+++ b/queries/analytics/event/saveEvent.js@@ -3,6 +3,7 @@ import { CLICKHOUSE, PRISMA, runQuery } from 'lib/db';import kafka from 'lib/kafka';import prisma from 'lib/prisma';import { uuid } from 'lib/crypto';+import redis from 'lib/redis';export async function saveEvent(...args) {return runQuery({@@ -43,11 +44,13 @@ async function clickhouseQuery({ session: { country, sessionUuid, ...sessionArgs }, eventUuid, url, eventName, eventData },) {const { getDateFormat, sendMessage } = kafka;+ const website = await redis.get(`website:${websiteId}`);const params = {session_id: sessionUuid,event_id: eventUuid,website_id: websiteId,+ rev_id: website?.revId || 0,created_at: getDateFormat(new Date()),url: url?.substring(0, URL_LENGTH),event_name: eventName?.substring(0, EVENT_NAME_LENGTH),diff --git a/queries/analytics/pageview/getPageviewMetrics.js b/queries/analytics/pageview/getPageviewMetrics.jsindex 3acfdefb4..62cda1658 100644--- a/queries/analytics/pageview/getPageviewMetrics.js+++ b/queries/analytics/pageview/getPageviewMetrics.js@@ -1,6 +1,7 @@import prisma from 'lib/prisma';import clickhouse from 'lib/clickhouse';import { runQuery, CLICKHOUSE, PRISMA } from 'lib/db';+import redis from 'lib/redis';export async function getPageviewMetrics(...args) {return runQuery({@@ -37,13 +38,15 @@ async function relationalQuery(websiteId, { startDate, endDate, column, table, fasync function clickhouseQuery(websiteId, { startDate, endDate, column, filters = {} }) {const { rawQuery, parseFilters, getBetweenDates } = clickhouse;- const params = [websiteId];+ const website = await redis.get(`website:${websiteId}`);+ const params = [websiteId, website?.revId || 0];const { pageviewQuery, sessionQuery, eventQuery } = parseFilters(column, filters, params);return rawQuery(`select ${column} x, count(*) yfrom event- where website_id= $1+ where website_id = $1+ and rev_id = $2${column !== 'event_name' ? `and event_name = ''` : `and event_name != ''`}and ${getBetweenDates('created_at', startDate, endDate)}${pageviewQuery}diff --git a/queries/analytics/pageview/getPageviewStats.js b/queries/analytics/pageview/getPageviewStats.jsindex 57ff27f35..b18a3acc7 100644--- a/queries/analytics/pageview/getPageviewStats.js+++ b/queries/analytics/pageview/getPageviewStats.js@@ -1,6 +1,7 @@import prisma from 'lib/prisma';import clickhouse from 'lib/clickhouse';import { runQuery, CLICKHOUSE, PRISMA } from 'lib/db';+import redis from 'lib/redis';export async function getPageviewStats(...args) {return runQuery({@@ -51,7 +52,8 @@ async function clickhouseQuery({ start_at, end_at, timezone = 'UTC', unit = 'day', count = '*', filters = {} },) {const { parseFilters, rawQuery, getDateStringQuery, getDateQuery, getBetweenDates } = clickhouse;- const params = [websiteId];+ const website = await redis.get(`website:${websiteId}`);+ const params = [websiteId, website?.revId || 0];const { pageviewQuery, sessionQuery } = parseFilters(null, filters, params);return rawQuery(diff --git a/queries/analytics/pageview/savePageView.js b/queries/analytics/pageview/savePageView.jsindex c035bfdfc..0763ab18c 100644--- a/queries/analytics/pageview/savePageView.js+++ b/queries/analytics/pageview/savePageView.js@@ -2,6 +2,7 @@ import { URL_LENGTH } from 'lib/constants';import { CLICKHOUSE, PRISMA, runQuery } from 'lib/db';import kafka from 'lib/kafka';import prisma from 'lib/prisma';+import redis from 'lib/redis';export async function savePageView(...args) {return runQuery({@@ -29,6 +30,7 @@ async function clickhouseQuery(websiteId,{ session: { country, id: sessionId, ...sessionArgs }, url, referrer },) {+ const website = await redis.get(`website:${websiteId}`);const { getDateFormat, sendMessage } = kafka;const params = {session_id: sessionId,@@ -36,6 +38,7 @@ async function clickhouseQuery(created_at: getDateFormat(new Date()),url: url?.substring(0, URL_LENGTH),referrer: referrer?.substring(0, URL_LENGTH),+ rev_id: website?.revId || 0,...sessionArgs,country: country ? country : null,};diff --git a/queries/analytics/session/createSession.js b/queries/analytics/session/createSession.jsindex b121ef7cb..dfa8567b1 100644--- a/queries/analytics/session/createSession.js+++ b/queries/analytics/session/createSession.js@@ -7,34 +7,32 @@ export async function createSession(...args) {return runQuery({[PRISMA]: () => relationalQuery(...args),[CLICKHOUSE]: () => clickhouseQuery(...args),+ }).then(async data => {+ if (redis.enabled && data) {+ await redis.set(`session:${data.id}`, data);+ }++ return data;});}async function relationalQuery(websiteId, data) {- return prisma.client.session- .create({- data: {- websiteId,- ...data,- },- select: {- id: true,- hostname: true,- browser: true,- os: true,- screen: true,- language: true,- country: true,- device: true,- },- })- .then(async res => {- if (redis.enabled && res) {- await redis.set(`session:${res.id}`, 1);- }-- return res;- });+ return prisma.client.session.create({+ data: {+ websiteId,+ ...data,+ },+ select: {+ id: true,+ hostname: true,+ browser: true,+ os: true,+ screen: true,+ language: true,+ country: true,+ device: true,+ },+ });}async function clickhouseQuery(@@ -42,10 +40,12 @@ async function clickhouseQuery({ sessionId, hostname, browser, os, screen, language, country, device },) {const { getDateFormat, sendMessage } = kafka;+ const website = await redis.get(`website:${websiteId}`);- const params = {+ const data = {sessionId,website_id: websiteId,+ rev_id: website?.revId || 0,created_at: getDateFormat(new Date()),hostname,browser,@@ -56,9 +56,7 @@ async function clickhouseQuery(country: country ? country : null,};- await sendMessage(params, 'event');+ await sendMessage(data, 'event');- if (redis.enabled) {- await redis.set(`session:${sessionId}`, 1);- }+ return data;}diff --git a/queries/analytics/session/getSession.js b/queries/analytics/session/getSession.jsindex 6bb4e445c..36269fc01 100644--- a/queries/analytics/session/getSession.js+++ b/queries/analytics/session/getSession.js@@ -7,26 +7,24 @@ export async function getSession(...args) {return runQuery({[PRISMA]: () => relationalQuery(...args),[CLICKHOUSE]: () => clickhouseQuery(...args),+ }).then(async data => {+ if (redis.enabled && data) {+ await redis.set(`session:${data.id}`, data);+ }++ return data;});}async function relationalQuery(where) {- return prisma.client.session- .findUnique({- where,- })- .then(async res => {- if (redis.enabled && res) {- await redis.set(`session:${res.sessionUuid}`, 1);- }-- return res;- });+ return prisma.client.session.findUnique({+ where,+ });}-async function clickhouseQuery(sessionUuid) {+async function clickhouseQuery(sessionId) {const { rawQuery, findFirst } = clickhouse;- const params = [sessionUuid];+ const params = [sessionId];return rawQuery(`select distinct@@ -43,13 +41,5 @@ async function clickhouseQuery(sessionUuid) {from eventwhere session_id = $1`,params,- )- .then(result => findFirst(result))- .then(async res => {- if (redis.enabled && res) {- await redis.set(`session:${res.id}`, 1);- }-- return res;- });+ ).then(result => findFirst(result));}diff --git a/queries/analytics/session/getSessionMetrics.js b/queries/analytics/session/getSessionMetrics.jsindex e4e389d07..797230f8f 100644--- a/queries/analytics/session/getSessionMetrics.js+++ b/queries/analytics/session/getSessionMetrics.js@@ -1,6 +1,7 @@import prisma from 'lib/prisma';import clickhouse from 'lib/clickhouse';import { runQuery, CLICKHOUSE, PRISMA } from 'lib/db';+import redis from 'lib/redis';export async function getSessionMetrics(...args) {return runQuery({@@ -36,13 +37,15 @@ async function relationalQuery(websiteId, { startDate, endDate, field, filters =async function clickhouseQuery(websiteId, { startDate, endDate, field, filters = {} }) {const { parseFilters, getBetweenDates, rawQuery } = clickhouse;- const params = [websiteId];+ const website = await redis.get(`website:${websiteId}`);+ const params = [websiteId, website?.revId || 0];const { pageviewQuery, sessionQuery } = parseFilters(null, filters, params);return rawQuery(`select ${field} x, count(*) yfrom event as x- where website_id=$1+ where website_id = $1+ and rev_id = $2and event_name = ''and ${getBetweenDates('created_at', startDate, endDate)}${pageviewQuery}