Skip to content

armgit / umami

Public Imported from GitHub
0

Umami is a privacy-first web and product analytics platform. Traffic, campaigns, behavior, conversions, and revenue in one place. No cookies, no surveillance, self-hosted or in the cloud.

Added rev_id column. Updated redis calls.

Mike Cao3485b6268bc5
diff --git a/db/postgresql/schema.prisma b/db/postgresql/schema.prisma
index 9e71377aa..93b18608a 100644
--- a/db/postgresql/schema.prisma
+++ b/db/postgresql/schema.prisma
@@ -45,7 +45,7 @@ model website {
name String @db.VarChar(100)
domain String? @db.VarChar(500)
shareId String? @unique @map("share_id") @db.VarChar(64)
- revId Int @default(0) @map("rev_id") @db.Int
+ revId Int @default(0) @map("rev_id") @db.Integer
createdAt DateTime? @default(now()) @map("created_at") @db.Timestamptz(6)
isDeleted Boolean @default(false) @map("is_deleted")
diff --git a/lib/auth.js b/lib/auth.js
index f843065c2..ae66ce81a 100644
--- a/lib/auth.js
+++ b/lib/auth.js
@@ -1,4 +1,4 @@
-import { parseSecureToken, parseToken } from 'next-basics';
+import { getRandomChars, parseSecureToken, parseToken } from 'next-basics';
import { getUser, getWebsite } from 'queries';
import debug from 'debug';
import { SHARE_TOKEN_HEADER, TYPE_USER, TYPE_WEBSITE } from 'lib/constants';
@@ -6,10 +6,19 @@ import { secret } from 'lib/crypto';
const log = debug('umami:auth');
+export function generateAuthToken() {
+ return getRandomChars(32);
+}
+
+export function getAuthToken(req) {
+ const token = req.headers.authorization;
+
+ return token.split(' ')[1];
+}
+
export function parseAuthToken(req) {
try {
- const token = req.headers.authorization;
- return parseSecureToken(token.split(' ')[1], secret());
+ return parseSecureToken(getAuthToken(req), secret());
} catch (e) {
log(e);
return null;
diff --git a/lib/cache.js b/lib/cache.js
new file mode 100644
index 000000000..e69de29bb
diff --git a/lib/redis.js b/lib/redis.js
index 16cfce195..6c4e58efb 100644
--- a/lib/redis.js
+++ b/lib/redis.js
@@ -41,6 +41,12 @@ async function set(key, value) {
return redis.set(key, value);
}
+async function del(key) {
+ await connect();
+
+ return redis.del(key);
+}
+
async function connect() {
if (!redis) {
redis = process.env.REDIS_URL && (global[REDIS] || getClient());
@@ -49,4 +55,4 @@ async function connect() {
return redis;
}
-export default { enabled, client: redis, log, connect, get, set };
+export default { enabled, client: redis, log, connect, get, set, del };
diff --git a/pages/api/auth/login.js b/pages/api/auth/login.js
index fb9b5a1b9..dfdd3ec36 100644
--- a/pages/api/auth/login.js
+++ b/pages/api/auth/login.js
@@ -1,6 +1,8 @@
import { ok, unauthorized, badRequest, checkPassword, createSecureToken } from 'next-basics';
import { getUser } from 'queries';
import { secret } from 'lib/crypto';
+import redis from 'lib/redis';
+import { generateAuthToken } from 'lib/auth';
export default async (req, res) => {
const { username, password } = req.body;
@@ -13,6 +15,13 @@ export default async (req, res) => {
if (user && checkPassword(password, user.password)) {
const { id: userId, username, isAdmin } = user;
+
+ if (redis.enabled) {
+ const token = `auth:${generateAuthToken()}`;
+
+ return ok(res, { token, user });
+ }
+
const token = createSecureToken({ userId, username, isAdmin }, secret());
return ok(res, { token, user });
diff --git a/pages/api/auth/logout.js b/pages/api/auth/logout.js
new file mode 100644
index 000000000..88ea81ecf
--- /dev/null
+++ b/pages/api/auth/logout.js
@@ -0,0 +1,18 @@
+import { methodNotAllowed, ok } from 'next-basics';
+import { useAuth } from 'lib/middleware';
+import redis from 'lib/redis';
+import { getAuthToken } from 'lib/auth';
+
+export default async (req, res) => {
+ await useAuth(req, res);
+
+ if (req.method === 'POST') {
+ if (redis.enabled) {
+ await redis.del(`auth:${getAuthToken(req)}`);
+ }
+
+ return ok(res);
+ }
+
+ return methodNotAllowed(res);
+};
diff --git a/pages/logout.js b/pages/logout.js
index bcc99a10b..a91205a4c 100644
--- a/pages/logout.js
+++ b/pages/logout.js
@@ -1,14 +1,22 @@
import { useEffect } from 'react';
import { useRouter } from 'next/router';
-import { removeItem } from 'next-basics';
+import { removeItem, useApi } from 'next-basics';
import { AUTH_TOKEN } from 'lib/constants';
import { setUser } from 'store/app';
export default function LogoutPage() {
const router = useRouter();
+ const { post } = useApi();
useEffect(() => {
+ async function logout() {
+ await post('/logout');
+ }
+
removeItem(AUTH_TOKEN);
+
+ logout();
+
router.push('/login');
return () => setUser(null);
diff --git a/queries/admin/website/createWebsite.js b/queries/admin/website/createWebsite.js
index f9cb94bec..02bb2838f 100644
--- a/queries/admin/website/createWebsite.js
+++ b/queries/admin/website/createWebsite.js
@@ -15,7 +15,7 @@ export async function createWebsite(userId, data) {
})
.then(async res => {
if (redis.enabled && res) {
- await redis.set(`website:${res.id}`, 1);
+ await redis.set(`website:${res.id}`, res);
}
return res;
diff --git a/queries/admin/website/getWebsite.js b/queries/admin/website/getWebsite.js
index b7a468d29..46ec5114d 100644
--- a/queries/admin/website/getWebsite.js
+++ b/queries/admin/website/getWebsite.js
@@ -8,7 +8,7 @@ export async function getWebsite(where) {
})
.then(async data => {
if (redis.enabled && data) {
- await redis.set(`website:${data.id}`, 1);
+ await redis.set(`website:${data.id}`, data);
}
return data;
diff --git a/queries/analytics/event/getEventData.js b/queries/analytics/event/getEventData.js
index 72857af43..df2f7269b 100644
--- a/queries/analytics/event/getEventData.js
+++ b/queries/analytics/event/getEventData.js
@@ -1,11 +1,16 @@
import clickhouse from 'lib/clickhouse';
import { CLICKHOUSE, PRISMA, runQuery } from 'lib/db';
import prisma from 'lib/prisma';
+import redis from 'lib/redis';
export async function getEventData(...args) {
return runQuery({
[PRISMA]: () => relationalQuery(...args),
[CLICKHOUSE]: () => clickhouseQuery(...args),
+ }).then(results => {
+ return Object.keys(results[0]).map(a => {
+ return { x: a, y: results[0][`${a}`] };
+ });
});
}
@@ -21,7 +26,7 @@ async function relationalQuery(websiteId, { startDate, endDate, event_name, colu
on event.website_id = website.website_id
join event_data
on event.event_id = event_data.event_id
- where website.website_id='${websiteId}'
+ where website.website_id ='${websiteId}'
and event.created_at between $1 and $2
${event_name ? `and event_name = ${event_name}` : ''}
${
@@ -30,23 +35,21 @@ async function relationalQuery(websiteId, { startDate, endDate, event_name, colu
: ''
}`,
params,
- ).then(results => {
- return Object.keys(results[0]).map(a => {
- return { x: a, y: results[0][`${a}`] };
- });
- });
+ );
}
async function clickhouseQuery(websiteId, { startDate, endDate, event_name, columns, filters }) {
const { rawQuery, getBetweenDates, getEventDataColumnsQuery, getEventDataFilterQuery } =
clickhouse;
- const params = [websiteId];
+ const website = await redis.get(`website:${websiteId}`);
+ const params = [websiteId, website?.revId || 0];
return rawQuery(
`select
${getEventDataColumnsQuery('event_data', columns)}
from event
- where website_id= $1
+ where website_id = $1
+ and rev_id = $2
${event_name ? `and event_name = ${event_name}` : ''}
and ${getBetweenDates('created_at', startDate, endDate)}
${
@@ -55,9 +58,5 @@ async function clickhouseQuery(websiteId, { startDate, endDate, event_name, colu
: ''
}`,
params,
- ).then(results => {
- return Object.keys(results[0]).map(a => {
- return { x: a, y: results[0][`${a}`] };
- });
- });
+ );
}
diff --git a/queries/analytics/event/getEventMetrics.js b/queries/analytics/event/getEventMetrics.js
index 9dcf6d559..8b6c82633 100644
--- a/queries/analytics/event/getEventMetrics.js
+++ b/queries/analytics/event/getEventMetrics.js
@@ -1,6 +1,7 @@
import prisma from 'lib/prisma';
import clickhouse from 'lib/clickhouse';
import { runQuery, CLICKHOUSE, PRISMA } from 'lib/db';
+import redis from 'lib/redis';
export async function getEventMetrics(...args) {
return runQuery({
@@ -46,7 +47,8 @@ async function clickhouseQuery(
filters = {},
) {
const { rawQuery, getDateQuery, getBetweenDates, getFilterQuery } = clickhouse;
- const params = [websiteId];
+ const website = await redis.get(`website:${websiteId}`);
+ const params = [websiteId, website?.revId || 0];
return rawQuery(
`select
@@ -55,7 +57,8 @@ async function clickhouseQuery(
count(*) y
from event
where event_name != ''
- and website_id= $1
+ and website_id = $1
+ anbd rev_id = $2
and ${getBetweenDates('created_at', start_at, end_at)}
${getFilterQuery('event', filters, params)}
group by x, t
diff --git a/queries/analytics/event/saveEvent.js b/queries/analytics/event/saveEvent.js
index a6bbe2fad..c6d3c8275 100644
--- a/queries/analytics/event/saveEvent.js
+++ b/queries/analytics/event/saveEvent.js
@@ -3,6 +3,7 @@ import { CLICKHOUSE, PRISMA, runQuery } from 'lib/db';
import kafka from 'lib/kafka';
import prisma from 'lib/prisma';
import { uuid } from 'lib/crypto';
+import redis from 'lib/redis';
export async function saveEvent(...args) {
return runQuery({
@@ -43,11 +44,13 @@ async function clickhouseQuery(
{ session: { country, sessionUuid, ...sessionArgs }, eventUuid, url, eventName, eventData },
) {
const { getDateFormat, sendMessage } = kafka;
+ const website = await redis.get(`website:${websiteId}`);
const params = {
session_id: sessionUuid,
event_id: eventUuid,
website_id: websiteId,
+ rev_id: website?.revId || 0,
created_at: getDateFormat(new Date()),
url: url?.substring(0, URL_LENGTH),
event_name: eventName?.substring(0, EVENT_NAME_LENGTH),
diff --git a/queries/analytics/pageview/getPageviewMetrics.js b/queries/analytics/pageview/getPageviewMetrics.js
index 3acfdefb4..62cda1658 100644
--- a/queries/analytics/pageview/getPageviewMetrics.js
+++ b/queries/analytics/pageview/getPageviewMetrics.js
@@ -1,6 +1,7 @@
import prisma from 'lib/prisma';
import clickhouse from 'lib/clickhouse';
import { runQuery, CLICKHOUSE, PRISMA } from 'lib/db';
+import redis from 'lib/redis';
export async function getPageviewMetrics(...args) {
return runQuery({
@@ -37,13 +38,15 @@ async function relationalQuery(websiteId, { startDate, endDate, column, table, f
async function clickhouseQuery(websiteId, { startDate, endDate, column, filters = {} }) {
const { rawQuery, parseFilters, getBetweenDates } = clickhouse;
- const params = [websiteId];
+ const website = await redis.get(`website:${websiteId}`);
+ const params = [websiteId, website?.revId || 0];
const { pageviewQuery, sessionQuery, eventQuery } = parseFilters(column, filters, params);
return rawQuery(
`select ${column} x, count(*) y
from event
- where website_id= $1
+ where website_id = $1
+ and rev_id = $2
${column !== 'event_name' ? `and event_name = ''` : `and event_name != ''`}
and ${getBetweenDates('created_at', startDate, endDate)}
${pageviewQuery}
diff --git a/queries/analytics/pageview/getPageviewStats.js b/queries/analytics/pageview/getPageviewStats.js
index 57ff27f35..b18a3acc7 100644
--- a/queries/analytics/pageview/getPageviewStats.js
+++ b/queries/analytics/pageview/getPageviewStats.js
@@ -1,6 +1,7 @@
import prisma from 'lib/prisma';
import clickhouse from 'lib/clickhouse';
import { runQuery, CLICKHOUSE, PRISMA } from 'lib/db';
+import redis from 'lib/redis';
export async function getPageviewStats(...args) {
return runQuery({
@@ -51,7 +52,8 @@ async function clickhouseQuery(
{ start_at, end_at, timezone = 'UTC', unit = 'day', count = '*', filters = {} },
) {
const { parseFilters, rawQuery, getDateStringQuery, getDateQuery, getBetweenDates } = clickhouse;
- const params = [websiteId];
+ const website = await redis.get(`website:${websiteId}`);
+ const params = [websiteId, website?.revId || 0];
const { pageviewQuery, sessionQuery } = parseFilters(null, filters, params);
return rawQuery(
diff --git a/queries/analytics/pageview/savePageView.js b/queries/analytics/pageview/savePageView.js
index c035bfdfc..0763ab18c 100644
--- a/queries/analytics/pageview/savePageView.js
+++ b/queries/analytics/pageview/savePageView.js
@@ -2,6 +2,7 @@ import { URL_LENGTH } from 'lib/constants';
import { CLICKHOUSE, PRISMA, runQuery } from 'lib/db';
import kafka from 'lib/kafka';
import prisma from 'lib/prisma';
+import redis from 'lib/redis';
export async function savePageView(...args) {
return runQuery({
@@ -29,6 +30,7 @@ async function clickhouseQuery(
websiteId,
{ session: { country, id: sessionId, ...sessionArgs }, url, referrer },
) {
+ const website = await redis.get(`website:${websiteId}`);
const { getDateFormat, sendMessage } = kafka;
const params = {
session_id: sessionId,
@@ -36,6 +38,7 @@ async function clickhouseQuery(
created_at: getDateFormat(new Date()),
url: url?.substring(0, URL_LENGTH),
referrer: referrer?.substring(0, URL_LENGTH),
+ rev_id: website?.revId || 0,
...sessionArgs,
country: country ? country : null,
};
diff --git a/queries/analytics/session/createSession.js b/queries/analytics/session/createSession.js
index b121ef7cb..dfa8567b1 100644
--- a/queries/analytics/session/createSession.js
+++ b/queries/analytics/session/createSession.js
@@ -7,34 +7,32 @@ export async function createSession(...args) {
return runQuery({
[PRISMA]: () => relationalQuery(...args),
[CLICKHOUSE]: () => clickhouseQuery(...args),
+ }).then(async data => {
+ if (redis.enabled && data) {
+ await redis.set(`session:${data.id}`, data);
+ }
+
+ return data;
});
}
async function relationalQuery(websiteId, data) {
- return prisma.client.session
- .create({
- data: {
- websiteId,
- ...data,
- },
- select: {
- id: true,
- hostname: true,
- browser: true,
- os: true,
- screen: true,
- language: true,
- country: true,
- device: true,
- },
- })
- .then(async res => {
- if (redis.enabled && res) {
- await redis.set(`session:${res.id}`, 1);
- }
-
- return res;
- });
+ return prisma.client.session.create({
+ data: {
+ websiteId,
+ ...data,
+ },
+ select: {
+ id: true,
+ hostname: true,
+ browser: true,
+ os: true,
+ screen: true,
+ language: true,
+ country: true,
+ device: true,
+ },
+ });
}
async function clickhouseQuery(
@@ -42,10 +40,12 @@ async function clickhouseQuery(
{ sessionId, hostname, browser, os, screen, language, country, device },
) {
const { getDateFormat, sendMessage } = kafka;
+ const website = await redis.get(`website:${websiteId}`);
- const params = {
+ const data = {
sessionId,
website_id: websiteId,
+ rev_id: website?.revId || 0,
created_at: getDateFormat(new Date()),
hostname,
browser,
@@ -56,9 +56,7 @@ async function clickhouseQuery(
country: country ? country : null,
};
- await sendMessage(params, 'event');
+ await sendMessage(data, 'event');
- if (redis.enabled) {
- await redis.set(`session:${sessionId}`, 1);
- }
+ return data;
}
diff --git a/queries/analytics/session/getSession.js b/queries/analytics/session/getSession.js
index 6bb4e445c..36269fc01 100644
--- a/queries/analytics/session/getSession.js
+++ b/queries/analytics/session/getSession.js
@@ -7,26 +7,24 @@ export async function getSession(...args) {
return runQuery({
[PRISMA]: () => relationalQuery(...args),
[CLICKHOUSE]: () => clickhouseQuery(...args),
+ }).then(async data => {
+ if (redis.enabled && data) {
+ await redis.set(`session:${data.id}`, data);
+ }
+
+ return data;
});
}
async function relationalQuery(where) {
- return prisma.client.session
- .findUnique({
- where,
- })
- .then(async res => {
- if (redis.enabled && res) {
- await redis.set(`session:${res.sessionUuid}`, 1);
- }
-
- return res;
- });
+ return prisma.client.session.findUnique({
+ where,
+ });
}
-async function clickhouseQuery(sessionUuid) {
+async function clickhouseQuery(sessionId) {
const { rawQuery, findFirst } = clickhouse;
- const params = [sessionUuid];
+ const params = [sessionId];
return rawQuery(
`select distinct
@@ -43,13 +41,5 @@ async function clickhouseQuery(sessionUuid) {
from event
where session_id = $1`,
params,
- )
- .then(result => findFirst(result))
- .then(async res => {
- if (redis.enabled && res) {
- await redis.set(`session:${res.id}`, 1);
- }
-
- return res;
- });
+ ).then(result => findFirst(result));
}
diff --git a/queries/analytics/session/getSessionMetrics.js b/queries/analytics/session/getSessionMetrics.js
index e4e389d07..797230f8f 100644
--- a/queries/analytics/session/getSessionMetrics.js
+++ b/queries/analytics/session/getSessionMetrics.js
@@ -1,6 +1,7 @@
import prisma from 'lib/prisma';
import clickhouse from 'lib/clickhouse';
import { runQuery, CLICKHOUSE, PRISMA } from 'lib/db';
+import redis from 'lib/redis';
export async function getSessionMetrics(...args) {
return runQuery({
@@ -36,13 +37,15 @@ async function relationalQuery(websiteId, { startDate, endDate, field, filters =
async function clickhouseQuery(websiteId, { startDate, endDate, field, filters = {} }) {
const { parseFilters, getBetweenDates, rawQuery } = clickhouse;
- const params = [websiteId];
+ const website = await redis.get(`website:${websiteId}`);
+ const params = [websiteId, website?.revId || 0];
const { pageviewQuery, sessionQuery } = parseFilters(null, filters, params);
return rawQuery(
`select ${field} x, count(*) y
from event as x
- where website_id=$1
+ where website_id = $1
+ and rev_id = $2
and event_name = ''
and ${getBetweenDates('created_at', startDate, endDate)}
${pageviewQuery}