Skip to content

📈 Open source, privacy-first web analytics — a cookie-less Google Analytics alternative with built-in error tracking and performance monitoring. Self-host it or use our cloud. Get started for free!

analytics optimisations

Blue Mouse0e239cba504f
diff --git a/src/analytics/analytics.controller.ts b/src/analytics/analytics.controller.ts
index e7f42e750..3599bf884 100644
--- a/src/analytics/analytics.controller.ts
+++ b/src/analytics/analytics.controller.ts
@@ -91,8 +91,8 @@ export class AnalyticsController {
// TODO: data validation
// TODO: automatic timeBucket detection based on period provided
- let query = `SELECT * FROM analytics WHERE pid='${pid}'`
let queryCustoms = `SELECT ev FROM customEV WHERE pid='${pid}'`
+ let subQuery = `FROM analytics WHERE pid='${pid}'`
if (!_isEmpty(from) && !_isEmpty(to)) {
throw new NotImplementedException('Filtering by from/to params is currently not available')
@@ -102,25 +102,23 @@ export class AnalyticsController {
if (dayjs.utc(to).isAfter(dayjs.utc(), 'second')) {
groupTo = dayjs.utc().format('YYYY-MM-DD HH:mm:ss')
- query += `AND created BETWEEN ${from} AND ${groupTo}`
queryCustoms += `AND created BETWEEN ${from} AND ${groupTo}`
} else {
- query += `AND created BETWEEN ${from} AND ${to}`
queryCustoms += `AND created BETWEEN ${from} AND ${to}`
}
} else if (!_isEmpty(period)) {
- groupFrom = dayjs.utc().subtract(parseInt(period), _last(period)).format('YYYY-MM-DD HH:mm:ss')
- groupTo = dayjs.utc().format('YYYY-MM-DD HH:mm:ss')
- query += `AND created BETWEEN '${groupFrom}' AND '${groupTo}'`
+ groupFrom = dayjs.utc().subtract(parseInt(period), _last(period)).format('YYYY-MM-DD')
+ groupTo = dayjs.utc().format('YYYY-MM-DD 23:59:59')
queryCustoms += `AND created BETWEEN '${groupFrom}' AND '${groupTo}'`
+ subQuery += ` AND created BETWEEN '${groupFrom}' AND '${groupTo}'`
} else {
throw new BadRequestException('The timeframe (either from/to pair or period) to be provided')
}
- const response = await clickhouse.query(query).toPromise()
- const result = await this.analyticsService.groupByTimeBucket(response, timeBucket, groupFrom, groupTo)
+ const result = await this.analyticsService.groupByTimeBucket(timeBucket, groupFrom, groupTo, subQuery, pid)
const customs = await clickhouse.query(queryCustoms).toPromise()
+ console.log(customs)
return {
...result,
diff --git a/src/analytics/analytics.service.ts b/src/analytics/analytics.service.ts
index 998c59487..f0387f295 100644
--- a/src/analytics/analytics.service.ts
+++ b/src/analytics/analytics.service.ts
@@ -7,7 +7,9 @@ import * as _isNull from 'lodash/isNull'
import * as _includes from 'lodash/includes'
import * as _map from 'lodash/map'
import * as _join from 'lodash/join'
+import * as _every from 'lodash/every'
import * as _keys from 'lodash/keys'
+import * as _values from 'lodash/values'
import * as dayjs from 'dayjs'
import * as utc from 'dayjs/plugin/utc'
import { hash } from 'blake3'
@@ -32,6 +34,10 @@ dayjs.extend(utc)
export const getSessionKey = (ip: string, ua: string, pid: string, salt: string = '') => `ses_${hash(`${ua}${ip}${pid}${salt}`).toString('hex')}`
+const cols = [
+ 'cc', 'pg', 'lc', 'br', 'os', 'dv', 'ref','so', 'me', 'ca',
+]
+
@Injectable()
export class AnalyticsService {
constructor(
@@ -190,38 +196,6 @@ export class AnalyticsService {
return Boolean(session)
}
- processData(data: object): object {
- const res = {
- cc: {},
- pg: {},
- lc: {},
- br: {},
- os: {},
- dv: {},
- ref: {},
- so: {},
- me: {},
- ca: {},
- // lt: {},
- }
- const whitelist = _keys(res)
-
- for (let i = 0; i < _size(data); ++i) {
- const tfData = data[i].data
- for (let j = 0; j < _size(tfData); ++j) {
- for (let z = 0; z < _size(whitelist); ++z) {
- const currWLItem = whitelist[z]
- const tfDataRecord = tfData[j][currWLItem]
- if (!_isNull(tfDataRecord)) {
- res[currWLItem][tfDataRecord] = 1 + (res[currWLItem][tfDataRecord] || 0)
- }
- }
- }
- }
-
- return res
- }
-
async getSummary(pids: string[], period: 'w' | 'M' = 'w', advanced: boolean = false): Promise<Object> {
const result = {}
for (let i = 0; i < _size(pids); ++i) {
@@ -238,13 +212,20 @@ export class AnalyticsService {
const query1_unique = `SELECT COUNT() FROM analytics WHERE pid='${pid}' AND unique=1 AND created BETWEEN '${oneWeek}' AND '${now}'`
const query2_unique = `SELECT COUNT() FROM analytics WHERE pid='${pid}' AND unique=1 AND created BETWEEN '${twoWeeks}' AND '${oneWeek}'`
- // todo: save to redis
+ const query1 = `SELECT unique, count() FROM analytics WHERE pid='${pid}' AND created BETWEEN '${oneWeek}' AND '${now}' GROUP BY unique`
+ const query2 = `SELECT unique, count() FROM analytics WHERE pid='${pid}' AND created BETWEEN '${twoWeeks}' AND '${oneWeek}' GROUP BY unique`
+
+ const q1res = await clickhouse.query(query1).toPromise()
+ const q2res = await clickhouse.query(query2).toPromise()
+
+ console.log(q1res, q2res)
+
try {
const res1_pageviews = await clickhouse.query(query1_pageviews).toPromise()
const res2_pageviews = await clickhouse.query(query2_pageviews).toPromise()
const thisWeekPV = res1_pageviews[0]['count()']
const lastWeekPV = res2_pageviews[0]['count()']
-
+
if (advanced) {
const res1_unique = await clickhouse.query(query1_unique).toPromise()
const res2_unique = await clickhouse.query(query2_unique).toPromise()
@@ -274,13 +255,28 @@ export class AnalyticsService {
return result
}
- // TODO: REFACTOR THE FUNCTION; check if there's no date/time shifts
- async groupByTimeBucket(data: Object[], timeBucket: TimeBucketType, from: string, to: string): Promise<object | void> {
- if (_isEmpty(data)) return Promise.resolve()
- let groupDateIterator
- let clone = [...data]
- const res = []
+ async groupByTimeBucket(timeBucket: TimeBucketType, from: string, to: string, subQuery: string, pid: string): Promise<object | void> {
+ const params = {}
+
+ for (let i of cols) {
+ const query1 = `SELECT ${i}, count(*) ${subQuery} AND ${i} IS NOT NULL GROUP BY ${i}`
+ const res = await clickhouse.query(query1).toPromise()
+
+ params[i] = {}
+
+ const size = _size(res)
+ for (let j = 0; j < size; ++j) {
+ const key = res[j][i]
+ const value = res[j]['count()']
+ params[i][key] = value
+ }
+ }
+ if (_every(_values(params), _isEmpty)) {
+ return Promise.resolve()
+ }
+
+ let groupDateIterator
const now = dayjs.utc().endOf(timeBucket)
const djsTo = dayjs.utc(to).endOf(timeBucket)
const iterateTo = djsTo > now ? now : djsTo
@@ -305,40 +301,54 @@ export class AnalyticsService {
return Promise.reject()
}
- // the database has to use UTC timezone for this to work normally
+ const x = []
+
while (groupDateIterator < iterateTo) {
const nextIteration = groupDateIterator.add(1, timeBucket)
- const temp = []
- const tempUnique = []
-
- clone = _filter(clone, el => {
- const createdAt = dayjs.utc(el.created)
- if (groupDateIterator <= createdAt && createdAt < nextIteration) {
- if (el.unique) {
- tempUnique.push(el)
- }
- temp.push(el)
- return false
- } else {
- return true
- }
- })
-
- res.push({
- data: temp,
- total: _size(temp),
- totalUnique: _size(tempUnique),
- timeFrame: groupDateIterator.format('YYYY-MM-DD HH:mm:ss'),
- })
+ x.push(groupDateIterator.format('YYYY-MM-DD HH:mm:ss'))
groupDateIterator = nextIteration
}
+ const xM = [...x, groupDateIterator.format('YYYY-MM-DD HH:mm:ss')]
+ let query = ''
+
+ for (let i = 0; i < _size(x); ++i) {
+ if (i > 0) {
+ query += ' UNION ALL '
+ }
+
+ query += `select ${i} index, unique, count() from analytics where pid='${pid}' and created between '${xM[i]}' and '${xM[1 + i]}' group by unique`
+ }
+
+ // @ts-ignore
+ const result = (await clickhouse.query(query).toPromise()).sort((a, b) => a.index - b.index)
+ const visits = []
+ const uniques = []
+
+ for (let i = 0; i < _size(result); i += 2) {
+ const v = result[i]['count()']
+ const u = result[1 + i]['count()']
+ // @ts-ignore
+ const index = result[i].index
+
+ visits[index] = v + u
+ uniques[index] = u
+ }
+
+ for (let i = 0; i < _size(x); ++i) {
+ if (!visits[i]) {
+ visits[i] = 0
+ uniques[i] = 0
+ }
+ }
+
+ // TODO: SEEMS LIKE IT RETURNS INCORRECT DATA FOR HOUR PERIOD
return Promise.resolve({
- params: this.processData(res),
+ params,
chart: {
- x: _map(res, el => el.timeFrame),
- visits: _map(res, el => el.total),
- uniques: _map(res, el => el.totalUnique),
+ x,
+ visits,
+ uniques,
},
})
}
diff --git a/src/common/constants.ts b/src/common/constants.ts
index a962bea93..3613032c0 100644
--- a/src/common/constants.ts
+++ b/src/common/constants.ts
@@ -29,7 +29,7 @@ const clickhouse = new ClickHouse({
format: 'json',
raw: false,
config: {
- session_id: 'nestjsAPIapp',
+ session_id: 'swetrixAPI',
session_timeout: 60,
output_format_json_quote_64bit_integers: 0,
enable_http_compression: 0,