📈 Open source, privacy-first web analytics — a cookie-less Google Analytics alternative with built-in error tracking and performance monitoring. Self-host it or use our cloud. Get started for free!
TypeScript chartsclickhousedockergdprgoogle-analyticsmetricsmonitoringmysqlnestjsprivacyredisself-hostedselfhostedstatisticstailwindcssweb-analyticswebanalyticswebsite
more improvements and optimisations
Blue Mousec591ffad7368
diff --git a/src/analytics/analytics.controller.ts b/src/analytics/analytics.controller.tsindex 3599bf884..6d60172d8 100644--- a/src/analytics/analytics.controller.ts+++ b/src/analytics/analytics.controller.ts@@ -102,14 +102,14 @@ export class AnalyticsController {if (dayjs.utc(to).isAfter(dayjs.utc(), 'second')) {groupTo = dayjs.utc().format('YYYY-MM-DD HH:mm:ss')- queryCustoms += `AND created BETWEEN ${from} AND ${groupTo}`+ queryCustoms += ` AND created BETWEEN ${from} AND ${groupTo}`} else {- queryCustoms += `AND created BETWEEN ${from} AND ${to}`+ queryCustoms += ` AND created BETWEEN ${from} AND ${to}`}} else if (!_isEmpty(period)) {groupFrom = dayjs.utc().subtract(parseInt(period), _last(period)).format('YYYY-MM-DD')groupTo = dayjs.utc().format('YYYY-MM-DD 23:59:59')- queryCustoms += `AND created BETWEEN '${groupFrom}' AND '${groupTo}'`+ queryCustoms += ` AND created BETWEEN '${groupFrom}' AND '${groupTo}'`subQuery += ` AND created BETWEEN '${groupFrom}' AND '${groupTo}'`} else {throw new BadRequestException('The timeframe (either from/to pair or period) to be provided')@@ -118,6 +118,8 @@ export class AnalyticsController {const result = await this.analyticsService.groupByTimeBucket(timeBucket, groupFrom, groupTo, subQuery, pid)const customs = await clickhouse.query(queryCustoms).toPromise()+ // TODO: VERIFY THAT CUSTOM EVENTS WORK PROPERLY+ console.log(queryCustoms, _countBy(customs, 'ev'))console.log(customs)return {@@ -137,7 +139,7 @@ export class AnalyticsController {await this.analyticsService.checkProjectAccess(pidsArray[i], uid)}- return this.analyticsService.getSummary(pidsArray, 'w', true)+ return this.analyticsService.getSummary(pidsArray, 'w')}@UseGuards(SelfhostedGuard)diff --git a/src/analytics/analytics.service.ts b/src/analytics/analytics.service.tsindex f0387f295..f2a7f9cda 100644--- a/src/analytics/analytics.service.ts+++ b/src/analytics/analytics.service.ts@@ -7,8 +7,9 @@ import * as _isNull from 'lodash/isNull'import * as _includes from 'lodash/includes'import * as _map from 'lodash/map'import * as _join from 'lodash/join'-import * as _every from 'lodash/every'+import * as _some from 'lodash/some'import * as _keys from 'lodash/keys'+import * as _find from 'lodash/find'import * as _values from 'lodash/values'import * as dayjs from 'dayjs'import * as utc from 'dayjs/plugin/utc'@@ -38,6 +39,14 @@ const cols = ['cc', 'pg', 'lc', 'br', 'os', 'dv', 'ref','so', 'me', 'ca',]+interface chartCHResponse {+ index: number+ unique: number+ 'count()': number+}++const customEVvalidate = /^[a-zA-Z](?:[\w\.]){0,62}$/+@Injectable()export class AnalyticsService {constructor(@@ -148,8 +157,16 @@ export class AnalyticsService {const { pid } = logDTOthis.validatePID(pid)- if (logDTO instanceof EventsDTO && !_isEmpty(logDTO.ev) && _size(logDTO.ev) > 64) {- throw new BadRequestException('An incorrect event name (ev) is provided')+ if (logDTO instanceof EventsDTO) {+ const { ev } = logDTO++ if (_isEmpty(ev)) {+ throw new BadRequestException('Empty custom events are not allowed')+ }++ if (!customEVvalidate.test(ev)) {+ throw new BadRequestException('An incorrect event name (ev) is provided')+ }}const project = await this.getRedisProject(pid)@@ -196,8 +213,9 @@ export class AnalyticsService {return Boolean(session)}- async getSummary(pids: string[], period: 'w' | 'M' = 'w', advanced: boolean = false): Promise<Object> {+ async getSummary(pids: string[], period: 'w' | 'M' = 'w'): Promise<Object> {const result = {}+for (let i = 0; i < _size(pids); ++i) {const pid = pids[i]if (!isValidPID(pid)) throw new BadRequestException(`The provided Project ID (${pid}) is incorrect`)@@ -207,45 +225,25 @@ export class AnalyticsService {const oneWeek = oneWRaw.format('YYYY-MM-DD HH:mm:ss')const twoWeeks = oneWRaw.subtract(1, period).format('YYYY-MM-DD HH:mm:ss')- const query1_pageviews = `SELECT COUNT() FROM analytics WHERE pid='${pid}' AND created BETWEEN '${oneWeek}' AND '${now}'`- const query2_pageviews = `SELECT COUNT() FROM analytics WHERE pid='${pid}' AND created BETWEEN '${twoWeeks}' AND '${oneWeek}'`- const query1_unique = `SELECT COUNT() FROM analytics WHERE pid='${pid}' AND unique=1 AND created BETWEEN '${oneWeek}' AND '${now}'`- const query2_unique = `SELECT COUNT() FROM analytics WHERE pid='${pid}' AND unique=1 AND created BETWEEN '${twoWeeks}' AND '${oneWeek}'`-const query1 = `SELECT unique, count() FROM analytics WHERE pid='${pid}' AND created BETWEEN '${oneWeek}' AND '${now}' GROUP BY unique`const query2 = `SELECT unique, count() FROM analytics WHERE pid='${pid}' AND created BETWEEN '${twoWeeks}' AND '${oneWeek}' GROUP BY unique`-- const q1res = await clickhouse.query(query1).toPromise()- const q2res = await clickhouse.query(query2).toPromise()-- console.log(q1res, q2res)-+try {- const res1_pageviews = await clickhouse.query(query1_pageviews).toPromise()- const res2_pageviews = await clickhouse.query(query2_pageviews).toPromise()- const thisWeekPV = res1_pageviews[0]['count()']- const lastWeekPV = res2_pageviews[0]['count()']-- if (advanced) {- const res1_unique = await clickhouse.query(query1_unique).toPromise()- const res2_unique = await clickhouse.query(query2_unique).toPromise()- const thisWeekUnique = res1_unique[0]['count()']- const lastWeekUnique = res2_unique[0]['count()']-- result[pid] = {- thisWeek: thisWeekPV,- lastWeek: lastWeekPV,- thisWeekUnique,- lastWeekUnique,- percChange: getPercentageChange(thisWeekPV, lastWeekPV),- percChangeUnique: getPercentageChange(thisWeekUnique, lastWeekUnique),- }- } else {- result[pid] = {- thisWeek: thisWeekPV,- lastWeek: lastWeekPV,- percChange: getPercentageChange(thisWeekPV, lastWeekPV),- }+ const q1res = await clickhouse.query(query1).toPromise()+ const q2res = await clickhouse.query(query2).toPromise()++ const thisWeekUnique = _find(q1res, ({ unique }) => unique)?.['count()'] || 0+ const thisWeekPV = (_find(q1res, ({ unique }) => !unique)?.['count()'] || 0) + thisWeekUnique+ const lastWeekUnique = _find(q2res, ({ unique }) => unique)?.['count()'] || 0+ const lastWeekPV = (_find(q2res, ({ unique }) => !unique)?.['count()'] || 0) + lastWeekUnique++ result[pid] = {+ thisWeek: thisWeekPV,+ lastWeek: lastWeekPV,+ thisWeekUnique,+ lastWeekUnique,+ percChange: getPercentageChange(thisWeekPV, lastWeekPV),+ percChangeUnique: getPercentageChange(thisWeekUnique, lastWeekUnique),}} catch {throw new InternalServerErrorException('Can\'t process the provided PID. Please, try again later.')@@ -272,7 +270,7 @@ export class AnalyticsService {}}- if (_every(_values(params), _isEmpty)) {+ if (!_some(_values(params), (val) => !_isEmpty(val))) {return Promise.resolve()}@@ -321,18 +319,35 @@ export class AnalyticsService {}// @ts-ignore- const result = (await clickhouse.query(query).toPromise()).sort((a, b) => a.index - b.index)+ const result: Array<chartCHResponse> = (await clickhouse.query(query).toPromise()).sort((a, b) => a.index - b.index)const visits = []const uniques = []- for (let i = 0; i < _size(result); i += 2) {- const v = result[i]['count()']- const u = result[1 + i]['count()']- // @ts-ignore- const index = result[i].index+ let idx = 0+ const resSize = _size(result)- visits[index] = v + u- uniques[index] = u+ while (idx < resSize) {+ const index = result[idx].index+ const v = result[idx]['count()']++ if (index === result[1 + idx]?.index) {+ const u = result[1 + idx]['count()']+ visits[index] = v + u+ uniques[index] = u+ idx += 2+ continue+ }++ const unique = result[idx].unique++ if (unique) {+ visits[index] = v+ uniques[index] = v+ } else {+ visits[index] = v+ uniques[index] = 0+ }+ idx++}for (let i = 0; i < _size(x); ++i) {@@ -342,7 +357,6 @@ export class AnalyticsService {}}- // TODO: SEEMS LIKE IT RETURNS INCORRECT DATA FOR HOUR PERIODreturn Promise.resolve({params,chart: {diff --git a/src/project/entity/project.entity.ts b/src/project/entity/project.entity.tsindex 7202c1310..ae1b58c2f 100644--- a/src/project/entity/project.entity.ts+++ b/src/project/entity/project.entity.ts@@ -3,6 +3,7 @@ import { ApiProperty } from '@nestjs/swagger'import { User } from '../../user/entities/user.entity'+// In case of modifying some properties here, make sure to also edit them in common/constants.ts -> selfhosted -> clickhouse@Entity()export class Project {@ApiProperty()diff --git a/src/task-manager/task-manager.service.ts b/src/task-manager/task-manager.service.tsindex b898bda2f..d84bfa4a6 100644--- a/src/task-manager/task-manager.service.ts+++ b/src/task-manager/task-manager.service.ts@@ -89,7 +89,7 @@ export class TaskManagerService {}const ids = _map(users[i].projects, (p) => p.id)- const data = await this.analyticsService.getSummary(ids, 'w', true)+ const data = await this.analyticsService.getSummary(ids, 'w')const result = {type: 'w', // week@@ -130,7 +130,7 @@ export class TaskManagerService {}const ids = _map(users[i].projects, (p) => p.id)- const data = await this.analyticsService.getSummary(ids, 'M', true)+ const data = await this.analyticsService.getSummary(ids, 'M')const result = {type: 'M', // month