📈 Open source, privacy-first web analytics — a cookie-less Google Analytics alternative with built-in error tracking and performance monitoring. Self-host it or use our cloud. Get started for free!
TypeScript chartsclickhousedockergdprgoogle-analyticsmetricsmonitoringmysqlnestjsprivacyredisself-hostedselfhostedstatisticstailwindcssweb-analyticswebanalyticswebsite
Performance optimisations
Blue Mousedea3550e803f
diff --git a/backend/apps/production/src/alert/alert.service.ts b/backend/apps/production/src/alert/alert.service.tsindex 2763e8131..ec592e99c 100644--- a/backend/apps/production/src/alert/alert.service.ts+++ b/backend/apps/production/src/alert/alert.service.ts@@ -76,15 +76,4 @@ export class AlertService {async delete(id: string): Promise<any> {return this.alertsReporsitory.delete(id)}-- async deleteMultiple(ids: string[]): Promise<any> {- return (- this.alertsReporsitory- .createQueryBuilder()- .delete()- // TODO: !!! Enforce Prepared Statements and Parameterization- .where(`id IN (${ids})`)- .execute()- )- }}diff --git a/backend/apps/production/src/analytics/analytics.controller.ts b/backend/apps/production/src/analytics/analytics.controller.tsindex 3813547b2..b09988d48 100644--- a/backend/apps/production/src/analytics/analytics.controller.ts+++ b/backend/apps/production/src/analytics/analytics.controller.ts@@ -1673,7 +1673,6 @@ export class AnalyticsController {safeTimezone,diff,)- // TODO change to Uint64; filters.const subQuery ='FROM monitor_responses WHERE monitorId = {monitorId:UInt64} AND created BETWEEN {groupFrom:String} AND {groupTo:String}'const paramsData = {diff --git a/backend/apps/production/src/analytics/analytics.service.ts b/backend/apps/production/src/analytics/analytics.service.tsindex 9a8c08f89..32713f68e 100644--- a/backend/apps/production/src/analytics/analytics.service.ts+++ b/backend/apps/production/src/analytics/analytics.service.ts@@ -358,7 +358,7 @@ export class AnalyticsService {async checkBillingAccess(pid: string) {const project = await this.projectService.getRedisProject(pid)- if (project.admin.dashboardBlockReason) {+ if (project.admin?.dashboardBlockReason) {throw new HttpException('The account that owns this site is currently suspended, this is because of a billing issue. Please resolve the issue to continue.',HttpStatus.PAYMENT_REQUIRED,@@ -3151,7 +3151,6 @@ export class AnalyticsService {query,query_params: paramsData.params,})- // TODO: TYPE.then(resultSet => resultSet.json<any>())const { avgResponseTime } = this.extractUptimeChartData(data, x)diff --git a/backend/apps/production/src/main.ts b/backend/apps/production/src/main.tsindex 949566a41..d3b3e8a76 100644--- a/backend/apps/production/src/main.ts+++ b/backend/apps/production/src/main.ts@@ -49,7 +49,6 @@ async function bootstrap() {res.header('Access-Control-Allow-Headers', 'Authorization, *')if (req.method === 'OPTIONS') {- // TODO: INVESTIGATE// eslint-disable-next-line @typescript-eslint/return-awaitreturn await res.sendStatus(204)}diff --git a/backend/apps/production/src/project/project.controller.ts b/backend/apps/production/src/project/project.controller.tsindex beff79db0..060cc4795 100644--- a/backend/apps/production/src/project/project.controller.ts+++ b/backend/apps/production/src/project/project.controller.ts@@ -1129,7 +1129,6 @@ export class ProjectController {relations: ['share', 'share.user'],})- // todo: maybe the update project should be used here instead of delete?await deleteProjectRedis(pid)return processProjectUser(updatedProject)} catch (e) {@@ -2196,7 +2195,6 @@ export class ProjectController {}this.projectService.allowedToView(project, userId, headers['x-password'])- // TODO add cachingreturn this.projectService.sendPredictAiRequest(params.projectId)}diff --git a/backend/apps/production/src/project/project.service.ts b/backend/apps/production/src/project/project.service.tsindex 64a4a84f0..275c3bf44 100644--- a/backend/apps/production/src/project/project.service.ts+++ b/backend/apps/production/src/project/project.service.ts@@ -314,35 +314,40 @@ export class ProjectService {let project: string | Project = await redis.get(pidKey)if (_isEmpty(project)) {- // todo: optimise the relations - select- // select only required columns- // https://stackoverflow.com/questions/59645009/how-to-return-only-some-columns-of-a-relations-with-typeorm- project = await this.findOne(pid, {- relations: ['admin'],- select: [- 'origins',- 'active',- 'admin',- 'public',- 'ipBlacklist',- 'botsProtectionLevel',- 'captchaSecretKey',- 'isCaptchaEnabled',- 'passwordHash',- 'isPasswordProtected',- ],- })- if (_isEmpty(project))+ project = await this.projectsRepository+ .createQueryBuilder('project')+ .leftJoinAndSelect('project.admin', 'admin')+ .select([+ 'project.origins',+ 'project.active',+ 'project.public',+ 'project.ipBlacklist',+ 'project.botsProtectionLevel',+ 'project.captchaSecretKey',+ 'project.isCaptchaEnabled',+ 'project.passwordHash',+ 'project.isPasswordProtected',+ 'admin.id',+ 'admin.roles',+ 'admin.dashboardBlockReason',+ 'admin.isAccountBillingSuspended',+ ])+ .where('project.id = :pid', { pid })+ .getOne()++ if (_isEmpty(project)) {throw new BadRequestException('The provided Project ID (pid) is incorrect',)+ }++ const share = await this.projectShareRepository+ .createQueryBuilder('share')+ .leftJoinAndSelect('share.user', 'user')+ .select(['share.role', 'user.id'])+ .where('share.project = :pid', { pid })+ .getMany()- const share = await this.findShare({- where: {- project: pid,- },- relations: ['user'],- })project = { ...project, share }await redis.set(@@ -424,15 +429,12 @@ export class ProjectService {return this.projectsRepository.delete(id)}- async deleteMultiple(pids: string): Promise<any> {- return (- this.projectsRepository- .createQueryBuilder()- .delete()- // TODO: !!! Enforce Prepared Statements and Parameterization- .where(`id IN (${pids})`)- .execute()- )+ async deleteMultiple(pids: string[]): Promise<any> {+ return this.projectsRepository+ .createQueryBuilder()+ .delete()+ .whereInIds(pids)+ .execute()}async deleteMultipleShare(where: string): Promise<any> {@@ -473,7 +475,10 @@ export class ProjectService {return this.projectsRepository.findOne(id, { relations })}- findOne(id: string, params: object = {}): Promise<Project | null> {+ findOne(+ id: string,+ params: FindOneOptions<Project> = {},+ ): Promise<Project | null> {return this.projectsRepository.findOne(id, params)}@@ -1511,7 +1516,6 @@ export class ProjectService {monitorId: number,monitorHttpRequestDto: HttpRequestOptions,) {- // TODO crete interface for that stuffawait this.monitorQueue.add('http-request',{ ...monitorHttpRequestDto, monitorId },diff --git a/backend/apps/production/src/task-manager/task-manager.service.ts b/backend/apps/production/src/task-manager/task-manager.service.tsindex 79bdeed7d..d021fd261 100644--- a/backend/apps/production/src/task-manager/task-manager.service.ts+++ b/backend/apps/production/src/task-manager/task-manager.service.ts@@ -256,7 +256,7 @@ const EMAIL_REPORTS_MAP = {dayjsParams: [3, 'M'],analyticsParam: '3M',},-}+} as const@Injectable()export class TaskManagerService {@@ -302,15 +302,8 @@ export class TaskManagerService {): Promise<void> {const params = EMAIL_REPORTS_MAP[reportFrequency]- const users = await this.userService.find({- where: {- reportFrequency,- planCode: Not(PlanCode.none),- dashboardBlockReason: IsNull(),- },- relations: ['projects'],- select: ['id', 'email'],- })+ const users = await this.userService.getReportUsers(reportFrequency)+const now = dayjs.utc().format('DD.MM.YYYY')const timeAgo = dayjs.utc()@@ -323,11 +316,6 @@ export class TaskManagerService {const promises = _map(users, async user => {const { id, email, projects } = user- // todo: move _size(projects) to query- if (_isEmpty(projects) || _isNull(projects) || _size(projects) > 50) {- return- }-const unsubscribeUrl = this.generateUnsubscribeUrl(id, 'user-reports')const ids = _map(projects, p => p.id)diff --git a/backend/apps/production/src/user/user.controller.ts b/backend/apps/production/src/user/user.controller.tsindex 259762903..f2bf9f76e 100644--- a/backend/apps/production/src/user/user.controller.ts+++ b/backend/apps/production/src/user/user.controller.ts@@ -331,7 +331,7 @@ export class UserController {`DELETE FROM captcha WHERE pid IN (${pids})`,]- await this.projectService.deleteMultiple(pids)+ await this.projectService.deleteMultiple(user.projects.map(el => el.id))const promises = _map(queries, async query =>clickhouse.command({@@ -384,7 +384,7 @@ export class UserController {`DELETE FROM error_statuses WHERE pid IN (${pids})`,`DELETE FROM captcha WHERE pid IN (${pids})`,]- await this.projectService.deleteMultiple(pids)+ await this.projectService.deleteMultiple(user.projects.map(el => el.id))const promises = _map(queries, async query =>clickhouse.command({query,diff --git a/backend/apps/production/src/user/user.service.ts b/backend/apps/production/src/user/user.service.tsindex 5da1452de..009825e52 100644--- a/backend/apps/production/src/user/user.service.ts+++ b/backend/apps/production/src/user/user.service.ts@@ -4,7 +4,7 @@ import {InternalServerErrorException,} from '@nestjs/common'import { InjectRepository } from '@nestjs/typeorm'-import { In, Repository } from 'typeorm'+import { FindManyOptions, In, IsNull, Not, Repository } from 'typeorm'import axios from 'axios'import CryptoJS from 'crypto-js'import dayjs from 'dayjs'@@ -30,6 +30,7 @@ import { DeleteFeedback } from './entities/delete-feedback.entity'import { UserGoogleDTO } from './dto/user-google.dto'import { UserGithubDTO } from './dto/user-github.dto'import { EMAIL_ACTION_ENCRYPTION_KEY } from '../common/constants'+import { ReportFrequency } from '../project/enums'const EUR = {symbol: '€',@@ -189,8 +190,8 @@ export class UserService {})}- find(params: any): Promise<User[]> {- return this.usersRepository.find(params)+ find(options: FindManyOptions<User>): Promise<User[]> {+ return this.usersRepository.find(options)}findWhere(where: Record<string, unknown>): Promise<User[]> {@@ -598,4 +599,35 @@ export class UserService {async findUserV2(id: string, select?: (keyof User)[]) {return this.usersRepository.findOne({ select, where: { id } })}++ async getReportUsers(+ reportFrequency:+ | ReportFrequency.WEEKLY+ | ReportFrequency.MONTHLY+ | ReportFrequency.QUARTERLY,+ ) {+ // First get the user IDs that match our criteria+ const userIds = await this.usersRepository+ .createQueryBuilder('user')+ .leftJoin('user.projects', 'p')+ .select('user.id')+ .where({+ reportFrequency,+ planCode: Not(PlanCode.none),+ dashboardBlockReason: IsNull(),+ })+ .groupBy('user.id')+ .having('COUNT(p.id) BETWEEN 1 AND 50')+ .getMany()++ // Then fetch those users with their projects+ return this.usersRepository+ .createQueryBuilder('user')+ .leftJoinAndSelect('user.projects', 'projects')+ .select(['user.id', 'user.email', 'projects.id', 'projects.name'])+ .where('user.id IN (:...userIds)', {+ userIds: userIds.map(u => u.id),+ })+ .getMany()+ }}diff --git a/backend/apps/selfhosted/src/main.ts b/backend/apps/selfhosted/src/main.tsindex d22d6aaf9..f36619ea6 100644--- a/backend/apps/selfhosted/src/main.ts+++ b/backend/apps/selfhosted/src/main.ts@@ -43,7 +43,6 @@ async function bootstrap() {res.header('Access-Control-Allow-Headers', 'Authorization, *')if (req.method === 'OPTIONS') {- // TODO: INVESTIGATE// eslint-disable-next-line @typescript-eslint/return-awaitreturn await res.sendStatus(204)}diff --git a/backend/apps/selfhosted/src/project/project.service.ts b/backend/apps/selfhosted/src/project/project.service.tsindex a9d38988e..9af7e06f6 100644--- a/backend/apps/selfhosted/src/project/project.service.ts+++ b/backend/apps/selfhosted/src/project/project.service.ts@@ -64,7 +64,7 @@ export const deleteProjectRedis = async (id: string) => {@Injectable()export class ProjectService {- async getRedisProject(pid: string): Promise<Project | null> {+ async getRedisProject(pid: string) {const pidKey = getRedisProjectKey(pid)let project: string | Project = await redis.get(pidKey)@@ -90,8 +90,7 @@ export class ProjectService {}}- // @ts-ignore- return project+ return project as Project | null}allowedToView(