Skip to content

📈 Open source, privacy-first web analytics — a cookie-less Google Analytics alternative with built-in error tracking and performance monitoring. Self-host it or use our cloud. Get started for free!

Performance optimisations

Blue Mousedea3550e803f
diff --git a/backend/apps/production/src/alert/alert.service.ts b/backend/apps/production/src/alert/alert.service.ts
index 2763e8131..ec592e99c 100644
--- a/backend/apps/production/src/alert/alert.service.ts
+++ b/backend/apps/production/src/alert/alert.service.ts
@@ -76,15 +76,4 @@ export class AlertService {
async delete(id: string): Promise<any> {
return this.alertsReporsitory.delete(id)
}
-
- async deleteMultiple(ids: string[]): Promise<any> {
- return (
- this.alertsReporsitory
- .createQueryBuilder()
- .delete()
- // TODO: !!! Enforce Prepared Statements and Parameterization
- .where(`id IN (${ids})`)
- .execute()
- )
- }
}
diff --git a/backend/apps/production/src/analytics/analytics.controller.ts b/backend/apps/production/src/analytics/analytics.controller.ts
index 3813547b2..b09988d48 100644
--- a/backend/apps/production/src/analytics/analytics.controller.ts
+++ b/backend/apps/production/src/analytics/analytics.controller.ts
@@ -1673,7 +1673,6 @@ export class AnalyticsController {
safeTimezone,
diff,
)
- // TODO change to Uint64; filters.
const subQuery =
'FROM monitor_responses WHERE monitorId = {monitorId:UInt64} AND created BETWEEN {groupFrom:String} AND {groupTo:String}'
const paramsData = {
diff --git a/backend/apps/production/src/analytics/analytics.service.ts b/backend/apps/production/src/analytics/analytics.service.ts
index 9a8c08f89..32713f68e 100644
--- a/backend/apps/production/src/analytics/analytics.service.ts
+++ b/backend/apps/production/src/analytics/analytics.service.ts
@@ -358,7 +358,7 @@ export class AnalyticsService {
async checkBillingAccess(pid: string) {
const project = await this.projectService.getRedisProject(pid)
- if (project.admin.dashboardBlockReason) {
+ if (project.admin?.dashboardBlockReason) {
throw new HttpException(
'The account that owns this site is currently suspended, this is because of a billing issue. Please resolve the issue to continue.',
HttpStatus.PAYMENT_REQUIRED,
@@ -3151,7 +3151,6 @@ export class AnalyticsService {
query,
query_params: paramsData.params,
})
- // TODO: TYPE
.then(resultSet => resultSet.json<any>())
const { avgResponseTime } = this.extractUptimeChartData(data, x)
diff --git a/backend/apps/production/src/main.ts b/backend/apps/production/src/main.ts
index 949566a41..d3b3e8a76 100644
--- a/backend/apps/production/src/main.ts
+++ b/backend/apps/production/src/main.ts
@@ -49,7 +49,6 @@ async function bootstrap() {
res.header('Access-Control-Allow-Headers', 'Authorization, *')
if (req.method === 'OPTIONS') {
- // TODO: INVESTIGATE
// eslint-disable-next-line @typescript-eslint/return-await
return await res.sendStatus(204)
}
diff --git a/backend/apps/production/src/project/project.controller.ts b/backend/apps/production/src/project/project.controller.ts
index beff79db0..060cc4795 100644
--- a/backend/apps/production/src/project/project.controller.ts
+++ b/backend/apps/production/src/project/project.controller.ts
@@ -1129,7 +1129,6 @@ export class ProjectController {
relations: ['share', 'share.user'],
})
- // todo: maybe the update project should be used here instead of delete?
await deleteProjectRedis(pid)
return processProjectUser(updatedProject)
} catch (e) {
@@ -2196,7 +2195,6 @@ export class ProjectController {
}
this.projectService.allowedToView(project, userId, headers['x-password'])
- // TODO add caching
return this.projectService.sendPredictAiRequest(params.projectId)
}
diff --git a/backend/apps/production/src/project/project.service.ts b/backend/apps/production/src/project/project.service.ts
index 64a4a84f0..275c3bf44 100644
--- a/backend/apps/production/src/project/project.service.ts
+++ b/backend/apps/production/src/project/project.service.ts
@@ -314,35 +314,40 @@ export class ProjectService {
let project: string | Project = await redis.get(pidKey)
if (_isEmpty(project)) {
- // todo: optimise the relations - select
- // select only required columns
- // https://stackoverflow.com/questions/59645009/how-to-return-only-some-columns-of-a-relations-with-typeorm
- project = await this.findOne(pid, {
- relations: ['admin'],
- select: [
- 'origins',
- 'active',
- 'admin',
- 'public',
- 'ipBlacklist',
- 'botsProtectionLevel',
- 'captchaSecretKey',
- 'isCaptchaEnabled',
- 'passwordHash',
- 'isPasswordProtected',
- ],
- })
- if (_isEmpty(project))
+ project = await this.projectsRepository
+ .createQueryBuilder('project')
+ .leftJoinAndSelect('project.admin', 'admin')
+ .select([
+ 'project.origins',
+ 'project.active',
+ 'project.public',
+ 'project.ipBlacklist',
+ 'project.botsProtectionLevel',
+ 'project.captchaSecretKey',
+ 'project.isCaptchaEnabled',
+ 'project.passwordHash',
+ 'project.isPasswordProtected',
+ 'admin.id',
+ 'admin.roles',
+ 'admin.dashboardBlockReason',
+ 'admin.isAccountBillingSuspended',
+ ])
+ .where('project.id = :pid', { pid })
+ .getOne()
+
+ if (_isEmpty(project)) {
throw new BadRequestException(
'The provided Project ID (pid) is incorrect',
)
+ }
+
+ const share = await this.projectShareRepository
+ .createQueryBuilder('share')
+ .leftJoinAndSelect('share.user', 'user')
+ .select(['share.role', 'user.id'])
+ .where('share.project = :pid', { pid })
+ .getMany()
- const share = await this.findShare({
- where: {
- project: pid,
- },
- relations: ['user'],
- })
project = { ...project, share }
await redis.set(
@@ -424,15 +429,12 @@ export class ProjectService {
return this.projectsRepository.delete(id)
}
- async deleteMultiple(pids: string): Promise<any> {
- return (
- this.projectsRepository
- .createQueryBuilder()
- .delete()
- // TODO: !!! Enforce Prepared Statements and Parameterization
- .where(`id IN (${pids})`)
- .execute()
- )
+ async deleteMultiple(pids: string[]): Promise<any> {
+ return this.projectsRepository
+ .createQueryBuilder()
+ .delete()
+ .whereInIds(pids)
+ .execute()
}
async deleteMultipleShare(where: string): Promise<any> {
@@ -473,7 +475,10 @@ export class ProjectService {
return this.projectsRepository.findOne(id, { relations })
}
- findOne(id: string, params: object = {}): Promise<Project | null> {
+ findOne(
+ id: string,
+ params: FindOneOptions<Project> = {},
+ ): Promise<Project | null> {
return this.projectsRepository.findOne(id, params)
}
@@ -1511,7 +1516,6 @@ export class ProjectService {
monitorId: number,
monitorHttpRequestDto: HttpRequestOptions,
) {
- // TODO crete interface for that stuff
await this.monitorQueue.add(
'http-request',
{ ...monitorHttpRequestDto, monitorId },
diff --git a/backend/apps/production/src/task-manager/task-manager.service.ts b/backend/apps/production/src/task-manager/task-manager.service.ts
index 79bdeed7d..d021fd261 100644
--- a/backend/apps/production/src/task-manager/task-manager.service.ts
+++ b/backend/apps/production/src/task-manager/task-manager.service.ts
@@ -256,7 +256,7 @@ const EMAIL_REPORTS_MAP = {
dayjsParams: [3, 'M'],
analyticsParam: '3M',
},
-}
+} as const
@Injectable()
export class TaskManagerService {
@@ -302,15 +302,8 @@ export class TaskManagerService {
): Promise<void> {
const params = EMAIL_REPORTS_MAP[reportFrequency]
- const users = await this.userService.find({
- where: {
- reportFrequency,
- planCode: Not(PlanCode.none),
- dashboardBlockReason: IsNull(),
- },
- relations: ['projects'],
- select: ['id', 'email'],
- })
+ const users = await this.userService.getReportUsers(reportFrequency)
+
const now = dayjs.utc().format('DD.MM.YYYY')
const timeAgo = dayjs
.utc()
@@ -323,11 +316,6 @@ export class TaskManagerService {
const promises = _map(users, async user => {
const { id, email, projects } = user
- // todo: move _size(projects) to query
- if (_isEmpty(projects) || _isNull(projects) || _size(projects) > 50) {
- return
- }
-
const unsubscribeUrl = this.generateUnsubscribeUrl(id, 'user-reports')
const ids = _map(projects, p => p.id)
diff --git a/backend/apps/production/src/user/user.controller.ts b/backend/apps/production/src/user/user.controller.ts
index 259762903..f2bf9f76e 100644
--- a/backend/apps/production/src/user/user.controller.ts
+++ b/backend/apps/production/src/user/user.controller.ts
@@ -331,7 +331,7 @@ export class UserController {
`DELETE FROM captcha WHERE pid IN (${pids})`,
]
- await this.projectService.deleteMultiple(pids)
+ await this.projectService.deleteMultiple(user.projects.map(el => el.id))
const promises = _map(queries, async query =>
clickhouse.command({
@@ -384,7 +384,7 @@ export class UserController {
`DELETE FROM error_statuses WHERE pid IN (${pids})`,
`DELETE FROM captcha WHERE pid IN (${pids})`,
]
- await this.projectService.deleteMultiple(pids)
+ await this.projectService.deleteMultiple(user.projects.map(el => el.id))
const promises = _map(queries, async query =>
clickhouse.command({
query,
diff --git a/backend/apps/production/src/user/user.service.ts b/backend/apps/production/src/user/user.service.ts
index 5da1452de..009825e52 100644
--- a/backend/apps/production/src/user/user.service.ts
+++ b/backend/apps/production/src/user/user.service.ts
@@ -4,7 +4,7 @@ import {
InternalServerErrorException,
} from '@nestjs/common'
import { InjectRepository } from '@nestjs/typeorm'
-import { In, Repository } from 'typeorm'
+import { FindManyOptions, In, IsNull, Not, Repository } from 'typeorm'
import axios from 'axios'
import CryptoJS from 'crypto-js'
import dayjs from 'dayjs'
@@ -30,6 +30,7 @@ import { DeleteFeedback } from './entities/delete-feedback.entity'
import { UserGoogleDTO } from './dto/user-google.dto'
import { UserGithubDTO } from './dto/user-github.dto'
import { EMAIL_ACTION_ENCRYPTION_KEY } from '../common/constants'
+import { ReportFrequency } from '../project/enums'
const EUR = {
symbol: '€',
@@ -189,8 +190,8 @@ export class UserService {
})
}
- find(params: any): Promise<User[]> {
- return this.usersRepository.find(params)
+ find(options: FindManyOptions<User>): Promise<User[]> {
+ return this.usersRepository.find(options)
}
findWhere(where: Record<string, unknown>): Promise<User[]> {
@@ -598,4 +599,35 @@ export class UserService {
async findUserV2(id: string, select?: (keyof User)[]) {
return this.usersRepository.findOne({ select, where: { id } })
}
+
+ async getReportUsers(
+ reportFrequency:
+ | ReportFrequency.WEEKLY
+ | ReportFrequency.MONTHLY
+ | ReportFrequency.QUARTERLY,
+ ) {
+ // First get the user IDs that match our criteria
+ const userIds = await this.usersRepository
+ .createQueryBuilder('user')
+ .leftJoin('user.projects', 'p')
+ .select('user.id')
+ .where({
+ reportFrequency,
+ planCode: Not(PlanCode.none),
+ dashboardBlockReason: IsNull(),
+ })
+ .groupBy('user.id')
+ .having('COUNT(p.id) BETWEEN 1 AND 50')
+ .getMany()
+
+ // Then fetch those users with their projects
+ return this.usersRepository
+ .createQueryBuilder('user')
+ .leftJoinAndSelect('user.projects', 'projects')
+ .select(['user.id', 'user.email', 'projects.id', 'projects.name'])
+ .where('user.id IN (:...userIds)', {
+ userIds: userIds.map(u => u.id),
+ })
+ .getMany()
+ }
}
diff --git a/backend/apps/selfhosted/src/main.ts b/backend/apps/selfhosted/src/main.ts
index d22d6aaf9..f36619ea6 100644
--- a/backend/apps/selfhosted/src/main.ts
+++ b/backend/apps/selfhosted/src/main.ts
@@ -43,7 +43,6 @@ async function bootstrap() {
res.header('Access-Control-Allow-Headers', 'Authorization, *')
if (req.method === 'OPTIONS') {
- // TODO: INVESTIGATE
// eslint-disable-next-line @typescript-eslint/return-await
return await res.sendStatus(204)
}
diff --git a/backend/apps/selfhosted/src/project/project.service.ts b/backend/apps/selfhosted/src/project/project.service.ts
index a9d38988e..9af7e06f6 100644
--- a/backend/apps/selfhosted/src/project/project.service.ts
+++ b/backend/apps/selfhosted/src/project/project.service.ts
@@ -64,7 +64,7 @@ export const deleteProjectRedis = async (id: string) => {
@Injectable()
export class ProjectService {
- async getRedisProject(pid: string): Promise<Project | null> {
+ async getRedisProject(pid: string) {
const pidKey = getRedisProjectKey(pid)
let project: string | Project = await redis.get(pidKey)
@@ -90,8 +90,7 @@ export class ProjectService {
}
}
- // @ts-ignore
- return project
+ return project as Project | null
}
allowedToView(