Skip to content

📈 Open source, privacy-first web analytics — a cookie-less Google Analytics alternative with built-in error tracking and performance monitoring. Self-host it or use our cloud. Get started for free!

move extra project functionality into a separate service

Blue Mouse8cbfa319988f
diff --git a/backend/apps/cloud/src/project/project-extra.service.ts b/backend/apps/cloud/src/project/project-extra.service.ts
new file mode 100644
index 000000000..60d7711b3
--- /dev/null
+++ b/backend/apps/cloud/src/project/project-extra.service.ts
@@ -0,0 +1,693 @@
+import { Injectable } from '@nestjs/common'
+import { InjectRepository } from '@nestjs/typeorm'
+import { Brackets, In, Repository } from 'typeorm'
+import dayjs from 'dayjs'
+import utc from 'dayjs/plugin/utc'
+
+import { Pagination, PaginationOptionsInterface } from '../common/pagination'
+import { Project } from './entity/project.entity'
+import { clickhouse } from '../common/integrations/clickhouse'
+import { ProjectService } from './project.service'
+
+dayjs.extend(utc)
+
+interface TrafficPaginationOptions extends PaginationOptionsInterface {
+ mode?: 'high-traffic' | 'low-traffic' | 'performance' | 'lost-traffic'
+ period?: '1h' | '1d' | '7d' | '4w' | '3M' | '12M' | '24M' | 'all'
+}
+
+@Injectable()
+export class ProjectExtraService {
+ constructor(
+ @InjectRepository(Project)
+ private projectsRepository: Repository<Project>,
+ private readonly projectService: ProjectService,
+ ) {}
+
+ async paginateHostnameNavigation(
+ options: TrafficPaginationOptions,
+ userId: string,
+ search?: string,
+ ): Promise<Pagination<Project>> {
+ const projectIds = await this.getVisibleProjectIds(userId)
+
+ if (!projectIds.length) {
+ return new Pagination<Project>({ results: [], total: 0 })
+ }
+
+ const timeFrameClause = this.getTimeFrameClause(options.period)
+
+ // Process project IDs in chunks to avoid overwhelming Clickhouse
+ const CHUNK_SIZE = 1000
+ const chunks = []
+ for (let i = 0; i < projectIds.length; i += CHUNK_SIZE) {
+ chunks.push(projectIds.slice(i, i + CHUNK_SIZE))
+ }
+
+ // Query to get hosts and their associated PIDs
+ const query = `
+ WITH hostStats AS (
+ SELECT
+ host,
+ pid,
+ count() as visits
+ FROM analytics
+ WHERE pid IN {pids:Array(String)}
+ AND host IS NOT NULL
+ AND host != ''
+ ${search ? `AND host ILIKE {search:String}` : ''}
+ AND created BETWEEN ${timeFrameClause.currentStart} AND ${timeFrameClause.currentEnd}
+ GROUP BY host, pid
+ ORDER BY visits DESC
+ )
+ SELECT *
+ FROM hostStats
+ LIMIT {limit:UInt32}
+ OFFSET {offset:UInt32}
+ `
+
+ // Execute query for each chunk and combine results
+ let allResults = []
+ let total = 0
+
+ for (const chunk of chunks) {
+ // eslint-disable-next-line no-await-in-loop
+ const result = await clickhouse.query({
+ query,
+ query_params: {
+ pids: chunk,
+ limit: options.take || 10,
+ offset: options.skip || 0,
+ search: `%${search}%`,
+ },
+ format: 'JSONEachRow',
+ })
+
+ // eslint-disable-next-line no-await-in-loop
+ const chunkResults = await result.json()
+ allResults = allResults.concat(chunkResults)
+
+ // Get total count for this chunk
+ const countQuery = `
+ WITH hostStats AS (
+ SELECT
+ host,
+ pid,
+ count() as visits
+ FROM analytics
+ WHERE pid IN {pids:Array(String)}
+ AND host IS NOT NULL
+ AND host != ''
+ ${search ? `AND host ILIKE {search:String}` : ''}
+ AND created BETWEEN ${timeFrameClause.currentStart} AND ${timeFrameClause.currentEnd}
+ GROUP BY host, pid
+ )
+ SELECT count() as total FROM hostStats
+ `
+
+ // eslint-disable-next-line no-await-in-loop
+ const countResult = await clickhouse.query({
+ query: countQuery,
+ query_params: {
+ pids: chunk,
+ search: `%${search}%`,
+ },
+ format: 'JSONEachRow',
+ })
+
+ // eslint-disable-next-line no-await-in-loop
+ const countData = await countResult.json<{ total: number }>()
+ total += countData[0]?.total || 0
+ }
+
+ // Apply pagination to combined results
+ const paginatedResults = allResults.slice(
+ options.skip || 0,
+ (options.skip || 0) + (options.take || 10),
+ )
+
+ const uniquePids = [...new Set(paginatedResults.map(r => r.pid))]
+
+ // Fetch full project details for the paginated results
+ const projects = await this.projectService.find({
+ where: { id: In(uniquePids) },
+ relations: [
+ 'admin',
+ 'share',
+ 'share.user',
+ 'organisation',
+ 'organisation.members',
+ 'organisation.members.user',
+ ],
+ })
+
+ const enrichedProjects = projects
+ .map(project => {
+ const chResults = paginatedResults.filter(r => r.pid === project.id)
+
+ return chResults.map(chResult => ({
+ ...project,
+ name: chResult.host || project.name,
+ trafficStats: {
+ visits: chResult.visits || 0,
+ },
+ }))
+ })
+ .flat()
+
+ const results = new Pagination<Project>({
+ results: enrichedProjects.sort(
+ (a, b) => (b.trafficStats?.visits || 0) - (a.trafficStats?.visits || 0),
+ ),
+ total,
+ })
+
+ return results
+ }
+
+ async getVisibleProjectIds(
+ userId: string,
+ search?: string,
+ ): Promise<string[]> {
+ const queryBuilder = this.projectsRepository
+ .createQueryBuilder('project')
+ .select('project.id')
+ .leftJoin('project.admin', 'admin')
+ .leftJoin('project.share', 'share')
+ .leftJoin('project.organisation', 'organisation')
+ .leftJoin('organisation.members', 'organisationMembers')
+ .where('project.isAnalyticsProject = true')
+ .where(
+ new Brackets(qb => {
+ qb.where('admin.id = :userId', { userId })
+ .orWhere(
+ new Brackets(qb2 => {
+ qb2
+ .where('share.user.id = :userId')
+ .andWhere('share.confirmed = true')
+ }),
+ )
+ .orWhere(
+ new Brackets(qb3 => {
+ qb3
+ .where('organisationMembers.user.id = :userId')
+ .andWhere('organisationMembers.confirmed = true')
+ }),
+ )
+ }),
+ )
+
+ if (search?.trim()) {
+ queryBuilder
+ .andWhere('project.name LIKE :search')
+ .setParameter('search', `%${search.trim()}%`)
+ }
+
+ const projects = await queryBuilder.getMany()
+ return projects.map(p => p.id)
+ }
+
+ async paginateByTraffic(
+ options: TrafficPaginationOptions,
+ userId: string,
+ search?: string,
+ isHostnameNavigationEnabled?: boolean,
+ ): Promise<Pagination<Project>> {
+ // Get all visible project IDs
+ const projectIds = await this.getVisibleProjectIds(
+ userId,
+ isHostnameNavigationEnabled ? undefined : search,
+ )
+
+ if (!projectIds.length) {
+ return new Pagination<Project>({ results: [], total: 0 })
+ }
+
+ // Process project IDs in chunks to avoid overwhelming Clickhouse
+ const CHUNK_SIZE = 1000
+ const chunks = []
+ for (let i = 0; i < projectIds.length; i += CHUNK_SIZE) {
+ chunks.push(projectIds.slice(i, i + CHUNK_SIZE))
+ }
+
+ // Build the appropriate Clickhouse query based on mode
+ const timeFrameClause = this.getTimeFrameClause(options.period)
+ let query = ''
+ let countQuery = ''
+
+ if (isHostnameNavigationEnabled) {
+ if (options.mode === 'performance') {
+ query = `
+ WITH
+ currentPeriod AS (
+ SELECT host, pid, count() as visits
+ FROM analytics
+ WHERE pid IN {pids:Array(String)}
+ AND created BETWEEN ${timeFrameClause.currentStart} AND ${timeFrameClause.currentEnd}
+ AND host IS NOT NULL
+ AND host != ''
+ ${search ? `AND host ILIKE {search:String}` : ''}
+ GROUP BY host, pid
+ ),
+ previousPeriod AS (
+ SELECT host, pid, count() as visits
+ FROM analytics
+ WHERE pid IN {pids:Array(String)}
+ AND created BETWEEN ${timeFrameClause.previousStart} AND ${timeFrameClause.previousEnd}
+ AND host IS NOT NULL
+ AND host != ''
+ ${search ? `AND host ILIKE {search:String}` : ''}
+ GROUP BY host, pid
+ ),
+ hostStats AS (
+ SELECT
+ cp.host,
+ cp.pid,
+ sum(cp.visits) as current_visits,
+ sum(pp.visits) as previous_visits
+ FROM currentPeriod cp
+ JOIN previousPeriod pp ON cp.host = pp.host AND cp.pid = pp.pid
+ GROUP BY cp.host, cp.pid
+ )
+ SELECT
+ host,
+ pid,
+ current_visits,
+ previous_visits,
+ ((current_visits - previous_visits) / previous_visits * 100) as percentage_change
+ FROM hostStats
+ WHERE previous_visits > 0
+ ORDER BY abs(percentage_change) DESC
+ LIMIT {limit:UInt32}
+ OFFSET {offset:UInt32}
+ `
+
+ countQuery = `
+ WITH
+ currentPeriod AS (
+ SELECT host, pid, count() as visits
+ FROM analytics
+ WHERE pid IN {pids:Array(String)}
+ AND created BETWEEN ${timeFrameClause.currentStart} AND ${timeFrameClause.currentEnd}
+ AND host IS NOT NULL
+ AND host != ''
+ ${search ? `AND host ILIKE {search:String}` : ''}
+ GROUP BY host, pid
+ ),
+ previousPeriod AS (
+ SELECT host, pid, count() as visits
+ FROM analytics
+ WHERE pid IN {pids:Array(String)}
+ AND created BETWEEN ${timeFrameClause.previousStart} AND ${timeFrameClause.previousEnd}
+ AND host IS NOT NULL
+ AND host != ''
+ ${search ? `AND host ILIKE {search:String}` : ''}
+ GROUP BY host, pid
+ ),
+ hostStats AS (
+ SELECT
+ cp.host,
+ cp.pid
+ FROM currentPeriod cp
+ JOIN previousPeriod pp ON cp.host = pp.host AND cp.pid = pp.pid
+ WHERE pp.visits > 0
+ )
+ SELECT count() as total FROM hostStats
+ `
+ } else if (options.mode === 'lost-traffic') {
+ query = `
+ WITH last_visits AS (
+ SELECT
+ host,
+ pid,
+ max(created) as last_visit,
+ count() as total_visits
+ FROM analytics
+ WHERE pid IN {pids:Array(String)}
+ AND host IS NOT NULL
+ AND host != ''
+ ${search ? `AND host ILIKE {search:String}` : ''}
+ GROUP BY host, pid
+ ),
+ hostStats AS (
+ SELECT
+ host,
+ pid,
+ max(last_visit) as last_visit,
+ sum(total_visits) as total_visits
+ FROM last_visits
+ GROUP BY host, pid
+ )
+ SELECT *
+ FROM hostStats
+ WHERE last_visit < (now() - INTERVAL 48 HOUR)
+ AND total_visits > 0
+ ORDER BY last_visit DESC
+ LIMIT {limit:UInt32}
+ OFFSET {offset:UInt32}
+ `
+
+ countQuery = `
+ WITH last_visits AS (
+ SELECT
+ host,
+ pid,
+ max(created) as last_visit,
+ count() as total_visits
+ FROM analytics
+ WHERE pid IN {pids:Array(String)}
+ AND host IS NOT NULL
+ AND host != ''
+ ${search ? `AND host ILIKE {search:String}` : ''}
+ GROUP BY host, pid
+ ),
+ hostStats AS (
+ SELECT
+ host,
+ pid
+ FROM last_visits
+ WHERE last_visit < (now() - INTERVAL 48 HOUR)
+ AND total_visits > 0
+ )
+ SELECT count() as total FROM hostStats
+ `
+ } else {
+ // High/Low traffic query
+ query = `
+ WITH hostStats AS (
+ SELECT
+ host,
+ pid,
+ count() as visits
+ FROM analytics
+ WHERE pid IN {pids:Array(String)}
+ AND created BETWEEN ${timeFrameClause.currentStart} AND ${timeFrameClause.currentEnd}
+ AND host IS NOT NULL
+ AND host != ''
+ ${search ? `AND host ILIKE {search:String}` : ''}
+ GROUP BY host, pid
+ )
+ SELECT *
+ FROM hostStats
+ ORDER BY visits ${options.mode === 'high-traffic' ? 'DESC' : 'ASC'}
+ LIMIT {limit:UInt32}
+ OFFSET {offset:UInt32}
+ `
+
+ countQuery = `
+ WITH hostStats AS (
+ SELECT
+ host,
+ pid
+ FROM analytics
+ WHERE pid IN {pids:Array(String)}
+ AND created BETWEEN ${timeFrameClause.currentStart} AND ${timeFrameClause.currentEnd}
+ AND host IS NOT NULL
+ AND host != ''
+ ${search ? `AND host ILIKE {search:String}` : ''}
+ GROUP BY host, pid
+ )
+ SELECT count() as total FROM hostStats
+ `
+ }
+ } else if (options.mode === 'performance') {
+ query = `
+ WITH
+ currentPeriod AS (
+ SELECT pid, count() as visits
+ FROM analytics
+ WHERE pid IN {pids:Array(String)}
+ AND created BETWEEN ${timeFrameClause.currentStart} AND ${timeFrameClause.currentEnd}
+ GROUP BY pid
+ ),
+ previousPeriod AS (
+ SELECT pid, count() as visits
+ FROM analytics
+ WHERE pid IN {pids:Array(String)}
+ AND created BETWEEN ${timeFrameClause.previousStart} AND ${timeFrameClause.previousEnd}
+ GROUP BY pid
+ )
+ SELECT
+ cp.pid,
+ cp.visits as current_visits,
+ pp.visits as previous_visits,
+ ((cp.visits - pp.visits) / pp.visits * 100) as percentage_change
+ FROM currentPeriod cp
+ JOIN previousPeriod pp ON cp.pid = pp.pid
+ WHERE pp.visits > 0
+ ORDER BY abs(percentage_change) DESC
+ LIMIT {limit:UInt32}
+ OFFSET {offset:UInt32}
+ `
+
+ countQuery = `
+ WITH
+ currentPeriod AS (
+ SELECT pid, count() as visits
+ FROM analytics
+ WHERE pid IN {pids:Array(String)}
+ AND created BETWEEN ${timeFrameClause.currentStart} AND ${timeFrameClause.currentEnd}
+ GROUP BY pid
+ ),
+ previousPeriod AS (
+ SELECT pid, count() as visits
+ FROM analytics
+ WHERE pid IN {pids:Array(String)}
+ AND created BETWEEN ${timeFrameClause.previousStart} AND ${timeFrameClause.previousEnd}
+ GROUP BY pid
+ ),
+ projectStats AS (
+ SELECT
+ cp.pid
+ FROM currentPeriod cp
+ JOIN previousPeriod pp ON cp.pid = pp.pid
+ WHERE pp.visits > 0
+ )
+ SELECT count() as total FROM projectStats
+ `
+ } else if (options.mode === 'lost-traffic') {
+ query = `
+ WITH last_visits AS (
+ SELECT
+ pid,
+ max(created) as last_visit,
+ count() as total_visits
+ FROM analytics
+ WHERE pid IN {pids:Array(String)}
+ GROUP BY pid
+ )
+ SELECT *
+ FROM last_visits
+ WHERE last_visit < (now() - INTERVAL 48 HOUR)
+ AND total_visits > 0
+ ORDER BY last_visit DESC
+ LIMIT {limit:UInt32}
+ OFFSET {offset:UInt32}
+ `
+
+ countQuery = `
+ WITH last_visits AS (
+ SELECT
+ pid,
+ max(created) as last_visit,
+ count() as total_visits
+ FROM analytics
+ WHERE pid IN {pids:Array(String)}
+ GROUP BY pid
+ HAVING last_visit < (now() - INTERVAL 48 HOUR)
+ AND total_visits > 0
+ )
+ SELECT count() as total FROM last_visits
+ `
+ } else {
+ // High/Low traffic query
+ query = `
+ SELECT
+ pid,
+ count() as visits
+ FROM analytics
+ WHERE pid IN {pids:Array(String)}
+ AND created BETWEEN ${timeFrameClause.currentStart} AND ${timeFrameClause.currentEnd}
+ GROUP BY pid
+ ORDER BY visits ${options.mode === 'high-traffic' ? 'DESC' : 'ASC'}
+ LIMIT {limit:UInt32}
+ OFFSET {offset:UInt32}
+ `
+
+ countQuery = `
+ WITH projectStats AS (
+ SELECT
+ pid
+ FROM analytics
+ WHERE pid IN {pids:Array(String)}
+ AND created BETWEEN ${timeFrameClause.currentStart} AND ${timeFrameClause.currentEnd}
+ GROUP BY pid
+ )
+ SELECT count() as total FROM projectStats
+ `
+ }
+
+ // Execute query for each chunk and combine results
+ let allResults = []
+ let total = 0
+
+ for (const chunk of chunks) {
+ // eslint-disable-next-line no-await-in-loop
+ const result = await clickhouse.query({
+ query,
+ query_params: {
+ pids: chunk,
+ limit: options.take || 10,
+ offset: options.skip || 0,
+ search: `%${search}%`,
+ },
+ format: 'JSONEachRow',
+ })
+
+ // eslint-disable-next-line no-await-in-loop
+ const chunkResults = await result.json()
+ allResults = allResults.concat(chunkResults)
+
+ // eslint-disable-next-line no-await-in-loop
+ const countResult = await clickhouse.query({
+ query: countQuery,
+ query_params: {
+ pids: chunk,
+ search: `%${search}%`,
+ },
+ format: 'JSONEachRow',
+ })
+
+ // eslint-disable-next-line no-await-in-loop
+ const countData = await countResult.json<{ total: number }>()
+ total += countData[0]?.total || 0
+ }
+
+ // Apply pagination to combined results
+ const paginatedResults = allResults.slice(
+ options.skip || 0,
+ (options.skip || 0) + (options.take || 10),
+ )
+
+ const uniquePids = [...new Set(paginatedResults.map(r => r.pid))]
+
+ // Fetch full project details for the paginated results
+ const projects = await this.projectService.find({
+ where: { id: In(uniquePids) },
+ relations: [
+ 'admin',
+ 'share',
+ 'share.user',
+ 'organisation',
+ 'organisation.members',
+ 'organisation.members.user',
+ ],
+ })
+
+ // Add traffic stats to projects
+ const enrichedProjects = projects
+ .map(project => {
+ const chResults = paginatedResults.filter(r => r.pid === project.id)
+
+ return chResults.map(chResult => ({
+ ...project,
+ name: chResult.host || project.name,
+ trafficStats: {
+ visits: chResult.visits || chResult.current_visits || 0,
+ percentageChange: chResult.percentage_change,
+ },
+ }))
+ })
+ .flat()
+
+ const results = new Pagination<Project>({
+ results: enrichedProjects.sort((a, b) => {
+ if (options.mode === 'performance') {
+ return (
+ Math.abs(b.trafficStats.percentageChange) -
+ Math.abs(a.trafficStats.percentageChange)
+ )
+ }
+
+ if (options.mode === 'high-traffic') {
+ return (b.trafficStats.visits || 0) - (a.trafficStats.visits || 0)
+ }
+
+ if (options.mode === 'low-traffic') {
+ return (a.trafficStats.visits || 0) - (b.trafficStats.visits || 0)
+ }
+
+ return (b.trafficStats.visits || 0) - (a.trafficStats.visits || 0)
+ }),
+ total,
+ })
+
+ return this.projectService.processDefaultResults(results, userId)
+ }
+
+ private getTimeFrameClause(period: TrafficPaginationOptions['period']): {
+ currentStart: string
+ currentEnd: string
+ previousStart?: string
+ previousEnd?: string
+ } {
+ const now = 'now()'
+
+ switch (period) {
+ case '1h':
+ return {
+ currentStart: `${now} - INTERVAL 1 HOUR`,
+ currentEnd: now,
+ previousStart: `${now} - INTERVAL 2 HOUR`,
+ previousEnd: `${now} - INTERVAL 1 HOUR`,
+ }
+ case '1d':
+ return {
+ currentStart: `${now} - INTERVAL 1 DAY`,
+ currentEnd: now,
+ previousStart: `${now} - INTERVAL 2 DAY`,
+ previousEnd: `${now} - INTERVAL 1 DAY`,
+ }
+ case '7d':
+ return {
+ currentStart: `${now} - INTERVAL 7 DAY`,
+ currentEnd: now,
+ previousStart: `${now} - INTERVAL 14 DAY`,
+ previousEnd: `${now} - INTERVAL 7 DAY`,
+ }
+ case '4w':
+ return {
+ currentStart: `${now} - INTERVAL 28 DAY`,
+ currentEnd: now,
+ previousStart: `${now} - INTERVAL 56 DAY`,
+ previousEnd: `${now} - INTERVAL 28 DAY`,
+ }
+ case '3M':
+ return {
+ currentStart: `${now} - INTERVAL 90 DAY`,
+ currentEnd: now,
+ previousStart: `${now} - INTERVAL 180 DAY`,
+ previousEnd: `${now} - INTERVAL 90 DAY`,
+ }
+ case '12M':
+ return {
+ currentStart: `${now} - INTERVAL 365 DAY`,
+ currentEnd: now,
+ previousStart: `${now} - INTERVAL 730 DAY`,
+ previousEnd: `${now} - INTERVAL 365 DAY`,
+ }
+ case '24M':
+ return {
+ currentStart: `${now} - INTERVAL 730 DAY`,
+ currentEnd: now,
+ previousStart: `${now} - INTERVAL 1460 DAY`,
+ previousEnd: `${now} - INTERVAL 730 DAY`,
+ }
+ default: // all time
+ return {
+ currentStart: "toDate('2020-01-01')", // or your earliest date
+ currentEnd: now,
+ }
+ }
+ }
+}
diff --git a/backend/apps/cloud/src/project/project.controller.ts b/backend/apps/cloud/src/project/project.controller.ts
index 0009bb965..34bcb89b9 100644
--- a/backend/apps/cloud/src/project/project.controller.ts
+++ b/backend/apps/cloud/src/project/project.controller.ts
@@ -115,6 +115,7 @@ import { BulkAddUsersResponse } from './interfaces/bulk-add-users'
import { OrganisationService } from '../organisation/organisation.service'
import { Organisation } from '../organisation/entity/organisation.entity'
import { ProjectOrganisationDto } from './dto/project-organisation.dto'
+import { ProjectExtraService } from './project-extra.service'
const PROJECTS_MAXIMUM = 50
@@ -137,6 +138,7 @@ export class ProjectController {
private readonly mailerService: MailerService,
private readonly projectsViewsRepository: ProjectsViewsRepository,
private readonly organisationService: OrganisationService,
+ private readonly projectExtraService: ProjectExtraService,
) {}
@ApiBearerAuth()
@@ -193,7 +195,7 @@ export class ProjectController {
let paginated: Pagination<Project>
if (isHostnameNavigationEnabled) {
- paginated = await this.projectService.paginateHostnameNavigation(
+ paginated = await this.projectExtraService.paginateHostnameNavigation(
{ take, skip, period },
userId,
search,
@@ -209,7 +211,7 @@ export class ProjectController {
return this.projectService.processDefaultResults(paginated, userId)
}
- return this.projectService.paginateByTraffic(
+ return this.projectExtraService.paginateByTraffic(
{
take,
skip,
diff --git a/backend/apps/cloud/src/project/project.module.ts b/backend/apps/cloud/src/project/project.module.ts
index 27d6aab3c..ec67dae55 100644
--- a/backend/apps/cloud/src/project/project.module.ts
+++ b/backend/apps/cloud/src/project/project.module.ts
@@ -16,6 +16,7 @@ import { ProjectViewCustomEventEntity } from './entity/project-view-custom-event
import { MonitorConsumer } from './consumers/monitor.consumer'
import { MonitorEntity } from './entity/monitor.entity'
import { OrganisationModule } from '../organisation/organisation.module'
+import { ProjectExtraService } from './project-extra.service'
@Module({
imports: [
@@ -54,8 +55,13 @@ import { OrganisationModule } from '../organisation/organisation.module'
ActionTokensModule,
MailerModule,
],
- providers: [ProjectService, ProjectsViewsRepository, MonitorConsumer],
- exports: [ProjectService, ProjectsViewsRepository],
+ providers: [
+ ProjectService,
+ ProjectExtraService,
+ ProjectsViewsRepository,
+ MonitorConsumer,
+ ],
+ exports: [ProjectService, ProjectExtraService, ProjectsViewsRepository],
controllers: [ProjectController],
})
export class ProjectModule {}
diff --git a/backend/apps/cloud/src/project/project.service.ts b/backend/apps/cloud/src/project/project.service.ts
index 23f88cd81..48e11db22 100644
--- a/backend/apps/cloud/src/project/project.service.ts
+++ b/backend/apps/cloud/src/project/project.service.ts
@@ -16,7 +16,6 @@ import {
DeepPartial,
Brackets,
FindOptionsWhere,
- In,
} from 'typeorm'
import { customAlphabet } from 'nanoid'
import handlebars from 'handlebars'
@@ -305,11 +304,6 @@ const getCompiledHTML = (title: string, desc: string, styles: string) => {
})
}
-interface TrafficPaginationOptions extends PaginationOptionsInterface {
- mode?: 'high-traffic' | 'low-traffic' | 'performance' | 'lost-traffic'
- period?: '1h' | '1d' | '7d' | '4w' | '3M' | '12M' | '24M' | 'all'
-}
-
@Injectable()
export class ProjectService {
constructor(
@@ -455,147 +449,6 @@ export class ProjectService {
return this.projectsRepository.findOne(query)
}
- async paginateHostnameNavigation(
- options: TrafficPaginationOptions,
- userId: string,
- search?: string,
- ): Promise<Pagination<Project>> {
- const projectIds = await this.getVisibleProjectIds(userId)
-
- if (!projectIds.length) {
- return new Pagination<Project>({ results: [], total: 0 })
- }
-
- const timeFrameClause = this.getTimeFrameClause(options.period)
-
- // Process project IDs in chunks to avoid overwhelming Clickhouse
- const CHUNK_SIZE = 1000
- const chunks = []
- for (let i = 0; i < projectIds.length; i += CHUNK_SIZE) {
- chunks.push(projectIds.slice(i, i + CHUNK_SIZE))
- }
-
- // Query to get hosts and their associated PIDs
- const query = `
- WITH hostStats AS (
- SELECT
- host,
- pid,
- count() as visits
- FROM analytics
- WHERE pid IN {pids:Array(String)}
- AND host IS NOT NULL
- AND host != ''
- ${search ? `AND host ILIKE {search:String}` : ''}
- AND created BETWEEN ${timeFrameClause.currentStart} AND ${timeFrameClause.currentEnd}
- GROUP BY host, pid
- ORDER BY visits DESC
- )
- SELECT *
- FROM hostStats
- LIMIT {limit:UInt32}
- OFFSET {offset:UInt32}
- `
-
- // Execute query for each chunk and combine results
- let allResults = []
- let total = 0
-
- for (const chunk of chunks) {
- // eslint-disable-next-line no-await-in-loop
- const result = await clickhouse.query({
- query,
- query_params: {
- pids: chunk,
- limit: options.take || 10,
- offset: options.skip || 0,
- search: `%${search}%`,
- },
- format: 'JSONEachRow',
- })
-
- // eslint-disable-next-line no-await-in-loop
- const chunkResults = await result.json()
- allResults = allResults.concat(chunkResults)
-
- // Get total count for this chunk
- const countQuery = `
- WITH hostStats AS (
- SELECT
- host,
- pid,
- count() as visits
- FROM analytics
- WHERE pid IN {pids:Array(String)}
- AND host IS NOT NULL
- AND host != ''
- ${search ? `AND host ILIKE {search:String}` : ''}
- AND created BETWEEN ${timeFrameClause.currentStart} AND ${timeFrameClause.currentEnd}
- GROUP BY host, pid
- )
- SELECT count() as total FROM hostStats
- `
-
- // eslint-disable-next-line no-await-in-loop
- const countResult = await clickhouse.query({
- query: countQuery,
- query_params: {
- pids: chunk,
- search: `%${search}%`,
- },
- format: 'JSONEachRow',
- })
-
- // eslint-disable-next-line no-await-in-loop
- const countData = await countResult.json<{ total: number }>()
- total += countData[0]?.total || 0
- }
-
- // Apply pagination to combined results
- const paginatedResults = allResults.slice(
- options.skip || 0,
- (options.skip || 0) + (options.take || 10),
- )
-
- const uniquePids = [...new Set(paginatedResults.map(r => r.pid))]
-
- // Fetch full project details for the paginated results
- const projects = await this.projectsRepository.find({
- where: { id: In(uniquePids) },
- relations: [
- 'admin',
- 'share',
- 'share.user',
- 'organisation',
- 'organisation.members',
- 'organisation.members.user',
- ],
- })
-
- const enrichedProjects = projects
- .map(project => {
- const chResults = paginatedResults.filter(r => r.pid === project.id)
-
- return chResults.map(chResult => ({
- ...project,
- name: chResult.host || project.name,
- trafficStats: {
- visits: chResult.visits || 0,
- },
- }))
- })
- .flat()
-
- const results = new Pagination<Project>({
- results: enrichedProjects.sort(
- (a, b) => (b.trafficStats?.visits || 0) - (a.trafficStats?.visits || 0),
- ),
- total,
- })
-
- return results
- }
-
async paginate(
options: PaginationOptionsInterface,
userId: string,
@@ -1884,532 +1737,6 @@ export class ProjectService {
} as Project)
}
- async getVisibleProjectIds(
- userId: string,
- search?: string,
- ): Promise<string[]> {
- const queryBuilder = this.projectsRepository
- .createQueryBuilder('project')
- .select('project.id')
- .leftJoin('project.admin', 'admin')
- .leftJoin('project.share', 'share')
- .leftJoin('project.organisation', 'organisation')
- .leftJoin('organisation.members', 'organisationMembers')
- .where('project.isAnalyticsProject = true')
- .where(
- new Brackets(qb => {
- qb.where('admin.id = :userId', { userId })
- .orWhere(
- new Brackets(qb2 => {
- qb2
- .where('share.user.id = :userId')
- .andWhere('share.confirmed = true')
- }),
- )
- .orWhere(
- new Brackets(qb3 => {
- qb3
- .where('organisationMembers.user.id = :userId')
- .andWhere('organisationMembers.confirmed = true')
- }),
- )
- }),
- )
-
- if (search?.trim()) {
- queryBuilder
- .andWhere('project.name LIKE :search')
- .setParameter('search', `%${search.trim()}%`)
- }
-
- const projects = await queryBuilder.getMany()
- return projects.map(p => p.id)
- }
-
- async paginateByTraffic(
- options: TrafficPaginationOptions,
- userId: string,
- search?: string,
- isHostnameNavigationEnabled?: boolean,
- ): Promise<Pagination<Project>> {
- // Get all visible project IDs
- const projectIds = await this.getVisibleProjectIds(
- userId,
- isHostnameNavigationEnabled ? undefined : search,
- )
-
- if (!projectIds.length) {
- return new Pagination<Project>({ results: [], total: 0 })
- }
-
- // Process project IDs in chunks to avoid overwhelming Clickhouse
- const CHUNK_SIZE = 1000
- const chunks = []
- for (let i = 0; i < projectIds.length; i += CHUNK_SIZE) {
- chunks.push(projectIds.slice(i, i + CHUNK_SIZE))
- }
-
- // Build the appropriate Clickhouse query based on mode
- const timeFrameClause = this.getTimeFrameClause(options.period)
- let query = ''
- let countQuery = ''
-
- if (isHostnameNavigationEnabled) {
- if (options.mode === 'performance') {
- query = `
- WITH
- currentPeriod AS (
- SELECT host, pid, count() as visits
- FROM analytics
- WHERE pid IN {pids:Array(String)}
- AND created BETWEEN ${timeFrameClause.currentStart} AND ${timeFrameClause.currentEnd}
- AND host IS NOT NULL
- AND host != ''
- ${search ? `AND host ILIKE {search:String}` : ''}
- GROUP BY host, pid
- ),
- previousPeriod AS (
- SELECT host, pid, count() as visits
- FROM analytics
- WHERE pid IN {pids:Array(String)}
- AND created BETWEEN ${timeFrameClause.previousStart} AND ${timeFrameClause.previousEnd}
- AND host IS NOT NULL
- AND host != ''
- ${search ? `AND host ILIKE {search:String}` : ''}
- GROUP BY host, pid
- ),
- hostStats AS (
- SELECT
- cp.host,
- cp.pid,
- sum(cp.visits) as current_visits,
- sum(pp.visits) as previous_visits
- FROM currentPeriod cp
- JOIN previousPeriod pp ON cp.host = pp.host AND cp.pid = pp.pid
- GROUP BY cp.host, cp.pid
- )
- SELECT
- host,
- pid,
- current_visits,
- previous_visits,
- ((current_visits - previous_visits) / previous_visits * 100) as percentage_change
- FROM hostStats
- WHERE previous_visits > 0
- ORDER BY abs(percentage_change) DESC
- LIMIT {limit:UInt32}
- OFFSET {offset:UInt32}
- `
-
- countQuery = `
- WITH
- currentPeriod AS (
- SELECT host, pid, count() as visits
- FROM analytics
- WHERE pid IN {pids:Array(String)}
- AND created BETWEEN ${timeFrameClause.currentStart} AND ${timeFrameClause.currentEnd}
- AND host IS NOT NULL
- AND host != ''
- ${search ? `AND host ILIKE {search:String}` : ''}
- GROUP BY host, pid
- ),
- previousPeriod AS (
- SELECT host, pid, count() as visits
- FROM analytics
- WHERE pid IN {pids:Array(String)}
- AND created BETWEEN ${timeFrameClause.previousStart} AND ${timeFrameClause.previousEnd}
- AND host IS NOT NULL
- AND host != ''
- ${search ? `AND host ILIKE {search:String}` : ''}
- GROUP BY host, pid
- ),
- hostStats AS (
- SELECT
- cp.host,
- cp.pid
- FROM currentPeriod cp
- JOIN previousPeriod pp ON cp.host = pp.host AND cp.pid = pp.pid
- WHERE pp.visits > 0
- )
- SELECT count() as total FROM hostStats
- `
- } else if (options.mode === 'lost-traffic') {
- query = `
- WITH last_visits AS (
- SELECT
- host,
- pid,
- max(created) as last_visit,
- count() as total_visits
- FROM analytics
- WHERE pid IN {pids:Array(String)}
- AND host IS NOT NULL
- AND host != ''
- ${search ? `AND host ILIKE {search:String}` : ''}
- GROUP BY host, pid
- ),
- hostStats AS (
- SELECT
- host,
- pid,
- max(last_visit) as last_visit,
- sum(total_visits) as total_visits
- FROM last_visits
- GROUP BY host, pid
- )
- SELECT *
- FROM hostStats
- WHERE last_visit < (now() - INTERVAL 48 HOUR)
- AND total_visits > 0
- ORDER BY last_visit DESC
- LIMIT {limit:UInt32}
- OFFSET {offset:UInt32}
- `
-
- countQuery = `
- WITH last_visits AS (
- SELECT
- host,
- pid,
- max(created) as last_visit,
- count() as total_visits
- FROM analytics
- WHERE pid IN {pids:Array(String)}
- AND host IS NOT NULL
- AND host != ''
- ${search ? `AND host ILIKE {search:String}` : ''}
- GROUP BY host, pid
- ),
- hostStats AS (
- SELECT
- host,
- pid
- FROM last_visits
- WHERE last_visit < (now() - INTERVAL 48 HOUR)
- AND total_visits > 0
- )
- SELECT count() as total FROM hostStats
- `
- } else {
- // High/Low traffic query
- query = `
- WITH hostStats AS (
- SELECT
- host,
- pid,
- count() as visits
- FROM analytics
- WHERE pid IN {pids:Array(String)}
- AND created BETWEEN ${timeFrameClause.currentStart} AND ${timeFrameClause.currentEnd}
- AND host IS NOT NULL
- AND host != ''
- ${search ? `AND host ILIKE {search:String}` : ''}
- GROUP BY host, pid
- )
- SELECT *
- FROM hostStats
- ORDER BY visits ${options.mode === 'high-traffic' ? 'DESC' : 'ASC'}
- LIMIT {limit:UInt32}
- OFFSET {offset:UInt32}
- `
-
- countQuery = `
- WITH hostStats AS (
- SELECT
- host,
- pid
- FROM analytics
- WHERE pid IN {pids:Array(String)}
- AND created BETWEEN ${timeFrameClause.currentStart} AND ${timeFrameClause.currentEnd}
- AND host IS NOT NULL
- AND host != ''
- ${search ? `AND host ILIKE {search:String}` : ''}
- GROUP BY host, pid
- )
- SELECT count() as total FROM hostStats
- `
- }
- } else if (options.mode === 'performance') {
- query = `
- WITH
- currentPeriod AS (
- SELECT pid, count() as visits
- FROM analytics
- WHERE pid IN {pids:Array(String)}
- AND created BETWEEN ${timeFrameClause.currentStart} AND ${timeFrameClause.currentEnd}
- GROUP BY pid
- ),
- previousPeriod AS (
- SELECT pid, count() as visits
- FROM analytics
- WHERE pid IN {pids:Array(String)}
- AND created BETWEEN ${timeFrameClause.previousStart} AND ${timeFrameClause.previousEnd}
- GROUP BY pid
- )
- SELECT
- cp.pid,
- cp.visits as current_visits,
- pp.visits as previous_visits,
- ((cp.visits - pp.visits) / pp.visits * 100) as percentage_change
- FROM currentPeriod cp
- JOIN previousPeriod pp ON cp.pid = pp.pid
- WHERE pp.visits > 0
- ORDER BY abs(percentage_change) DESC
- LIMIT {limit:UInt32}
- OFFSET {offset:UInt32}
- `
-
- countQuery = `
- WITH
- currentPeriod AS (
- SELECT pid, count() as visits
- FROM analytics
- WHERE pid IN {pids:Array(String)}
- AND created BETWEEN ${timeFrameClause.currentStart} AND ${timeFrameClause.currentEnd}
- GROUP BY pid
- ),
- previousPeriod AS (
- SELECT pid, count() as visits
- FROM analytics
- WHERE pid IN {pids:Array(String)}
- AND created BETWEEN ${timeFrameClause.previousStart} AND ${timeFrameClause.previousEnd}
- GROUP BY pid
- ),
- projectStats AS (
- SELECT
- cp.pid
- FROM currentPeriod cp
- JOIN previousPeriod pp ON cp.pid = pp.pid
- WHERE pp.visits > 0
- )
- SELECT count() as total FROM projectStats
- `
- } else if (options.mode === 'lost-traffic') {
- query = `
- WITH last_visits AS (
- SELECT
- pid,
- max(created) as last_visit,
- count() as total_visits
- FROM analytics
- WHERE pid IN {pids:Array(String)}
- GROUP BY pid
- )
- SELECT *
- FROM last_visits
- WHERE last_visit < (now() - INTERVAL 48 HOUR)
- AND total_visits > 0
- ORDER BY last_visit DESC
- LIMIT {limit:UInt32}
- OFFSET {offset:UInt32}
- `
-
- countQuery = `
- WITH last_visits AS (
- SELECT
- pid,
- max(created) as last_visit,
- count() as total_visits
- FROM analytics
- WHERE pid IN {pids:Array(String)}
- GROUP BY pid
- HAVING last_visit < (now() - INTERVAL 48 HOUR)
- AND total_visits > 0
- )
- SELECT count() as total FROM last_visits
- `
- } else {
- // High/Low traffic query
- query = `
- SELECT
- pid,
- count() as visits
- FROM analytics
- WHERE pid IN {pids:Array(String)}
- AND created BETWEEN ${timeFrameClause.currentStart} AND ${timeFrameClause.currentEnd}
- GROUP BY pid
- ORDER BY visits ${options.mode === 'high-traffic' ? 'DESC' : 'ASC'}
- LIMIT {limit:UInt32}
- OFFSET {offset:UInt32}
- `
-
- countQuery = `
- WITH projectStats AS (
- SELECT
- pid
- FROM analytics
- WHERE pid IN {pids:Array(String)}
- AND created BETWEEN ${timeFrameClause.currentStart} AND ${timeFrameClause.currentEnd}
- GROUP BY pid
- )
- SELECT count() as total FROM projectStats
- `
- }
-
- // Execute query for each chunk and combine results
- let allResults = []
- let total = 0
-
- for (const chunk of chunks) {
- // eslint-disable-next-line no-await-in-loop
- const result = await clickhouse.query({
- query,
- query_params: {
- pids: chunk,
- limit: options.take || 10,
- offset: options.skip || 0,
- search: `%${search}%`,
- },
- format: 'JSONEachRow',
- })
-
- // eslint-disable-next-line no-await-in-loop
- const chunkResults = await result.json()
- allResults = allResults.concat(chunkResults)
-
- // eslint-disable-next-line no-await-in-loop
- const countResult = await clickhouse.query({
- query: countQuery,
- query_params: {
- pids: chunk,
- search: `%${search}%`,
- },
- format: 'JSONEachRow',
- })
-
- // eslint-disable-next-line no-await-in-loop
- const countData = await countResult.json<{ total: number }>()
- total += countData[0]?.total || 0
- }
-
- // Apply pagination to combined results
- const paginatedResults = allResults.slice(
- options.skip || 0,
- (options.skip || 0) + (options.take || 10),
- )
-
- const uniquePids = [...new Set(paginatedResults.map(r => r.pid))]
-
- // Fetch full project details for the paginated results
- const projects = await this.projectsRepository.find({
- where: { id: In(uniquePids) },
- relations: [
- 'admin',
- 'share',
- 'share.user',
- 'organisation',
- 'organisation.members',
- 'organisation.members.user',
- ],
- })
-
- // Add traffic stats to projects
- const enrichedProjects = projects
- .map(project => {
- const chResults = paginatedResults.filter(r => r.pid === project.id)
-
- return chResults.map(chResult => ({
- ...project,
- name: chResult.host || project.name,
- trafficStats: {
- visits: chResult.visits || chResult.current_visits || 0,
- percentageChange: chResult.percentage_change,
- },
- }))
- })
- .flat()
-
- const results = new Pagination<Project>({
- results: enrichedProjects.sort((a, b) => {
- if (options.mode === 'performance') {
- return (
- Math.abs(b.trafficStats.percentageChange) -
- Math.abs(a.trafficStats.percentageChange)
- )
- }
-
- if (options.mode === 'high-traffic') {
- return (b.trafficStats.visits || 0) - (a.trafficStats.visits || 0)
- }
-
- if (options.mode === 'low-traffic') {
- return (a.trafficStats.visits || 0) - (b.trafficStats.visits || 0)
- }
-
- return (b.trafficStats.visits || 0) - (a.trafficStats.visits || 0)
- }),
- total,
- })
-
- return this.processDefaultResults(results, userId)
- }
-
- private getTimeFrameClause(period: TrafficPaginationOptions['period']): {
- currentStart: string
- currentEnd: string
- previousStart?: string
- previousEnd?: string
- } {
- const now = 'now()'
-
- switch (period) {
- case '1h':
- return {
- currentStart: `${now} - INTERVAL 1 HOUR`,
- currentEnd: now,
- previousStart: `${now} - INTERVAL 2 HOUR`,
- previousEnd: `${now} - INTERVAL 1 HOUR`,
- }
- case '1d':
- return {
- currentStart: `${now} - INTERVAL 1 DAY`,
- currentEnd: now,
- previousStart: `${now} - INTERVAL 2 DAY`,
- previousEnd: `${now} - INTERVAL 1 DAY`,
- }
- case '7d':
- return {
- currentStart: `${now} - INTERVAL 7 DAY`,
- currentEnd: now,
- previousStart: `${now} - INTERVAL 14 DAY`,
- previousEnd: `${now} - INTERVAL 7 DAY`,
- }
- case '4w':
- return {
- currentStart: `${now} - INTERVAL 28 DAY`,
- currentEnd: now,
- previousStart: `${now} - INTERVAL 56 DAY`,
- previousEnd: `${now} - INTERVAL 28 DAY`,
- }
- case '3M':
- return {
- currentStart: `${now} - INTERVAL 90 DAY`,
- currentEnd: now,
- previousStart: `${now} - INTERVAL 180 DAY`,
- previousEnd: `${now} - INTERVAL 90 DAY`,
- }
- case '12M':
- return {
- currentStart: `${now} - INTERVAL 365 DAY`,
- currentEnd: now,
- previousStart: `${now} - INTERVAL 730 DAY`,
- previousEnd: `${now} - INTERVAL 365 DAY`,
- }
- case '24M':
- return {
- currentStart: `${now} - INTERVAL 730 DAY`,
- currentEnd: now,
- previousStart: `${now} - INTERVAL 1460 DAY`,
- previousEnd: `${now} - INTERVAL 730 DAY`,
- }
- default: // all time
- return {
- currentStart: "toDate('2020-01-01')", // or your earliest date
- currentEnd: now,
- }
- }
- }
-
async processDefaultResults(paginated: Pagination<Project>, userId: string) {
const pidsWithData = await this.getPIDsWhereAnalyticsDataExists(
_map(paginated.results, ({ id }) => id),