📈 Open source, privacy-first web analytics — a cookie-less Google Analytics alternative with built-in error tracking and performance monitoring. Self-host it or use our cloud. Get started for free!
TypeScript chartsclickhousedockergdprgoogle-analyticsmetricsmonitoringmysqlnestjsprivacyredisself-hostedselfhostedstatisticstailwindcssweb-analyticswebanalyticswebsite
Bots protection setting: selfhosted support
Blue Mouse91d28aa06a1b
diff --git a/backend/apps/production/src/project/project.service.ts b/backend/apps/production/src/project/project.service.tsindex 13c47910e..a51d41ca0 100644--- a/backend/apps/production/src/project/project.service.ts+++ b/backend/apps/production/src/project/project.service.ts@@ -965,7 +965,7 @@ export class ProjectService {query,query_params: params,})- .then(resultSet => resultSet.json())+ .then(resultSet => resultSet.json<{ pid: string }>())return _map(data, ({ pid }) => pid)}@@ -1014,7 +1014,7 @@ export class ProjectService {query,query_params: params,})- .then(resultSet => resultSet.json())+ .then(resultSet => resultSet.json<{ pid: string }>())return _map(data, ({ pid }) => pid)}diff --git a/backend/apps/selfhosted/src/analytics/analytics.controller.ts b/backend/apps/selfhosted/src/analytics/analytics.controller.tsindex a9f73b63c..cd86de215 100644--- a/backend/apps/selfhosted/src/analytics/analytics.controller.ts+++ b/backend/apps/selfhosted/src/analytics/analytics.controller.ts@@ -24,7 +24,6 @@ import {} from '@nestjs/common'import { ApiTags } from '@nestjs/swagger'import UAParser from 'ua-parser-js'-import { isbot } from 'isbot'import { OptionalJwtAccessTokenGuard } from '../auth/guards'import { Auth, Public } from '../auth/decorators'@@ -54,8 +53,6 @@ import {getGeoDetails,getIPFromHeaders,} from '../common/utils'-import { BotDetection } from '../common/decorators/bot-detection.decorator'-import { BotDetectionGuard } from '../common/guards/bot-detection.guard'import { GetCustomEventsDto } from './dto/get-custom-events.dto'import {IFunnel,@@ -77,6 +74,7 @@ import {trafficTransformer,} from './utils/transformers'import { MAX_METRICS_IN_VIEW } from '../project/dto/create-project-view.dto'+import { extensions } from './utils/ua-parser'dayjs.extend(utc)dayjs.extend(dayjsTimezone)@@ -919,10 +917,7 @@ export class AnalyticsController {return data}- // Log custom event@Post('custom')- @UseGuards(BotDetectionGuard)- @BotDetection()@Public()async logCustom(@Body() eventsDTO: EventsDTO,@@ -934,6 +929,8 @@ export class AnalyticsController {const ip = getIPFromHeaders(headers) || reqIP || ''this.analyticsService.validateCustomEVMeta(eventsDTO.meta)++ await this.analyticsService.throwIfBot(eventsDTO.pid, userAgent)await this.analyticsService.validate(eventsDTO, origin, 'custom', ip)if (eventsDTO.unique) {@@ -950,7 +947,7 @@ export class AnalyticsController {}}- const ua = UAParser(userAgent)+ const ua = UAParser(userAgent, extensions)const dv = ua.device.type || 'desktop'const br = ua.browser.nameconst os = ua.os.name@@ -1002,8 +999,6 @@ export class AnalyticsController {}@Post('hb')- @UseGuards(BotDetectionGuard)- @BotDetection()@Auth([], true, true)async heartbeat(@Body() logDTO: PageviewsDTO,@@ -1014,6 +1009,8 @@ export class AnalyticsController {const { pid } = logDTOconst ip = getIPFromHeaders(headers) || reqIP || ''+ await this.analyticsService.throwIfBot(logDTO.pid, userAgent)+await this.analyticsService.validateHeartbeat(logDTO, origin, ip)const [, psid] = await this.analyticsService.isUnique(pid, userAgent, ip)@@ -1027,10 +1024,7 @@ export class AnalyticsController {await this.analyticsService.processInteractionSD(psid, pid)}- // Log pageview event@Post()- @UseGuards(BotDetectionGuard)- @BotDetection()@Public()async log(@Body() logDTO: PageviewsDTO,@@ -1041,6 +1035,8 @@ export class AnalyticsController {const ip = getIPFromHeaders(headers) || reqIP || ''+ await this.analyticsService.throwIfBot(logDTO.pid, userAgent)+await this.analyticsService.validate(logDTO, origin, 'log', ip)const [unique, psid] = await this.analyticsService.isUnique(@@ -1059,7 +1055,7 @@ export class AnalyticsController {const { city, region, country } = getGeoDetails(ip, logDTO.tz, headers)- const ua = UAParser(userAgent)+ const ua = UAParser(userAgent, extensions)const dv = ua.device.type || 'desktop'const br = ua.browser.nameconst os = ua.os.name@@ -1161,8 +1157,9 @@ export class AnalyticsController {const { 'user-agent': userAgent, origin } = headersconst { pid } = data- // todo: create a decorator for bot traffic detection- if (isbot(userAgent)) {+ try {+ await this.analyticsService.throwIfBot(pid, userAgent)+ } catch {res.writeHead(200, { 'Content-Type': 'image/gif' })return res.end(TRANSPARENT_GIF_BUFFER, 'binary')}@@ -1182,7 +1179,7 @@ export class AnalyticsController {await this.analyticsService.processInteractionSD(psid, logDTO.pid)- const ua = UAParser(userAgent)+ const ua = UAParser(userAgent, extensions)const dv = ua.device.type || 'desktop'const br = ua.browser.nameconst os = ua.os.name@@ -1471,10 +1468,7 @@ export class AnalyticsController {return this.analyticsService.getErrorsFilters(pid, type)}- // Log error event@Post('error')- @UseGuards(BotDetectionGuard)- @BotDetection()@Public()async logError(@Body() errorDTO: ErrorDTO,@@ -1485,11 +1479,13 @@ export class AnalyticsController {const ip = getIPFromHeaders(headers) || reqIP || ''+ await this.analyticsService.throwIfBot(errorDTO.pid, userAgent)+await this.analyticsService.validate(errorDTO, origin, 'error', ip)const { city, region, country } = getGeoDetails(ip, errorDTO.tz, headers)- const ua = UAParser(userAgent)+ const ua = UAParser(userAgent, extensions)const dv = ua.device.type || 'desktop'const br = ua.browser.nameconst os = ua.os.namediff --git a/backend/apps/selfhosted/src/analytics/analytics.service.ts b/backend/apps/selfhosted/src/analytics/analytics.service.tsindex 103acba30..36b939410 100644--- a/backend/apps/selfhosted/src/analytics/analytics.service.ts+++ b/backend/apps/selfhosted/src/analytics/analytics.service.ts@@ -35,6 +35,7 @@ import {UnprocessableEntityException,PreconditionFailedException,} from '@nestjs/common'+import { isbot } from 'isbot'import { DEFAULT_TIMEZONE } from '../user/entities/user.entity'import {@@ -60,7 +61,7 @@ import {import { PageviewsDTO } from './dto/pageviews.dto'import { EventsDTO } from './dto/events.dto'import { ProjectService } from '../project/project.service'-import { Project } from '../project/entity/project.entity'+import { BotsProtectionLevel, Project } from '../project/entity/project.entity'import { GetCustomEventMetadata } from './dto/get-custom-event-meta.dto'import { TimeBucketType, ChartRenderMode } from './dto/getData.dto'import {@@ -352,6 +353,18 @@ export class AnalyticsService {this.projectService.allowedToView(project, uid, password)}+ async throwIfBot(pid: string, userAgent: string) {+ const project = await this.projectService.getRedisProject(pid)++ if (project.botsProtectionLevel === BotsProtectionLevel.OFF) {+ return+ }++ if (isbot(userAgent)) {+ throw new BadRequestException('Bot traffic detected')+ }+ }+checkOrigin(project: Project, origin: string): void {// For some reasons the project.origins sometimes may look like [''], let's filter it out// TODO: Properly validate the origins on project updatediff --git a/backend/apps/selfhosted/src/analytics/utils/ua-parser.ts b/backend/apps/selfhosted/src/analytics/utils/ua-parser.tsnew file mode 100644index 000000000..240d91473--- /dev/null+++ b/backend/apps/selfhosted/src/analytics/utils/ua-parser.ts@@ -0,0 +1,12 @@+import UAParser from 'ua-parser-js'++const CUSTOM_BROWSERS = [+ [/(wget|curl|lynx|GameVault)\/([\w.]+)/i],+ [UAParser.BROWSER.NAME, UAParser.BROWSER.VERSION],+]++const extensions = {+ browser: CUSTOM_BROWSERS,+}++export { extensions }diff --git a/backend/apps/selfhosted/src/common/utils.ts b/backend/apps/selfhosted/src/common/utils.tsindex 22b7b4622..87230c565 100644--- a/backend/apps/selfhosted/src/common/utils.ts+++ b/backend/apps/selfhosted/src/common/utils.ts@@ -28,7 +28,7 @@ import {} from './constants'import { clickhouse } from './integrations/clickhouse'import { DEFAULT_TIMEZONE, TimeFormat } from '../user/entities/user.entity'-import { Project } from '../project/entity/project.entity'+import { BotsProtectionLevel, Project } from '../project/entity/project.entity'import { ProjectViewEntity } from '../project/entity/project-view.entity'import { ClickhouseFunnel, ClickhouseSFUser } from './types'@@ -50,6 +50,7 @@ const ALLOWED_KEYS = ['public','isPasswordProtected','passwordHash',+ 'botsProtectionLevel',]const ALLOWED_FUNNEL_KEYS = ['name', 'steps']@@ -306,6 +307,7 @@ const createProjectClickhouse = async (project: Partial<Project>) => {public: 0,isPasswordProtected: 0,passwordHash: null,+ botsProtectionLevel: BotsProtectionLevel.BASIC,created: dayjs.utc().format('YYYY-MM-DD HH:mm:ss'),},],diff --git a/backend/apps/selfhosted/src/project/dto/project.dto.ts b/backend/apps/selfhosted/src/project/dto/project.dto.tsindex 38f36e056..f3991c75d 100644--- a/backend/apps/selfhosted/src/project/dto/project.dto.ts+++ b/backend/apps/selfhosted/src/project/dto/project.dto.ts@@ -1,5 +1,6 @@import { ApiProperty } from '@nestjs/swagger'-import { IsNotEmpty, Length } from 'class-validator'+import { IsEnum, IsNotEmpty, IsOptional, Length } from 'class-validator'+import { BotsProtectionLevel } from '../entity'export class ProjectDTO {@ApiProperty({@@ -26,6 +27,15 @@ export class ProjectDTO {})origins: string[] | null+ @ApiProperty({+ enum: BotsProtectionLevel,+ required: false,+ description: 'Bots protection level',+ })+ @IsEnum(BotsProtectionLevel)+ @IsOptional()+ botsProtectionLevel?: BotsProtectionLevel+@ApiProperty({example: '::1,127.0.0.1,192.168.0.1/32',required: false,diff --git a/backend/apps/selfhosted/src/project/entity/project.entity.ts b/backend/apps/selfhosted/src/project/entity/project.entity.tsindex 420ee292c..b160a1acf 100644--- a/backend/apps/selfhosted/src/project/entity/project.entity.ts+++ b/backend/apps/selfhosted/src/project/entity/project.entity.ts@@ -1,3 +1,8 @@+export enum BotsProtectionLevel {+ OFF = 'off',+ BASIC = 'basic',+}+export class Project {id: string@@ -13,6 +18,8 @@ export class Project {isPasswordProtected: boolean+ botsProtectionLevel: BotsProtectionLevel+passwordHash?: stringcreated: Datediff --git a/backend/apps/selfhosted/src/project/project.controller.ts b/backend/apps/selfhosted/src/project/project.controller.tsindex f519f3939..4e880c258 100644--- a/backend/apps/selfhosted/src/project/project.controller.ts+++ b/backend/apps/selfhosted/src/project/project.controller.ts@@ -503,6 +503,10 @@ export class ProjectController {project.ipBlacklist = []}+ if (projectDTO.botsProtectionLevel) {+ project.botsProtectionLevel = projectDTO.botsProtectionLevel+ }+if (projectDTO.name) {project.name = _trim(projectDTO.name)}diff --git a/backend/apps/selfhosted/src/project/project.service.ts b/backend/apps/selfhosted/src/project/project.service.tsindex ea458b271..83e3875e1 100644--- a/backend/apps/selfhosted/src/project/project.service.ts+++ b/backend/apps/selfhosted/src/project/project.service.ts@@ -188,7 +188,7 @@ export class ProjectService {query,query_params: params,})- .then(resultSet => resultSet.json())+ .then(resultSet => resultSet.json<{ pid: string }>())return _map(data, ({ pid }) => pid)}@@ -237,7 +237,7 @@ export class ProjectService {query,query_params: params,})- .then(resultSet => resultSet.json())+ .then(resultSet => resultSet.json<{ pid: string }>())return _map(data, ({ pid }) => pid)}diff --git a/backend/migrations/clickhouse/initialise_selfhosted.js b/backend/migrations/clickhouse/initialise_selfhosted.jsindex 201421164..805f02e71 100644--- a/backend/migrations/clickhouse/initialise_selfhosted.js+++ b/backend/migrations/clickhouse/initialise_selfhosted.js@@ -12,6 +12,7 @@ const CLICKHOUSE_INIT_QUERIES = [active Int8,public Int8,isPasswordProtected Int8,+ botsProtectionLevel String DEFAULT 'basic',passwordHash Nullable(String),created DateTime)diff --git a/backend/migrations/clickhouse/selfhosted_2024_10_17.js b/backend/migrations/clickhouse/selfhosted_2024_10_17.jsnew file mode 100644index 000000000..d14f85283--- /dev/null+++ b/backend/migrations/clickhouse/selfhosted_2024_10_17.js@@ -0,0 +1,8 @@+const { queriesRunner, dbName } = require('./setup')++const queries = [+ // Added 'botsProtectionLevel' column+ `ALTER TABLE ${dbName}.project ADD COLUMN botsProtectionLevel String DEFAULT 'basic' AFTER isPasswordProtected;`,+]++queriesRunner(queries)