Skip to content

📈 Open source, privacy-first web analytics — a cookie-less Google Analytics alternative with built-in error tracking and performance monitoring. Self-host it or use our cloud. Get started for free!

Bots protection setting: selfhosted support

Blue Mouse91d28aa06a1b
diff --git a/backend/apps/production/src/project/project.service.ts b/backend/apps/production/src/project/project.service.ts
index 13c47910e..a51d41ca0 100644
--- a/backend/apps/production/src/project/project.service.ts
+++ b/backend/apps/production/src/project/project.service.ts
@@ -965,7 +965,7 @@ export class ProjectService {
query,
query_params: params,
})
- .then(resultSet => resultSet.json())
+ .then(resultSet => resultSet.json<{ pid: string }>())
return _map(data, ({ pid }) => pid)
}
@@ -1014,7 +1014,7 @@ export class ProjectService {
query,
query_params: params,
})
- .then(resultSet => resultSet.json())
+ .then(resultSet => resultSet.json<{ pid: string }>())
return _map(data, ({ pid }) => pid)
}
diff --git a/backend/apps/selfhosted/src/analytics/analytics.controller.ts b/backend/apps/selfhosted/src/analytics/analytics.controller.ts
index a9f73b63c..cd86de215 100644
--- a/backend/apps/selfhosted/src/analytics/analytics.controller.ts
+++ b/backend/apps/selfhosted/src/analytics/analytics.controller.ts
@@ -24,7 +24,6 @@ import {
} from '@nestjs/common'
import { ApiTags } from '@nestjs/swagger'
import UAParser from 'ua-parser-js'
-import { isbot } from 'isbot'
import { OptionalJwtAccessTokenGuard } from '../auth/guards'
import { Auth, Public } from '../auth/decorators'
@@ -54,8 +53,6 @@ import {
getGeoDetails,
getIPFromHeaders,
} from '../common/utils'
-import { BotDetection } from '../common/decorators/bot-detection.decorator'
-import { BotDetectionGuard } from '../common/guards/bot-detection.guard'
import { GetCustomEventsDto } from './dto/get-custom-events.dto'
import {
IFunnel,
@@ -77,6 +74,7 @@ import {
trafficTransformer,
} from './utils/transformers'
import { MAX_METRICS_IN_VIEW } from '../project/dto/create-project-view.dto'
+import { extensions } from './utils/ua-parser'
dayjs.extend(utc)
dayjs.extend(dayjsTimezone)
@@ -919,10 +917,7 @@ export class AnalyticsController {
return data
}
- // Log custom event
@Post('custom')
- @UseGuards(BotDetectionGuard)
- @BotDetection()
@Public()
async logCustom(
@Body() eventsDTO: EventsDTO,
@@ -934,6 +929,8 @@ export class AnalyticsController {
const ip = getIPFromHeaders(headers) || reqIP || ''
this.analyticsService.validateCustomEVMeta(eventsDTO.meta)
+
+ await this.analyticsService.throwIfBot(eventsDTO.pid, userAgent)
await this.analyticsService.validate(eventsDTO, origin, 'custom', ip)
if (eventsDTO.unique) {
@@ -950,7 +947,7 @@ export class AnalyticsController {
}
}
- const ua = UAParser(userAgent)
+ const ua = UAParser(userAgent, extensions)
const dv = ua.device.type || 'desktop'
const br = ua.browser.name
const os = ua.os.name
@@ -1002,8 +999,6 @@ export class AnalyticsController {
}
@Post('hb')
- @UseGuards(BotDetectionGuard)
- @BotDetection()
@Auth([], true, true)
async heartbeat(
@Body() logDTO: PageviewsDTO,
@@ -1014,6 +1009,8 @@ export class AnalyticsController {
const { pid } = logDTO
const ip = getIPFromHeaders(headers) || reqIP || ''
+ await this.analyticsService.throwIfBot(logDTO.pid, userAgent)
+
await this.analyticsService.validateHeartbeat(logDTO, origin, ip)
const [, psid] = await this.analyticsService.isUnique(pid, userAgent, ip)
@@ -1027,10 +1024,7 @@ export class AnalyticsController {
await this.analyticsService.processInteractionSD(psid, pid)
}
- // Log pageview event
@Post()
- @UseGuards(BotDetectionGuard)
- @BotDetection()
@Public()
async log(
@Body() logDTO: PageviewsDTO,
@@ -1041,6 +1035,8 @@ export class AnalyticsController {
const ip = getIPFromHeaders(headers) || reqIP || ''
+ await this.analyticsService.throwIfBot(logDTO.pid, userAgent)
+
await this.analyticsService.validate(logDTO, origin, 'log', ip)
const [unique, psid] = await this.analyticsService.isUnique(
@@ -1059,7 +1055,7 @@ export class AnalyticsController {
const { city, region, country } = getGeoDetails(ip, logDTO.tz, headers)
- const ua = UAParser(userAgent)
+ const ua = UAParser(userAgent, extensions)
const dv = ua.device.type || 'desktop'
const br = ua.browser.name
const os = ua.os.name
@@ -1161,8 +1157,9 @@ export class AnalyticsController {
const { 'user-agent': userAgent, origin } = headers
const { pid } = data
- // todo: create a decorator for bot traffic detection
- if (isbot(userAgent)) {
+ try {
+ await this.analyticsService.throwIfBot(pid, userAgent)
+ } catch {
res.writeHead(200, { 'Content-Type': 'image/gif' })
return res.end(TRANSPARENT_GIF_BUFFER, 'binary')
}
@@ -1182,7 +1179,7 @@ export class AnalyticsController {
await this.analyticsService.processInteractionSD(psid, logDTO.pid)
- const ua = UAParser(userAgent)
+ const ua = UAParser(userAgent, extensions)
const dv = ua.device.type || 'desktop'
const br = ua.browser.name
const os = ua.os.name
@@ -1471,10 +1468,7 @@ export class AnalyticsController {
return this.analyticsService.getErrorsFilters(pid, type)
}
- // Log error event
@Post('error')
- @UseGuards(BotDetectionGuard)
- @BotDetection()
@Public()
async logError(
@Body() errorDTO: ErrorDTO,
@@ -1485,11 +1479,13 @@ export class AnalyticsController {
const ip = getIPFromHeaders(headers) || reqIP || ''
+ await this.analyticsService.throwIfBot(errorDTO.pid, userAgent)
+
await this.analyticsService.validate(errorDTO, origin, 'error', ip)
const { city, region, country } = getGeoDetails(ip, errorDTO.tz, headers)
- const ua = UAParser(userAgent)
+ const ua = UAParser(userAgent, extensions)
const dv = ua.device.type || 'desktop'
const br = ua.browser.name
const os = ua.os.name
diff --git a/backend/apps/selfhosted/src/analytics/analytics.service.ts b/backend/apps/selfhosted/src/analytics/analytics.service.ts
index 103acba30..36b939410 100644
--- a/backend/apps/selfhosted/src/analytics/analytics.service.ts
+++ b/backend/apps/selfhosted/src/analytics/analytics.service.ts
@@ -35,6 +35,7 @@ import {
UnprocessableEntityException,
PreconditionFailedException,
} from '@nestjs/common'
+import { isbot } from 'isbot'
import { DEFAULT_TIMEZONE } from '../user/entities/user.entity'
import {
@@ -60,7 +61,7 @@ import {
import { PageviewsDTO } from './dto/pageviews.dto'
import { EventsDTO } from './dto/events.dto'
import { ProjectService } from '../project/project.service'
-import { Project } from '../project/entity/project.entity'
+import { BotsProtectionLevel, Project } from '../project/entity/project.entity'
import { GetCustomEventMetadata } from './dto/get-custom-event-meta.dto'
import { TimeBucketType, ChartRenderMode } from './dto/getData.dto'
import {
@@ -352,6 +353,18 @@ export class AnalyticsService {
this.projectService.allowedToView(project, uid, password)
}
+ async throwIfBot(pid: string, userAgent: string) {
+ const project = await this.projectService.getRedisProject(pid)
+
+ if (project.botsProtectionLevel === BotsProtectionLevel.OFF) {
+ return
+ }
+
+ if (isbot(userAgent)) {
+ throw new BadRequestException('Bot traffic detected')
+ }
+ }
+
checkOrigin(project: Project, origin: string): void {
// For some reasons the project.origins sometimes may look like [''], let's filter it out
// TODO: Properly validate the origins on project update
diff --git a/backend/apps/selfhosted/src/analytics/utils/ua-parser.ts b/backend/apps/selfhosted/src/analytics/utils/ua-parser.ts
new file mode 100644
index 000000000..240d91473
--- /dev/null
+++ b/backend/apps/selfhosted/src/analytics/utils/ua-parser.ts
@@ -0,0 +1,12 @@
+import UAParser from 'ua-parser-js'
+
+const CUSTOM_BROWSERS = [
+ [/(wget|curl|lynx|GameVault)\/([\w.]+)/i],
+ [UAParser.BROWSER.NAME, UAParser.BROWSER.VERSION],
+]
+
+const extensions = {
+ browser: CUSTOM_BROWSERS,
+}
+
+export { extensions }
diff --git a/backend/apps/selfhosted/src/common/utils.ts b/backend/apps/selfhosted/src/common/utils.ts
index 22b7b4622..87230c565 100644
--- a/backend/apps/selfhosted/src/common/utils.ts
+++ b/backend/apps/selfhosted/src/common/utils.ts
@@ -28,7 +28,7 @@ import {
} from './constants'
import { clickhouse } from './integrations/clickhouse'
import { DEFAULT_TIMEZONE, TimeFormat } from '../user/entities/user.entity'
-import { Project } from '../project/entity/project.entity'
+import { BotsProtectionLevel, Project } from '../project/entity/project.entity'
import { ProjectViewEntity } from '../project/entity/project-view.entity'
import { ClickhouseFunnel, ClickhouseSFUser } from './types'
@@ -50,6 +50,7 @@ const ALLOWED_KEYS = [
'public',
'isPasswordProtected',
'passwordHash',
+ 'botsProtectionLevel',
]
const ALLOWED_FUNNEL_KEYS = ['name', 'steps']
@@ -306,6 +307,7 @@ const createProjectClickhouse = async (project: Partial<Project>) => {
public: 0,
isPasswordProtected: 0,
passwordHash: null,
+ botsProtectionLevel: BotsProtectionLevel.BASIC,
created: dayjs.utc().format('YYYY-MM-DD HH:mm:ss'),
},
],
diff --git a/backend/apps/selfhosted/src/project/dto/project.dto.ts b/backend/apps/selfhosted/src/project/dto/project.dto.ts
index 38f36e056..f3991c75d 100644
--- a/backend/apps/selfhosted/src/project/dto/project.dto.ts
+++ b/backend/apps/selfhosted/src/project/dto/project.dto.ts
@@ -1,5 +1,6 @@
import { ApiProperty } from '@nestjs/swagger'
-import { IsNotEmpty, Length } from 'class-validator'
+import { IsEnum, IsNotEmpty, IsOptional, Length } from 'class-validator'
+import { BotsProtectionLevel } from '../entity'
export class ProjectDTO {
@ApiProperty({
@@ -26,6 +27,15 @@ export class ProjectDTO {
})
origins: string[] | null
+ @ApiProperty({
+ enum: BotsProtectionLevel,
+ required: false,
+ description: 'Bots protection level',
+ })
+ @IsEnum(BotsProtectionLevel)
+ @IsOptional()
+ botsProtectionLevel?: BotsProtectionLevel
+
@ApiProperty({
example: '::1,127.0.0.1,192.168.0.1/32',
required: false,
diff --git a/backend/apps/selfhosted/src/project/entity/project.entity.ts b/backend/apps/selfhosted/src/project/entity/project.entity.ts
index 420ee292c..b160a1acf 100644
--- a/backend/apps/selfhosted/src/project/entity/project.entity.ts
+++ b/backend/apps/selfhosted/src/project/entity/project.entity.ts
@@ -1,3 +1,8 @@
+export enum BotsProtectionLevel {
+ OFF = 'off',
+ BASIC = 'basic',
+}
+
export class Project {
id: string
@@ -13,6 +18,8 @@ export class Project {
isPasswordProtected: boolean
+ botsProtectionLevel: BotsProtectionLevel
+
passwordHash?: string
created: Date
diff --git a/backend/apps/selfhosted/src/project/project.controller.ts b/backend/apps/selfhosted/src/project/project.controller.ts
index f519f3939..4e880c258 100644
--- a/backend/apps/selfhosted/src/project/project.controller.ts
+++ b/backend/apps/selfhosted/src/project/project.controller.ts
@@ -503,6 +503,10 @@ export class ProjectController {
project.ipBlacklist = []
}
+ if (projectDTO.botsProtectionLevel) {
+ project.botsProtectionLevel = projectDTO.botsProtectionLevel
+ }
+
if (projectDTO.name) {
project.name = _trim(projectDTO.name)
}
diff --git a/backend/apps/selfhosted/src/project/project.service.ts b/backend/apps/selfhosted/src/project/project.service.ts
index ea458b271..83e3875e1 100644
--- a/backend/apps/selfhosted/src/project/project.service.ts
+++ b/backend/apps/selfhosted/src/project/project.service.ts
@@ -188,7 +188,7 @@ export class ProjectService {
query,
query_params: params,
})
- .then(resultSet => resultSet.json())
+ .then(resultSet => resultSet.json<{ pid: string }>())
return _map(data, ({ pid }) => pid)
}
@@ -237,7 +237,7 @@ export class ProjectService {
query,
query_params: params,
})
- .then(resultSet => resultSet.json())
+ .then(resultSet => resultSet.json<{ pid: string }>())
return _map(data, ({ pid }) => pid)
}
diff --git a/backend/migrations/clickhouse/initialise_selfhosted.js b/backend/migrations/clickhouse/initialise_selfhosted.js
index 201421164..805f02e71 100644
--- a/backend/migrations/clickhouse/initialise_selfhosted.js
+++ b/backend/migrations/clickhouse/initialise_selfhosted.js
@@ -12,6 +12,7 @@ const CLICKHOUSE_INIT_QUERIES = [
active Int8,
public Int8,
isPasswordProtected Int8,
+ botsProtectionLevel String DEFAULT 'basic',
passwordHash Nullable(String),
created DateTime
)
diff --git a/backend/migrations/clickhouse/selfhosted_2024_10_17.js b/backend/migrations/clickhouse/selfhosted_2024_10_17.js
new file mode 100644
index 000000000..d14f85283
--- /dev/null
+++ b/backend/migrations/clickhouse/selfhosted_2024_10_17.js
@@ -0,0 +1,8 @@
+const { queriesRunner, dbName } = require('./setup')
+
+const queries = [
+ // Added 'botsProtectionLevel' column
+ `ALTER TABLE ${dbName}.project ADD COLUMN botsProtectionLevel String DEFAULT 'basic' AFTER isPasswordProtected;`,
+]
+
+queriesRunner(queries)