📈 Open source, privacy-first web analytics — a cookie-less Google Analytics alternative with built-in error tracking and performance monitoring. Self-host it or use our cloud. Get started for free!
TypeScript chartsclickhousedockergdprgoogle-analyticsmetricsmonitoringmysqlnestjsprivacyredisself-hostedselfhostedstatisticstailwindcssweb-analyticswebanalyticswebsite
add missing captcha stuff
Blue Mousec95bf31893a0
diff --git a/backend/apps/community/src/analytics/analytics.controller.ts b/backend/apps/community/src/analytics/analytics.controller.tsindex d7a0d8425..84ae9d684 100644--- a/backend/apps/community/src/analytics/analytics.controller.ts+++ b/backend/apps/community/src/analytics/analytics.controller.ts@@ -200,6 +200,7 @@ export class AnalyticsController {@Query() data: GetDataDto,@CurrentUserId() uid: string,@Headers() headers: { 'x-password'?: string },+ isCaptcha = false,) {const {pid,@@ -219,12 +220,16 @@ export class AnalyticsController {headers['x-password'],)- const parsedMetrics = this.analyticsService.parseMetrics(metrics)+ let parsedMetrics- if (_size(parsedMetrics) > MAX_METRICS_IN_VIEW) {- throw new BadRequestException(- `The maximum number of metrics within one request is ${MAX_METRICS_IN_VIEW}`,- )+ if (!isCaptcha) {+ parsedMetrics = this.analyticsService.parseMetrics(metrics)++ if (_size(parsedMetrics) > MAX_METRICS_IN_VIEW) {+ throw new BadRequestException(+ `The maximum number of metrics within one request is ${MAX_METRICS_IN_VIEW}`,+ )+ }}let newTimebucket = timeBucket@@ -235,7 +240,7 @@ export class AnalyticsController {if (period === 'all') {const res = await this.analyticsService.calculateTimeBucketForAllTime(pid,- 'analytics',+ isCaptcha ? 'captcha' : 'analytics',)diff = res.diff@@ -247,7 +252,10 @@ export class AnalyticsController {}const [filtersQuery, filtersParams, parsedFilters, customEVFilterApplied] =- this.analyticsService.getFiltersQuery(filters, DataType.ANALYTICS)+ this.analyticsService.getFiltersQuery(+ filters,+ isCaptcha ? DataType.CAPTCHA : DataType.ANALYTICS,+ )const safeTimezone = this.analyticsService.getSafeTimezone(timezone)const { groupFrom, groupTo, groupFromUTC, groupToUTC } =@@ -260,9 +268,11 @@ export class AnalyticsController {diff,)- let subQuery = `FROM analytics WHERE pid = {pid:FixedString(12)} ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String}`+ let subQuery = `FROM ${+ isCaptcha ? 'captcha' : 'analytics'+ } WHERE pid = {pid:FixedString(12)} ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String}`- if (customEVFilterApplied) {+ if (customEVFilterApplied && !isCaptcha) {subQuery = `FROM customEV WHERE pid = {pid:FixedString(12)} ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String}`}@@ -275,18 +285,41 @@ export class AnalyticsController {},}- const result = await this.analyticsService.groupByTimeBucket(- newTimebucket,- groupFrom,- groupTo,- subQuery,- filtersQuery,- paramsData,- safeTimezone,- customEVFilterApplied,- parsedFilters,- mode,- )+ let result: any | void++ if (isCaptcha) {+ result = await this.analyticsService.groupCaptchaByTimeBucket(+ newTimebucket,+ groupFrom,+ groupTo,+ subQuery,+ filtersQuery,+ paramsData,+ safeTimezone,+ mode,+ )+ } else {+ result = await this.analyticsService.groupByTimeBucket(+ newTimebucket,+ groupFrom,+ groupTo,+ subQuery,+ filtersQuery,+ paramsData,+ safeTimezone,+ customEVFilterApplied,+ parsedFilters,+ mode,+ )+ }++ if (isCaptcha) {+ return {+ ...result,+ appliedFilters: parsedFilters,+ timeBucket: allowedTumebucketForPeriodAll,+ }+ }const customs = await this.analyticsService.getCustomEvents(filtersQuery,@@ -327,6 +360,15 @@ export class AnalyticsController {}}+ @Get('captcha')+ @Auth(true, true)+ async getCaptchaData(+ @Query() data: GetDataDto,+ @CurrentUserId() uid: string,+ ) {+ return this.getData(data, uid, {}, true)+ }+@Get('funnel')@Auth(true, true)async getFunnel(diff --git a/backend/apps/community/src/analytics/analytics.service.ts b/backend/apps/community/src/analytics/analytics.service.tsindex ae26c78a1..c24f24506 100644--- a/backend/apps/community/src/analytics/analytics.service.ts+++ b/backend/apps/community/src/analytics/analytics.service.ts@@ -44,6 +44,7 @@ import {TRAFFIC_COLUMNS,PERFORMANCE_COLUMNS,ERROR_COLUMNS,+ CAPTCHA_COLUMNS,MIN_PAGES_IN_FUNNEL,MAX_PAGES_IN_FUNNEL,ALL_COLUMNS,@@ -243,7 +244,7 @@ const generateParamsQuery = (subQuery: string,customEVFilterApplied: boolean,isPageInclusiveFilterSet: boolean,- type: 'traffic' | 'performance' | 'errors',+ type: 'traffic' | 'performance' | 'errors' | 'captcha',measure?: PerfMeasure,): string => {let columns = [`${col} as name`]@@ -281,6 +282,10 @@ const generateParamsQuery = (return `SELECT ${columnsQuery}, count(*) as count ${subQuery} ${EXCLUDE_NULL_FOR.includes(col) ? `AND ${col} IS NOT NULL` : ''} GROUP BY ${columnsQuery}`}+ if (type === 'captcha') {+ return `SELECT ${columnsQuery}, count(*) as count ${subQuery} GROUP BY ${col}`+ }+if (customEVFilterApplied) {return `SELECT ${columnsQuery}, count(*) as count ${subQuery} GROUP BY ${columnsQuery}`}@@ -300,6 +305,7 @@ export enum DataType {ANALYTICS = 'analytics',PERFORMANCE = 'performance',ERRORS = 'errors',+ CAPTCHA = 'captcha',}const isValidOrigin = (origins: string[], origin: string) => {@@ -555,7 +561,11 @@ export class AnalyticsService {return ERROR_COLUMNS}- return PERFORMANCE_COLUMNS+ if (dataType === DataType.PERFORMANCE) {+ return PERFORMANCE_COLUMNS+ }++ return CAPTCHA_COLUMNS}getGroupFromTo(@@ -821,7 +831,7 @@ export class AnalyticsService {async calculateTimeBucketForAllTime(pid: string,- table: 'analytics' | 'customEV' | 'performance' | 'errors',+ table: 'analytics' | 'customEV' | 'performance' | 'errors' | 'captcha',): Promise<{timeBucket: TimeBucketType[]diff: number@@ -2193,7 +2203,7 @@ export class AnalyticsService {subQuery: string,customEVFilterApplied: boolean,paramsData: any,- type: 'traffic' | 'performance' | 'errors',+ type: 'traffic' | 'performance' | 'errors' | 'captcha',measure?: PerfMeasure,) {// We need this to display all the pageview related data (e.g. country, browser) when user applies an inclusive filter on the Page column@@ -2218,6 +2228,10 @@ export class AnalyticsService {columns = PERFORMANCE_COLUMNS}+ if (type === 'captcha') {+ columns = CAPTCHA_COLUMNS+ }+// Build a single query combining all metricsconst withClauses = columns.map(col => {const baseQuery = generateParamsQuery(@@ -3330,6 +3344,132 @@ export class AnalyticsService {})}+ generateCaptchaAggregationQuery(+ timeBucket: TimeBucketType,+ filtersQuery: string,+ mode: ChartRenderMode,+ ): string {+ const timeBucketFunc = timeBucketConversion[timeBucket]+ const [selector, groupBy] = this.getGroupSubquery(timeBucket)+ const tzFromDate = `toTimeZone(parseDateTimeBestEffort({groupFrom:String}), {timezone:String})`+ const tzToDate = `toTimeZone(parseDateTimeBestEffort({groupTo:String}), {timezone:String})`++ const baseQuery = `+ SELECT+ ${selector},+ count() as count+ FROM (+ SELECT *,+ ${timeBucketFunc}(toTimeZone(created, {timezone:String})) as tz_created+ FROM captcha+ WHERE+ pid = {pid:FixedString(12)}+ AND created BETWEEN ${tzFromDate} AND ${tzToDate}+ ${filtersQuery}+ ) as subquery+ GROUP BY ${groupBy}+ ORDER BY ${groupBy}+ `++ if (mode === ChartRenderMode.CUMULATIVE) {+ return `+ SELECT+ *,+ sum(count) OVER (ORDER BY ${groupBy}) as count+ FROM (${baseQuery})+ `+ }++ return baseQuery+ }++ extractCaptchaChartData(result, x: string[]): any {+ const count = Array(x.length).fill(0)++ for (let row = 0; row < _size(result); ++row) {+ const dateString = this.generateDateString(result[row])++ const index = x.indexOf(dateString)++ if (index !== -1) {+ count[index] = result[row].count+ }+ }++ return {+ count,+ }+ }++ async groupCaptchaByTimeBucket(+ timeBucket: TimeBucketType,+ from: string,+ to: string,+ subQuery: string,+ filtersQuery: string,+ paramsData: any,+ safeTimezone: string,+ mode: ChartRenderMode,+ ): Promise<object | void> {+ let params: unknown = {}+ let chart: unknown = {}++ const promises = [+ // Getting params+ (async () => {+ params = await this.generateParams(+ null,+ subQuery,+ false,+ paramsData,+ 'captcha',+ )++ if (!_some(_values(params), val => !_isEmpty(val))) {+ throw new BadRequestException(+ 'There are no parameters for the specified time frames',+ )+ }+ })(),++ // Getting CAPTCHA chart data+ (async () => {+ const { xShifted } = this.generateXAxis(+ timeBucket,+ from,+ to,+ safeTimezone,+ )++ const query = this.generateCaptchaAggregationQuery(+ timeBucket,+ filtersQuery,+ mode,+ )++ const { data } = await clickhouse+ .query({+ query,+ query_params: { ...paramsData.params, timezone: safeTimezone },+ })+ .then(resultSet => resultSet.json<TrafficCEFilterCHResponse>())+ const { count } = this.extractCaptchaChartData(data, xShifted)++ chart = {+ x: xShifted,+ results: count,+ }+ })(),+ ]++ await Promise.all(promises)++ return Promise.resolve({+ params,+ chart,+ })+ }+extractPerformanceChartData(result, x: string[]): any {const dns = Array(x.length).fill(0)const tls = Array(x.length).fill(0)diff --git a/backend/apps/community/src/common/utils.ts b/backend/apps/community/src/common/utils.tsindex 2394b2804..54ef707a0 100644--- a/backend/apps/community/src/common/utils.ts+++ b/backend/apps/community/src/common/utils.ts@@ -128,6 +128,7 @@ const ALLOWED_KEYS = ['botsProtectionLevel','websiteUrl','captchaSecretKey',+ 'captchaDifficulty',]const CLICKHOUSE_PROJECT_UPDATABLE_KEYS = [...ALLOWED_KEYS, 'adminId']@@ -392,11 +393,17 @@ const updateProjectClickhouse = async (}const INT8_COLUMNS = ['active', 'public', 'isPasswordProtected']+ const UINT8_COLUMNS = ['captchaDifficulty']const params: Record<string, any> = { id: project.id }const assignments = _map(columns, col => {params[col] = filtered[col]- const type = INT8_COLUMNS.includes(col) ? 'Int8' : 'String'+ let type = 'String'+ if (INT8_COLUMNS.includes(col)) {+ type = 'Int8'+ } else if (UINT8_COLUMNS.includes(col)) {+ type = 'UInt8'+ }return `${col}={${col}:${type}}`}).join(', ')diff --git a/backend/apps/community/src/project/dto/update-project.dto.ts b/backend/apps/community/src/project/dto/update-project.dto.tsindex a6dbcde0c..8cfbbbde7 100644--- a/backend/apps/community/src/project/dto/update-project.dto.ts+++ b/backend/apps/community/src/project/dto/update-project.dto.ts@@ -1,5 +1,6 @@import { IntersectionType, PartialType } from '@nestjs/mapped-types'import { ApiProperty } from '@nestjs/swagger'+import { IsInt, IsOptional, Max, Min } from 'class-validator'import { ProjectDTO } from './project.dto'import { ProjectPasswordDto } from './project-password.dto'@@ -20,4 +21,14 @@ export class UpdateProjectDto extends IntersectionType("When true, anyone on the internet (including Google) would be able to see the project's Dashboard.",})public?: boolean++ @ApiProperty({+ required: false,+ description: 'CAPTCHA difficulty level (2-6). Higher = harder for bots.',+ })+ @IsInt()+ @Min(2)+ @Max(6)+ @IsOptional()+ captchaDifficulty?: number}diff --git a/backend/apps/community/src/project/project.controller.ts b/backend/apps/community/src/project/project.controller.tsindex 3ce2642e3..082c447ae 100644--- a/backend/apps/community/src/project/project.controller.ts+++ b/backend/apps/community/src/project/project.controller.ts@@ -1111,6 +1111,10 @@ export class ProjectController {: null}+ if (projectDTO.captchaDifficulty !== undefined) {+ project.captchaDifficulty = projectDTO.captchaDifficulty+ }+await updateProjectClickhouse(this.projectService.formatToClickhouse(project),)diff --git a/web/app/pages/Project/Settings/ProjectSettings.tsx b/web/app/pages/Project/Settings/ProjectSettings.tsxindex 6c1bc73b4..f892eb2f0 100644--- a/web/app/pages/Project/Settings/ProjectSettings.tsx+++ b/web/app/pages/Project/Settings/ProjectSettings.tsx@@ -345,7 +345,7 @@ const ProjectSettings = () => {id: 'captcha',label: t('project.settings.tabs.captcha'),icon: ShieldCheckIcon,- visible: !isSelfhosted,+ visible: true,},{id: 'integrations',