📈 Open source, privacy-first web analytics — a cookie-less Google Analytics alternative with built-in error tracking and performance monitoring. Self-host it or use our cloud. Get started for free!
TypeScript chartsclickhousedockergdprgoogle-analyticsmetricsmonitoringmysqlnestjsprivacyredisself-hostedselfhostedstatisticstailwindcssweb-analyticswebanalyticswebsite
captcha clean-up
Blue Mouse7666703a8378
diff --git a/apps/selfhosted/src/analytics/analytics.controller.ts b/apps/selfhosted/src/analytics/analytics.controller.tsindex 6013acb74..1d2d8798f 100644--- a/apps/selfhosted/src/analytics/analytics.controller.ts+++ b/apps/selfhosted/src/analytics/analytics.controller.ts@@ -255,7 +255,6 @@ export class AnalyticsController {async getData(@Query() data: AnalyticsGET_DTO,@CurrentUserId() uid: string,- isCaptcha = false,): Promise<any> {const {pid,@@ -274,10 +273,7 @@ export class AnalyticsController {this.analyticsService.validateTimebucket(timeBucket)const [filtersQuery, filtersParams, parsedFilters, customEVFilterApplied] =- this.analyticsService.getFiltersQuery(- filters,- isCaptcha ? DataType.CAPTCHA : DataType.ANALYTICS,- )+ this.analyticsService.getFiltersQuery(filters, DataType.ANALYTICS)const safeTimezone = this.analyticsService.getSafeTimezone(timezone)const { groupFrom, groupTo, groupFromUTC, groupToUTC } =@@ -291,11 +287,9 @@ export class AnalyticsController {await this.analyticsService.checkProjectAccess(pid, uid)let queryCustoms = `SELECT ev, count() FROM customEV WHERE pid = {pid:FixedString(12)} ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String} GROUP BY ev`- let subQuery = `FROM ${- isCaptcha ? 'captcha' : 'analytics'- } WHERE pid = {pid:FixedString(12)} ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String}`+ let subQuery = `FROM analytics WHERE pid = {pid:FixedString(12)} ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String}`- if (customEVFilterApplied && !isCaptcha) {+ if (customEVFilterApplied) {queryCustoms = `SELECT ev, count() FROM customEV WHERE pid = {pid:FixedString(12)} ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String} GROUP BY ev`subQuery = `FROM customEV WHERE pid = {pid:FixedString(12)} ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String}`@@ -310,38 +304,17 @@ export class AnalyticsController {},}- let result: object | void-- if (isCaptcha) {- result = await this.analyticsService.groupCaptchaByTimeBucket(- timeBucket,- groupFrom,- groupTo,- subQuery,- filtersQuery,- paramsData,- safeTimezone,- )- } else {- result = await this.analyticsService.groupByTimeBucket(- timeBucket,- groupFrom,- groupTo,- subQuery,- filtersQuery,- paramsData,- safeTimezone,- customEVFilterApplied,- parsedFilters,- )- }-- if (isCaptcha) {- return {- ...result,- appliedFilters: parsedFilters,- }- }+ const result = await this.analyticsService.groupByTimeBucket(+ timeBucket,+ groupFrom,+ groupTo,+ subQuery,+ filtersQuery,+ paramsData,+ safeTimezone,+ customEVFilterApplied,+ parsedFilters,+ )const customs = await this.analyticsService.processCustomEV(queryCustoms,@@ -543,15 +516,6 @@ export class AnalyticsController {}}- @Get('captcha')- @Auth([], true, true)- async getCaptchaData(- @Query() data: AnalyticsGET_DTO,- @CurrentUserId() uid: string,- ): Promise<any> {- return this.getData(data, uid, true)- }-@Get('user-flow')@Auth([], true, true)async getUserFlow(@@ -614,26 +578,6 @@ export class AnalyticsController {return this.analyticsService.getSummary(pidsArray, 'w')}- @Get('captcha/birdseye')- @Auth([], true, true)- // returns overall short statistics per CAPTCHA project- async getCaptchaOverallStats(- @Query() data,- @CurrentUserId() uid: string,- ): Promise<any> {- const { pids, pid } = data- const pidsArray = getPIDsArray(pids, pid)-- const validationPromises = _map(pidsArray, async currentPID => {- this.analyticsService.validatePID(currentPID)- await this.analyticsService.checkProjectAccess(currentPID, uid)- })-- await Promise.all(validationPromises)-- return this.analyticsService.getCaptchaSummary(pidsArray, 'w')- }-@Get('hb')async getHeartBeatStats(@Query() data,diff --git a/apps/selfhosted/src/analytics/analytics.service.ts b/apps/selfhosted/src/analytics/analytics.service.tsindex e6edfc6d4..d5cc74225 100644--- a/apps/selfhosted/src/analytics/analytics.service.ts+++ b/apps/selfhosted/src/analytics/analytics.service.ts@@ -84,7 +84,6 @@ const GMT_0_TIMEZONES = [const cols = ['cc', 'pg', 'lc', 'br', 'os', 'dv', 'ref', 'so', 'me', 'ca']-const captchaColumns = ['cc', 'br', 'os', 'dv']const perfColumns = ['cc', 'pg', 'dv', 'br']const validPeriods = [@@ -179,32 +178,36 @@ const generateParamsQuery = (subQuery: string,customEVFilterApplied: boolean,isPageInclusiveFilterSet: boolean,- isCaptcha?: boolean,isPerformance?: boolean,): string => {- if (isPerformance) {- return `SELECT ${col}, avg(pageLoad) ${subQuery} AND ${col} IS NOT NULL GROUP BY ${col}`+ let columns = [`${col} as name`]++ // For regions and cities we'll return an array of objects, that will also include the country code+ // We need the conutry code to display the flag next to the region/city name+ if (col === 'rg' || col === 'ct') {+ columns = [...columns, 'cc']}- if (isCaptcha) {- return `SELECT ${col}, count(*) ${subQuery} AND ${col} IS NOT NULL GROUP BY ${col}`+ const columnsQuery = columns.join(', ')++ if (isPerformance) {+ return `SELECT ${columnsQuery}, round(divide(avg(pageLoad), 1000), 2) as count ${subQuery} AND ${col} IS NOT NULL GROUP BY ${columnsQuery}`}if (customEVFilterApplied) {- return `SELECT ${col}, count(*) ${subQuery} AND ${col} IS NOT NULL GROUP BY ${col}`+ return `SELECT ${columnsQuery}, count(*) as count ${subQuery} AND ${col} IS NOT NULL GROUP BY ${columnsQuery}`}if (col === 'pg' || isPageInclusiveFilterSet) {- return `SELECT ${col}, count(*) ${subQuery} AND ${col} IS NOT NULL GROUP BY ${col}`+ return `SELECT ${columnsQuery}, count(*) as count ${subQuery} AND ${col} IS NOT NULL GROUP BY ${columnsQuery}`}- return `SELECT ${col}, count(*) ${subQuery} AND ${col} IS NOT NULL AND unique='1' GROUP BY ${col}`+ return `SELECT ${columnsQuery}, count(*) as count ${subQuery} AND ${col} IS NOT NULL AND unique='1' GROUP BY ${columnsQuery}`}export enum DataType {ANALYTICS = 'analytics',PERFORMANCE = 'performance',- CAPTCHA = 'captcha',}const isValidOrigin = (origins: string[], origin: string) => {@@ -353,11 +356,7 @@ export class AnalyticsService {return cols}- if (dataType === DataType.PERFORMANCE) {- return perfColumns- }-- return captchaColumns+ return perfColumns}getGroupFromTo(@@ -730,13 +729,9 @@ export class AnalyticsService {return this.getSummaryStats(pids, 'analytics', period, amountToSubtract)}- async getCaptchaSummary(pids: string[], period: 'w' | 'M' = 'w') {- return this.getSummaryStats(pids, 'captcha', period)- }-async getSummaryStats(pids: string[],- tableName: 'analytics' | 'captcha',+ tableName: 'analytics',period: 'w' | 'M' = 'w',amountToSubtract = 1,) {@@ -834,28 +829,22 @@ export class AnalyticsService {subQuery: string,customEVFilterApplied: boolean,paramsData: any,- isCaptcha: boolean,isPerformance: boolean,): Promise<any> {const params = {}// We need this to display all the pageview related data (e.g. country, browser) when user applies an inclusive filter on the Page column- const isPageInclusiveFilterSet =- isCaptcha || isPerformance- ? false- : !_isEmpty(- _find(- parsedFilters,- filter => filter.column === 'pg' && !filter.isExclusive,- ),- )+ const isPageInclusiveFilterSet = isPerformance+ ? false+ : !_isEmpty(+ _find(+ parsedFilters,+ filter => filter.column === 'pg' && !filter.isExclusive,+ ),+ )let columns = cols- if (isCaptcha) {- columns = captchaColumns- }-if (isPerformance) {columns = perfColumns}@@ -866,7 +855,6 @@ export class AnalyticsService {subQuery,customEVFilterApplied,isPageInclusiveFilterSet,- isCaptcha,isPerformance,)const res = await clickhouse.query(query, paramsData).toPromise()@@ -1162,34 +1150,6 @@ export class AnalyticsService {`}- generateCaptchaAggregationQuery(- timeBucket: TimeBucketType,- filtersQuery: string,- safeTimezone: string,- ): string {- const timeBucketFunc = timeBucketConversion[timeBucket]- const [selector, groupBy] = this.getGroupSubquery(timeBucket)- const tzFromDate = `toTimeZone(parseDateTimeBestEffort({groupFrom:String}), '${safeTimezone}')`- const tzToDate = `toTimeZone(parseDateTimeBestEffort({groupTo:String}), '${safeTimezone}')`-- return `- SELECT- ${selector},- count() as count- FROM (- SELECT *,- ${timeBucketFunc}(toTimeZone(created, '${safeTimezone}')) as tz_created- FROM captcha- WHERE- pid = {pid:FixedString(12)}- AND created BETWEEN ${tzFromDate} AND ${tzToDate}- ${filtersQuery}- ) as subquery- GROUP BY ${groupBy}- ORDER BY ${groupBy}- `- }-async groupByTimeBucket(timeBucket: TimeBucketType,from: string,@@ -1264,7 +1224,6 @@ export class AnalyticsService {customEVFilterApplied,paramsData,false,- false,)}@@ -1334,90 +1293,6 @@ export class AnalyticsService {})}- extractCaptchaChartData(result, x: string[]): any {- const count = Array(x.length).fill(0)-- for (let row = 0; row < _size(result); ++row) {- const dateString = this.generateDateString(result[row])-- const index = x.indexOf(dateString)-- if (index !== -1) {- count[index] = result[row].count- }- }-- return {- count,- }- }-- async groupCaptchaByTimeBucket(- timeBucket: TimeBucketType,- from: string,- to: string,- subQuery: string,- filtersQuery: string,- paramsData: object,- safeTimezone: string,- ): Promise<object | void> {- let params: unknown = {}- let chart: unknown = {}-- const promises = [- // Getting params- (async () => {- params = await this.generateParams(- null,- subQuery,- false,- paramsData,- true,- false,- )-- if (!_some(_values(params), val => !_isEmpty(val))) {- throw new BadRequestException(- 'The are no parameters for the specified time frames',- )- }- })(),-- // Getting CAPTCHA chart data- (async () => {- const { xShifted } = this.generateXAxis(- timeBucket,- from,- to,- safeTimezone,- )-- const query = this.generateCaptchaAggregationQuery(- timeBucket,- filtersQuery,- safeTimezone,- )-- const result = <Array<TrafficCEFilterCHResponse>>(- await clickhouse.query(query, paramsData).toPromise()- )- const { count } = this.extractCaptchaChartData(result, xShifted)-- chart = {- x: xShifted,- results: count,- }- })(),- ]-- await Promise.all(promises)-- return Promise.resolve({- params,- chart,- })- }-extractPerformanceChartData(result, x: string[]): any {const dns = Array(x.length).fill(0)const tls = Array(x.length).fill(0)@@ -1500,7 +1375,6 @@ export class AnalyticsService {subQuery,false,paramsData,- false,true,)