Skip to content

📈 Open source, privacy-first web analytics — a cookie-less Google Analytics alternative with built-in error tracking and performance monitoring. Self-host it or use our cloud. Get started for free!

captcha clean-up

Blue Mouse7666703a8378
diff --git a/apps/selfhosted/src/analytics/analytics.controller.ts b/apps/selfhosted/src/analytics/analytics.controller.ts
index 6013acb74..1d2d8798f 100644
--- a/apps/selfhosted/src/analytics/analytics.controller.ts
+++ b/apps/selfhosted/src/analytics/analytics.controller.ts
@@ -255,7 +255,6 @@ export class AnalyticsController {
async getData(
@Query() data: AnalyticsGET_DTO,
@CurrentUserId() uid: string,
- isCaptcha = false,
): Promise<any> {
const {
pid,
@@ -274,10 +273,7 @@ export class AnalyticsController {
this.analyticsService.validateTimebucket(timeBucket)
const [filtersQuery, filtersParams, parsedFilters, customEVFilterApplied] =
- this.analyticsService.getFiltersQuery(
- filters,
- isCaptcha ? DataType.CAPTCHA : DataType.ANALYTICS,
- )
+ this.analyticsService.getFiltersQuery(filters, DataType.ANALYTICS)
const safeTimezone = this.analyticsService.getSafeTimezone(timezone)
const { groupFrom, groupTo, groupFromUTC, groupToUTC } =
@@ -291,11 +287,9 @@ export class AnalyticsController {
await this.analyticsService.checkProjectAccess(pid, uid)
let queryCustoms = `SELECT ev, count() FROM customEV WHERE pid = {pid:FixedString(12)} ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String} GROUP BY ev`
- let subQuery = `FROM ${
- isCaptcha ? 'captcha' : 'analytics'
- } WHERE pid = {pid:FixedString(12)} ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String}`
+ let subQuery = `FROM analytics WHERE pid = {pid:FixedString(12)} ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String}`
- if (customEVFilterApplied && !isCaptcha) {
+ if (customEVFilterApplied) {
queryCustoms = `SELECT ev, count() FROM customEV WHERE pid = {pid:FixedString(12)} ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String} GROUP BY ev`
subQuery = `FROM customEV WHERE pid = {pid:FixedString(12)} ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String}`
@@ -310,38 +304,17 @@ export class AnalyticsController {
},
}
- let result: object | void
-
- if (isCaptcha) {
- result = await this.analyticsService.groupCaptchaByTimeBucket(
- timeBucket,
- groupFrom,
- groupTo,
- subQuery,
- filtersQuery,
- paramsData,
- safeTimezone,
- )
- } else {
- result = await this.analyticsService.groupByTimeBucket(
- timeBucket,
- groupFrom,
- groupTo,
- subQuery,
- filtersQuery,
- paramsData,
- safeTimezone,
- customEVFilterApplied,
- parsedFilters,
- )
- }
-
- if (isCaptcha) {
- return {
- ...result,
- appliedFilters: parsedFilters,
- }
- }
+ const result = await this.analyticsService.groupByTimeBucket(
+ timeBucket,
+ groupFrom,
+ groupTo,
+ subQuery,
+ filtersQuery,
+ paramsData,
+ safeTimezone,
+ customEVFilterApplied,
+ parsedFilters,
+ )
const customs = await this.analyticsService.processCustomEV(
queryCustoms,
@@ -543,15 +516,6 @@ export class AnalyticsController {
}
}
- @Get('captcha')
- @Auth([], true, true)
- async getCaptchaData(
- @Query() data: AnalyticsGET_DTO,
- @CurrentUserId() uid: string,
- ): Promise<any> {
- return this.getData(data, uid, true)
- }
-
@Get('user-flow')
@Auth([], true, true)
async getUserFlow(
@@ -614,26 +578,6 @@ export class AnalyticsController {
return this.analyticsService.getSummary(pidsArray, 'w')
}
- @Get('captcha/birdseye')
- @Auth([], true, true)
- // returns overall short statistics per CAPTCHA project
- async getCaptchaOverallStats(
- @Query() data,
- @CurrentUserId() uid: string,
- ): Promise<any> {
- const { pids, pid } = data
- const pidsArray = getPIDsArray(pids, pid)
-
- const validationPromises = _map(pidsArray, async currentPID => {
- this.analyticsService.validatePID(currentPID)
- await this.analyticsService.checkProjectAccess(currentPID, uid)
- })
-
- await Promise.all(validationPromises)
-
- return this.analyticsService.getCaptchaSummary(pidsArray, 'w')
- }
-
@Get('hb')
async getHeartBeatStats(
@Query() data,
diff --git a/apps/selfhosted/src/analytics/analytics.service.ts b/apps/selfhosted/src/analytics/analytics.service.ts
index e6edfc6d4..d5cc74225 100644
--- a/apps/selfhosted/src/analytics/analytics.service.ts
+++ b/apps/selfhosted/src/analytics/analytics.service.ts
@@ -84,7 +84,6 @@ const GMT_0_TIMEZONES = [
const cols = ['cc', 'pg', 'lc', 'br', 'os', 'dv', 'ref', 'so', 'me', 'ca']
-const captchaColumns = ['cc', 'br', 'os', 'dv']
const perfColumns = ['cc', 'pg', 'dv', 'br']
const validPeriods = [
@@ -179,32 +178,36 @@ const generateParamsQuery = (
subQuery: string,
customEVFilterApplied: boolean,
isPageInclusiveFilterSet: boolean,
- isCaptcha?: boolean,
isPerformance?: boolean,
): string => {
- if (isPerformance) {
- return `SELECT ${col}, avg(pageLoad) ${subQuery} AND ${col} IS NOT NULL GROUP BY ${col}`
+ let columns = [`${col} as name`]
+
+ // For regions and cities we'll return an array of objects, that will also include the country code
+ // We need the conutry code to display the flag next to the region/city name
+ if (col === 'rg' || col === 'ct') {
+ columns = [...columns, 'cc']
}
- if (isCaptcha) {
- return `SELECT ${col}, count(*) ${subQuery} AND ${col} IS NOT NULL GROUP BY ${col}`
+ const columnsQuery = columns.join(', ')
+
+ if (isPerformance) {
+ return `SELECT ${columnsQuery}, round(divide(avg(pageLoad), 1000), 2) as count ${subQuery} AND ${col} IS NOT NULL GROUP BY ${columnsQuery}`
}
if (customEVFilterApplied) {
- return `SELECT ${col}, count(*) ${subQuery} AND ${col} IS NOT NULL GROUP BY ${col}`
+ return `SELECT ${columnsQuery}, count(*) as count ${subQuery} AND ${col} IS NOT NULL GROUP BY ${columnsQuery}`
}
if (col === 'pg' || isPageInclusiveFilterSet) {
- return `SELECT ${col}, count(*) ${subQuery} AND ${col} IS NOT NULL GROUP BY ${col}`
+ return `SELECT ${columnsQuery}, count(*) as count ${subQuery} AND ${col} IS NOT NULL GROUP BY ${columnsQuery}`
}
- return `SELECT ${col}, count(*) ${subQuery} AND ${col} IS NOT NULL AND unique='1' GROUP BY ${col}`
+ return `SELECT ${columnsQuery}, count(*) as count ${subQuery} AND ${col} IS NOT NULL AND unique='1' GROUP BY ${columnsQuery}`
}
export enum DataType {
ANALYTICS = 'analytics',
PERFORMANCE = 'performance',
- CAPTCHA = 'captcha',
}
const isValidOrigin = (origins: string[], origin: string) => {
@@ -353,11 +356,7 @@ export class AnalyticsService {
return cols
}
- if (dataType === DataType.PERFORMANCE) {
- return perfColumns
- }
-
- return captchaColumns
+ return perfColumns
}
getGroupFromTo(
@@ -730,13 +729,9 @@ export class AnalyticsService {
return this.getSummaryStats(pids, 'analytics', period, amountToSubtract)
}
- async getCaptchaSummary(pids: string[], period: 'w' | 'M' = 'w') {
- return this.getSummaryStats(pids, 'captcha', period)
- }
-
async getSummaryStats(
pids: string[],
- tableName: 'analytics' | 'captcha',
+ tableName: 'analytics',
period: 'w' | 'M' = 'w',
amountToSubtract = 1,
) {
@@ -834,28 +829,22 @@ export class AnalyticsService {
subQuery: string,
customEVFilterApplied: boolean,
paramsData: any,
- isCaptcha: boolean,
isPerformance: boolean,
): Promise<any> {
const params = {}
// We need this to display all the pageview related data (e.g. country, browser) when user applies an inclusive filter on the Page column
- const isPageInclusiveFilterSet =
- isCaptcha || isPerformance
- ? false
- : !_isEmpty(
- _find(
- parsedFilters,
- filter => filter.column === 'pg' && !filter.isExclusive,
- ),
- )
+ const isPageInclusiveFilterSet = isPerformance
+ ? false
+ : !_isEmpty(
+ _find(
+ parsedFilters,
+ filter => filter.column === 'pg' && !filter.isExclusive,
+ ),
+ )
let columns = cols
- if (isCaptcha) {
- columns = captchaColumns
- }
-
if (isPerformance) {
columns = perfColumns
}
@@ -866,7 +855,6 @@ export class AnalyticsService {
subQuery,
customEVFilterApplied,
isPageInclusiveFilterSet,
- isCaptcha,
isPerformance,
)
const res = await clickhouse.query(query, paramsData).toPromise()
@@ -1162,34 +1150,6 @@ export class AnalyticsService {
`
}
- generateCaptchaAggregationQuery(
- timeBucket: TimeBucketType,
- filtersQuery: string,
- safeTimezone: string,
- ): string {
- const timeBucketFunc = timeBucketConversion[timeBucket]
- const [selector, groupBy] = this.getGroupSubquery(timeBucket)
- const tzFromDate = `toTimeZone(parseDateTimeBestEffort({groupFrom:String}), '${safeTimezone}')`
- const tzToDate = `toTimeZone(parseDateTimeBestEffort({groupTo:String}), '${safeTimezone}')`
-
- return `
- SELECT
- ${selector},
- count() as count
- FROM (
- SELECT *,
- ${timeBucketFunc}(toTimeZone(created, '${safeTimezone}')) as tz_created
- FROM captcha
- WHERE
- pid = {pid:FixedString(12)}
- AND created BETWEEN ${tzFromDate} AND ${tzToDate}
- ${filtersQuery}
- ) as subquery
- GROUP BY ${groupBy}
- ORDER BY ${groupBy}
- `
- }
-
async groupByTimeBucket(
timeBucket: TimeBucketType,
from: string,
@@ -1264,7 +1224,6 @@ export class AnalyticsService {
customEVFilterApplied,
paramsData,
false,
- false,
)
}
@@ -1334,90 +1293,6 @@ export class AnalyticsService {
})
}
- extractCaptchaChartData(result, x: string[]): any {
- const count = Array(x.length).fill(0)
-
- for (let row = 0; row < _size(result); ++row) {
- const dateString = this.generateDateString(result[row])
-
- const index = x.indexOf(dateString)
-
- if (index !== -1) {
- count[index] = result[row].count
- }
- }
-
- return {
- count,
- }
- }
-
- async groupCaptchaByTimeBucket(
- timeBucket: TimeBucketType,
- from: string,
- to: string,
- subQuery: string,
- filtersQuery: string,
- paramsData: object,
- safeTimezone: string,
- ): Promise<object | void> {
- let params: unknown = {}
- let chart: unknown = {}
-
- const promises = [
- // Getting params
- (async () => {
- params = await this.generateParams(
- null,
- subQuery,
- false,
- paramsData,
- true,
- false,
- )
-
- if (!_some(_values(params), val => !_isEmpty(val))) {
- throw new BadRequestException(
- 'The are no parameters for the specified time frames',
- )
- }
- })(),
-
- // Getting CAPTCHA chart data
- (async () => {
- const { xShifted } = this.generateXAxis(
- timeBucket,
- from,
- to,
- safeTimezone,
- )
-
- const query = this.generateCaptchaAggregationQuery(
- timeBucket,
- filtersQuery,
- safeTimezone,
- )
-
- const result = <Array<TrafficCEFilterCHResponse>>(
- await clickhouse.query(query, paramsData).toPromise()
- )
- const { count } = this.extractCaptchaChartData(result, xShifted)
-
- chart = {
- x: xShifted,
- results: count,
- }
- })(),
- ]
-
- await Promise.all(promises)
-
- return Promise.resolve({
- params,
- chart,
- })
- }
-
extractPerformanceChartData(result, x: string[]): any {
const dns = Array(x.length).fill(0)
const tls = Array(x.length).fill(0)
@@ -1500,7 +1375,6 @@ export class AnalyticsService {
subQuery,
false,
paramsData,
- false,
true,
)