Skip to content

📈 Open source, privacy-first web analytics — a cookie-less Google Analytics alternative with built-in error tracking and performance monitoring. Self-host it or use our cloud. Get started for free!

(feature) Error tracking (#224)

Andrii Romasiund7b6c73e0f75
diff --git a/apps/production/src/analytics/analytics.controller.ts b/apps/production/src/analytics/analytics.controller.ts
index eb7f9fc4f..f26e8e506 100644
--- a/apps/production/src/analytics/analytics.controller.ts
+++ b/apps/production/src/analytics/analytics.controller.ts
@@ -19,6 +19,7 @@ import {
UseGuards,
Get,
Post,
+ Patch,
Headers,
BadRequestException,
InternalServerErrorException,
@@ -63,8 +64,13 @@ import {
REDIS_EVENTS_COUNT_KEY,
REDIS_SESSION_SALT_KEY,
clickhouse,
+ REDIS_LOG_ERROR_CACHE_KEY,
} from '../common/constants'
-import { getGeoDetails, getIPFromHeaders } from '../common/utils'
+import {
+ checkRateLimit,
+ getGeoDetails,
+ getIPFromHeaders,
+} from '../common/utils'
import { BotDetection } from '../common/decorators/bot-detection.decorator'
import { BotDetectionGuard } from '../common/guards/bot-detection.guard'
import { GetCustomEventsDto } from './dto/get-custom-events.dto'
@@ -78,6 +84,10 @@ import {
} from './interfaces'
import { GetSessionsDto } from './dto/get-sessions.dto'
import { GetSessionDto } from './dto/get-session.dto'
+import { ErrorDTO } from './dto/error.dto'
+import { GetErrorsDto } from './dto/get-errors.dto'
+import { GetErrorDTO } from './dto/get-error.dto'
+import { PatchStatusDTO } from './dto/patch-status.dto'
// eslint-disable-next-line @typescript-eslint/no-var-requires
const mysql = require('mysql2')
@@ -216,6 +226,43 @@ const customLogDTO = (
]
}
+const errorLogDTO = (
+ eid: string,
+ pid: string,
+ pg: string,
+ dv: string,
+ br: string,
+ os: string,
+ lc: string,
+ cc: string,
+ rg: string,
+ ct: string,
+ name: string,
+ message: string,
+ lineno: number,
+ colno: number,
+ filename: string,
+): Array<string | number | string[]> => {
+ return [
+ eid,
+ pid,
+ pg,
+ dv,
+ br,
+ os,
+ lc,
+ cc,
+ rg,
+ ct,
+ name,
+ message,
+ lineno,
+ colno,
+ filename,
+ dayjs.utc().format('YYYY-MM-DD HH:mm:ss'),
+ ]
+}
+
export const getElValue = el => {
if (_isArray(el)) {
return `[${_map(el, getElValue).join(',')}]`
@@ -280,6 +327,27 @@ const getPIDsArray = (pids, pid) => {
return pids
}
+const getEIDsArray = (eids, eid) => {
+ const eidsEmpty = _isEmpty(eids)
+ const eidEmpty = _isEmpty(eid)
+ if (eidsEmpty && eidEmpty) {
+ throw new BadRequestException(
+ "An array of Error ID's (eids) or a Error ID (eid) has to be provided",
+ )
+ }
+ if (!eidsEmpty && !eidEmpty) {
+ throw new BadRequestException(
+ "Please provide either an array of Error ID's (eids) or a Error ID (eid), but not both",
+ )
+ }
+
+ if (!eidsEmpty) {
+ return eids
+ }
+
+ return [eid]
+}
+
// needed for serving 1x1 px GIF
const TRANSPARENT_GIF_BUFFER = Buffer.from(
'R0lGODlhAQABAIAAAP///wAAACwAAAAAAQABAAACAkQBADs=',
@@ -555,6 +623,27 @@ export class AnalyticsController {
return this.analyticsService.getFilters(pid, type)
}
+ @Get('errors-filters')
+ @Auth([], true, true)
+ async getErrorsFilters(
+ @Query() data: GetFiltersDto,
+ @CurrentUserId() uid: string,
+ @Headers() headers: { 'x-password'?: string },
+ ): Promise<any> {
+ const { pid, type } = data
+ this.analyticsService.validatePID(pid)
+
+ await this.analyticsService.checkProjectAccess(
+ pid,
+ uid,
+ headers['x-password'],
+ )
+
+ await this.analyticsService.checkBillingAccess(pid)
+
+ return this.analyticsService.getErrorsFilters(pid, type)
+ }
+
@Get('chart')
@Auth([], true, true)
async getChartData(
@@ -1088,6 +1177,89 @@ export class AnalyticsController {
return processed
}
+ // Log error event
+ @Post('error')
+ @UseGuards(BotDetectionGuard)
+ @BotDetection()
+ @Public()
+ async logError(
+ @Body() errorDTO: ErrorDTO,
+ @Headers() headers,
+ @Ip() reqIP,
+ ): Promise<any> {
+ const { 'user-agent': userAgent, origin } = headers
+
+ const ip = getIPFromHeaders(headers, true) || reqIP || ''
+
+ await this.analyticsService.validate(errorDTO, origin, 'error', ip)
+
+ const {
+ city = 'NULL',
+ region = 'NULL',
+ country = 'NULL',
+ } = getGeoDetails(ip, errorDTO.tz)
+
+ const ua = UAParser(userAgent)
+ const dv = ua.device.type || 'desktop'
+ const br = ua.browser.name
+ const os = ua.os.name
+
+ const { name, message, lineno, colno, filename } = errorDTO
+
+ const dto = errorLogDTO(
+ this.analyticsService.getErrorID(errorDTO),
+ errorDTO.pid,
+ errorDTO.pg,
+ dv,
+ br,
+ os,
+ errorDTO.lc,
+ country,
+ region,
+ city,
+ name,
+ message,
+ lineno,
+ colno,
+ filename,
+ )
+
+ try {
+ const values = `(${dto.map(getElValue).join(',')})`
+ await redis.rpush(REDIS_LOG_ERROR_CACHE_KEY, values)
+ } catch (e) {
+ this.logger.error(e)
+ throw new InternalServerErrorException(
+ 'Error occured while saving the custom event',
+ )
+ }
+ }
+
+ // Update error(s) status
+ @Patch('error-status')
+ @Auth([], true, true)
+ async patchStatus(
+ @Body() statusDTO: PatchStatusDTO,
+ @CurrentUserId() uid: string,
+ @Headers() headers,
+ @Ip() reqIP,
+ ): Promise<any> {
+ const { pid, eid, eids: unprocessedEids, status } = statusDTO
+ const ip = getIPFromHeaders(headers) || reqIP || ''
+
+ await checkRateLimit(ip, 'error-status', 100, 1800)
+
+ this.analyticsService.validatePID(pid)
+ await this.analyticsService.checkManageAccess(pid, uid)
+ await this.analyticsService.checkBillingAccess(pid)
+
+ const eids = getEIDsArray(unprocessedEids, eid)
+
+ await this.analyticsService.validateEIDs(eids, pid)
+
+ return this.analyticsService.updateEIDStatus(eids, status, pid)
+ }
+
// Log custom event
@Post('custom')
@UseGuards(BotDetectionGuard)
@@ -1469,6 +1641,173 @@ export class AnalyticsController {
}
}
+ @Get('errors')
+ @Auth([], true, true)
+ async getErrors(
+ @Query() data: GetErrorsDto,
+ @CurrentUserId() uid: string,
+ @Headers() headers: { 'x-password'?: string },
+ ): Promise<any> {
+ const {
+ pid,
+ period,
+ from,
+ to,
+ filters,
+ timezone = DEFAULT_TIMEZONE,
+ options,
+ } = data
+ this.analyticsService.validatePID(pid)
+
+ if (!_isEmpty(period)) {
+ this.analyticsService.validatePeriod(period)
+ }
+
+ await this.analyticsService.checkProjectAccess(
+ pid,
+ uid,
+ headers['x-password'],
+ )
+
+ await this.analyticsService.checkBillingAccess(pid)
+
+ const take = this.analyticsService.getSafeNumber(data.take, 30)
+ const skip = this.analyticsService.getSafeNumber(data.skip, 0)
+
+ if (take > 150) {
+ throw new BadRequestException(
+ 'The maximum number of errors to return is 150',
+ )
+ }
+
+ let timeBucket
+ let diff
+
+ if (period === 'all') {
+ const res = await this.analyticsService.getTimeBucketForAllTime(
+ pid,
+ period,
+ timezone,
+ )
+
+ // eslint-disable-next-line prefer-destructuring
+ timeBucket = res.timeBucket[0]
+ diff = res.diff
+ } else {
+ timeBucket = getLowestPossibleTimeBucket(period, from, to)
+ }
+
+ this.analyticsService.validateTimebucket(timeBucket)
+ const [filtersQuery, filtersParams, appliedFilters] =
+ this.analyticsService.getFiltersQuery(filters, DataType.ANALYTICS)
+
+ const safeTimezone = this.analyticsService.getSafeTimezone(timezone)
+ const { groupFrom, groupTo } = this.analyticsService.getGroupFromTo(
+ from,
+ to,
+ timeBucket,
+ period,
+ safeTimezone,
+ diff,
+ )
+
+ const paramsData = {
+ params: {
+ pid,
+ groupFrom,
+ groupTo,
+ ...filtersParams,
+ },
+ }
+
+ const errors = await this.analyticsService.getErrorsList(
+ options,
+ filtersQuery,
+ paramsData,
+ safeTimezone,
+ take,
+ skip,
+ )
+
+ return {
+ errors,
+ appliedFilters,
+ take,
+ skip,
+ }
+ }
+
+ @Get('get-error')
+ @Auth([], true, true)
+ async getError(
+ @Query() data: GetErrorDTO,
+ @CurrentUserId() uid: string,
+ @Headers() headers: { 'x-password'?: string },
+ ): Promise<any> {
+ const {
+ pid,
+ timezone = DEFAULT_TIMEZONE,
+ eid,
+ period,
+ from,
+ to,
+ //
+ } = data
+ this.analyticsService.validatePID(pid)
+
+ if (!_isEmpty(period)) {
+ this.analyticsService.validatePeriod(period)
+ }
+
+ await this.analyticsService.checkProjectAccess(
+ pid,
+ uid,
+ headers['x-password'],
+ )
+
+ await this.analyticsService.checkBillingAccess(pid)
+
+ let timeBucket
+ let diff
+
+ if (period === 'all') {
+ const res = await this.analyticsService.getTimeBucketForAllTime(
+ pid,
+ period,
+ timezone,
+ )
+
+ // eslint-disable-next-line prefer-destructuring
+ timeBucket = res.timeBucket[0]
+ diff = res.diff
+ } else {
+ timeBucket = getLowestPossibleTimeBucket(period, from, to)
+ }
+
+ this.analyticsService.validateTimebucket(timeBucket)
+
+ const safeTimezone = this.analyticsService.getSafeTimezone(timezone)
+ const { groupFrom, groupTo } = this.analyticsService.getGroupFromTo(
+ from,
+ to,
+ timeBucket,
+ period,
+ safeTimezone,
+ diff,
+ )
+
+ const result = await this.analyticsService.getErrorDetails(
+ pid,
+ eid,
+ safeTimezone,
+ groupFrom,
+ groupTo,
+ timeBucket,
+ )
+
+ return result
+ }
+
@Get('session')
@Auth([], true, true)
async getSession(
diff --git a/apps/production/src/analytics/analytics.service.ts b/apps/production/src/analytics/analytics.service.ts
index e0f301325..e15b152c8 100644
--- a/apps/production/src/analytics/analytics.service.ts
+++ b/apps/production/src/analytics/analytics.service.ts
@@ -48,6 +48,7 @@ import {
TRAFFIC_COLUMNS,
ALL_COLUMNS,
CAPTCHA_COLUMNS,
+ ERROR_COLUMNS,
PERFORMANCE_COLUMNS,
MIN_PAGES_IN_FUNNEL,
MAX_PAGES_IN_FUNNEL,
@@ -91,6 +92,7 @@ import {
IPageflow,
PerfMeasure,
} from './interfaces'
+import { ErrorDTO } from './dto/error.dto'
dayjs.extend(utc)
dayjs.extend(dayjsTimezone)
@@ -268,8 +270,7 @@ const generateParamsQuery = (
subQuery: string,
customEVFilterApplied: boolean,
isPageInclusiveFilterSet: boolean,
- isCaptcha?: boolean,
- isPerformance?: boolean,
+ type: 'traffic' | 'performance' | 'captcha' | 'errors',
measure?: PerfMeasure,
): string => {
let columns = [`${col} as name`]
@@ -282,7 +283,7 @@ const generateParamsQuery = (
const columnsQuery = columns.join(', ')
- if (isPerformance) {
+ if (type === 'performance') {
const processedMeasure = measure === 'quantiles' ? 'median' : measure
const fn = MEASURES_MAP[processedMeasure]
@@ -294,7 +295,11 @@ const generateParamsQuery = (
return `SELECT ${columnsQuery}, round(divide(${fn}(pageLoad), 1000), 2) as count ${subQuery} AND ${col} IS NOT NULL GROUP BY ${columnsQuery}`
}
- if (isCaptcha) {
+ if (type === 'errors') {
+ return `SELECT ${columnsQuery}, count(*) as count ${subQuery} AND ${col} IS NOT NULL GROUP BY ${columnsQuery}`
+ }
+
+ if (type === 'captcha') {
return `SELECT ${columnsQuery}, count(*) as count ${subQuery} AND ${col} IS NOT NULL GROUP BY ${col}`
}
@@ -359,6 +364,11 @@ export class AnalyticsService {
this.projectService.allowedToView(project, uid, password)
}
+ async checkManageAccess(pid: string, uid: string | null): Promise<void> {
+ const project = await this.projectService.getRedisProject(pid)
+ this.projectService.allowedToManage(project, uid)
+ }
+
async checkBillingAccess(pid: string) {
const project = await this.projectService.getRedisProject(pid)
@@ -384,6 +394,7 @@ export class AnalyticsService {
}
} else {
const { hostname } = new URL(origin)
+
if (!isValidOrigin(origins, hostname)) {
throw new BadRequestException(
"This origin is prohibited by the project's origins policy",
@@ -418,9 +429,9 @@ export class AnalyticsService {
}
async validate(
- logDTO: PageviewsDTO | EventsDTO,
+ logDTO: PageviewsDTO | EventsDTO | ErrorDTO,
origin: string,
- type: 'custom' | 'log' = 'log',
+ type: 'custom' | 'log' | 'error' = 'log',
ip?: string,
): Promise<string | null> {
if (_isEmpty(logDTO)) {
@@ -1747,35 +1758,53 @@ export class AnalyticsService {
return _map(results, type)
}
+ async getErrorsFilters(pid: string, type: string): Promise<Array<string>> {
+ if (!_includes(ERROR_COLUMNS, type)) {
+ throw new UnprocessableEntityException(
+ `The provided type (${type}) is incorrect`,
+ )
+ }
+
+ const query = `SELECT ${type} FROM errors WHERE pid={pid:FixedString(12)} AND ${type} IS NOT NULL GROUP BY ${type}`
+
+ const results = await clickhouse
+ .query(query, { params: { pid } })
+ .toPromise()
+
+ return _map(results, type)
+ }
+
async generateParams(
parsedFilters: Array<{ [key: string]: string }>,
subQuery: string,
customEVFilterApplied: boolean,
paramsData: any,
- isCaptcha: boolean,
- isPerformance: boolean,
+ type: 'traffic' | 'performance' | 'captcha' | 'errors',
measure?: PerfMeasure,
): Promise<any> {
const params = {}
// We need this to display all the pageview related data (e.g. country, browser) when user applies an inclusive filter on the Page column
- const isPageInclusiveFilterSet =
- isCaptcha || isPerformance
- ? false
- : !_isEmpty(
- _find(
- parsedFilters,
- filter => filter.column === 'pg' && !filter.isExclusive,
- ),
- )
+ const isPageInclusiveFilterSet = ['captcha', 'performance'].includes(type)
+ ? false
+ : !_isEmpty(
+ _find(
+ parsedFilters,
+ filter => filter.column === 'pg' && !filter.isExclusive,
+ ),
+ )
let columns = TRAFFIC_COLUMNS
- if (isCaptcha) {
+ if (type === 'captcha') {
columns = CAPTCHA_COLUMNS
}
- if (isPerformance) {
+ if (type === 'errors') {
+ columns = ERROR_COLUMNS
+ }
+
+ if (type === 'performance') {
columns = PERFORMANCE_COLUMNS
}
@@ -1785,8 +1814,7 @@ export class AnalyticsService {
subQuery,
customEVFilterApplied,
isPageInclusiveFilterSet,
- isCaptcha,
- isPerformance,
+ type,
measure,
)
const res = await clickhouse.query(query, paramsData).toPromise()
@@ -2189,6 +2217,47 @@ export class AnalyticsService {
return baseQuery
}
+ generateErrorsAggregationQuery(
+ timeBucket: TimeBucketType,
+ filtersQuery: string,
+ safeTimezone: string,
+ mode: ChartRenderMode,
+ ): string {
+ const timeBucketFunc = timeBucketConversion[timeBucket]
+ const [selector, groupBy] = this.getGroupSubquery(timeBucket)
+ const tzFromDate = `toTimeZone(parseDateTimeBestEffort({groupFrom:String}), '${safeTimezone}')`
+ const tzToDate = `toTimeZone(parseDateTimeBestEffort({groupTo:String}), '${safeTimezone}')`
+
+ const baseQuery = `
+ SELECT
+ ${selector},
+ count() as count
+ FROM (
+ SELECT pg, dv, br, os, lc, cc, rg, ct,
+ ${timeBucketFunc}(toTimeZone(created, '${safeTimezone}')) as tz_created
+ FROM errors
+ WHERE
+ pid = {pid:FixedString(12)}
+ AND created BETWEEN ${tzFromDate} AND ${tzToDate}
+ ${filtersQuery}
+ ) as subquery
+ GROUP BY ${groupBy}
+ ORDER BY ${groupBy}
+ `
+
+ if (mode === ChartRenderMode.CUMULATIVE) {
+ return `
+ SELECT
+ *,
+ sum(pageviews) OVER (ORDER BY ${groupBy}) as pageviews,
+ sum(uniques) OVER (ORDER BY ${groupBy}) as uniques
+ FROM (${baseQuery})
+ `
+ }
+
+ return baseQuery
+ }
+
async groupByTimeBucket(
timeBucket: TimeBucketType,
from: string,
@@ -2216,7 +2285,7 @@ export class AnalyticsService {
if (!_some(_values(params), val => !_isEmpty(val))) {
throw new BadRequestException(
- 'The are no parameters for the specified time frames',
+ 'There are no parameters for the specified time frames',
)
}
})(),
@@ -2258,8 +2327,7 @@ export class AnalyticsService {
subQuery,
customEVFilterApplied,
paramsData,
- false,
- false,
+ 'traffic',
)
}
@@ -2365,13 +2433,12 @@ export class AnalyticsService {
subQuery,
false,
paramsData,
- true,
- false,
+ 'captcha',
)
if (!_some(_values(params), val => !_isEmpty(val))) {
throw new BadRequestException(
- 'The are no parameters for the specified time frames',
+ 'There are no parameters for the specified time frames',
)
}
})(),
@@ -2412,6 +2479,73 @@ export class AnalyticsService {
})
}
+ async groupErrorsByTimeBucket(
+ timeBucket: TimeBucketType,
+ from: string,
+ to: string,
+ subQuery: string,
+ filtersQuery: string,
+ paramsData: object,
+ safeTimezone: string,
+ mode: ChartRenderMode,
+ ): Promise<object | void> {
+ let params: unknown = {}
+ let chart: unknown = {}
+
+ const promises = [
+ // Params
+ (async () => {
+ params = await this.generateParams(
+ null,
+ subQuery,
+ false,
+ paramsData,
+ 'errors',
+ )
+
+ if (!_some(_values(params), val => !_isEmpty(val))) {
+ throw new BadRequestException(
+ 'There are no error details for specified time frame',
+ )
+ }
+ })(),
+
+ // Chart data
+ (async () => {
+ const { xShifted } = this.generateXAxis(
+ timeBucket,
+ from,
+ to,
+ safeTimezone,
+ )
+
+ const query = this.generateErrorsAggregationQuery(
+ timeBucket,
+ filtersQuery,
+ safeTimezone,
+ mode,
+ )
+
+ const result = <Array<TrafficCEFilterCHResponse>>(
+ await clickhouse.query(query, paramsData).toPromise()
+ )
+ const { count } = this.extractCaptchaChartData(result, xShifted)
+
+ chart = {
+ x: xShifted,
+ occurrences: count,
+ }
+ })(),
+ ]
+
+ await Promise.all(promises)
+
+ return Promise.resolve({
+ params,
+ chart,
+ })
+ }
+
extractPerformanceChartData(result, x: string[]): any {
const dns = Array(x.length).fill(0)
const tls = Array(x.length).fill(0)
@@ -2545,14 +2679,13 @@ export class AnalyticsService {
subQuery,
false,
paramsData,
- false,
- true,
+ 'performance',
measure,
)
if (!_some(_values(params), val => !_isEmpty(val))) {
throw new BadRequestException(
- 'The are no parameters for the specified time frames',
+ 'There are no parameters for the specified time frames',
)
}
})(),
@@ -2927,6 +3060,236 @@ export class AnalyticsService {
return result
}
+ async getErrorsList(
+ options: string,
+ filtersQuery: string,
+ paramsData: object,
+ safeTimezone: string,
+ take = 30,
+ skip = 0,
+ ): Promise<object | void> {
+ const tzFromDate = `toTimeZone(parseDateTimeBestEffort({groupFrom:String}), '${safeTimezone}')`
+ const tzToDate = `toTimeZone(parseDateTimeBestEffort({groupTo:String}), '${safeTimezone}')`
+
+ let parsedOptions: {
+ showResolved?: boolean
+ } = {}
+
+ try {
+ parsedOptions = JSON.parse(options)
+ } catch (reason) {
+ console.error('[getErrorsList] Failed to parse options:', options)
+ }
+
+ const query = `
+ SELECT
+ eid,
+ any(name) as name,
+ any(message) as message,
+ any(filename) as filename,
+ count(*) as count,
+ max(created) as last_seen,
+ status.status
+ FROM (
+ SELECT eid, name, message, filename, created
+ FROM errors
+ WHERE pid = {pid:FixedString(12)}
+ AND created BETWEEN ${tzFromDate} AND ${tzToDate}
+ ${filtersQuery}
+ ) AS errors
+ LEFT JOIN (
+ SELECT
+ eid,
+ argMax(status, updated) AS status
+ FROM error_statuses
+ WHERE pid = {pid:FixedString(12)}
+ GROUP BY eid
+ ) AS status ON errors.eid = status.eid
+ ${
+ parsedOptions?.showResolved
+ ? ''
+ : "WHERE status.status = 'active' OR status.status = 'regressed'"
+ }
+ GROUP BY errors.eid, status.status
+ ORDER BY last_seen DESC
+ LIMIT ${take}
+ OFFSET ${skip};
+ `
+
+ const result = await clickhouse.query(query, paramsData).toPromise()
+
+ return result
+ }
+
+ async getErrorDetails(
+ pid: string,
+ eid: string,
+ safeTimezone: string,
+ groupFrom: string,
+ groupTo: string,
+ timeBucket: any,
+ ): Promise<any> {
+ const tzFromDate = `toTimeZone(parseDateTimeBestEffort({groupFrom:String}), '${safeTimezone}')`
+ const tzToDate = `toTimeZone(parseDateTimeBestEffort({groupTo:String}), '${safeTimezone}')`
+
+ const queryErrorDetails = `
+ SELECT
+ subquery.eid,
+ any(subquery.name) AS name,
+ any(subquery.message) AS message,
+ any(subquery.filename) AS filename,
+ any(subquery.colno) AS colno,
+ any(subquery.lineno) AS lineno,
+ count(*) AS count,
+ status.status
+ FROM (
+ SELECT
+ eid,
+ name,
+ message,
+ filename,
+ colno,
+ lineno
+ FROM errors
+ WHERE pid = {pid:FixedString(12)}
+ AND eid = {eid:FixedString(32)}
+ AND created BETWEEN ${tzFromDate} AND ${tzToDate}
+ ) AS subquery
+ LEFT JOIN (
+ SELECT
+ eid,
+ argMax(status, updated) AS status
+ FROM error_statuses
+ WHERE pid = {pid:FixedString(12)}
+ AND eid = {eid:FixedString(32)}
+ GROUP BY eid
+ ) AS status ON subquery.eid = status.eid
+ GROUP BY subquery.eid, status.status;
+ `
+
+ const queryFirstLastSeen = `
+ SELECT
+ max(created) AS last_seen,
+ min(created) AS first_seen
+ FROM errors
+ WHERE
+ pid = {pid:FixedString(12)}
+ AND eid = {eid:FixedString(32)};
+ `
+
+ const paramsData = {
+ params: {
+ pid,
+ eid,
+ groupFrom,
+ groupTo,
+ },
+ }
+
+ const details = (
+ await clickhouse.query(queryErrorDetails, paramsData).toPromise()
+ )[0]
+
+ const occurenceDetails = (
+ await clickhouse.query(queryFirstLastSeen, paramsData).toPromise()
+ )[0]
+
+ const groupedChart = await this.groupErrorsByTimeBucket(
+ timeBucket,
+ groupFrom,
+ groupTo,
+ `FROM errors WHERE eid = {eid:FixedString(32)} AND created BETWEEN {groupFrom:String} AND {groupTo:String}`,
+ 'AND eid = {eid:FixedString(32)}',
+ paramsData,
+ safeTimezone,
+ ChartRenderMode.PERIODICAL,
+ )
+
+ return {
+ details: {
+ ...details,
+ ...occurenceDetails,
+ },
+ ...groupedChart,
+ timeBucket,
+ }
+ }
+
+ getErrorID(errorDTO: ErrorDTO): string {
+ const { name, message, colno, lineno, filename } = errorDTO
+
+ return hash(`${name}${message}${colno}${lineno}${filename}`, {
+ length: 16,
+ }).toString('hex')
+ }
+
+ async validateEIDs(eids: string[], pid: string) {
+ const params = _reduce(
+ eids,
+ (acc, curr, index) => ({
+ ...acc,
+ [`e_${index}`]: curr,
+ }),
+ {},
+ )
+
+ const query = `SELECT count(DISTINCT eid) as count FROM errors WHERE pid = {pid:FixedString(12)} AND eid IN (${_map(
+ params,
+ (val, key) => `{${key}:FixedString(32)}`,
+ ).join(', ')})`
+
+ let result
+
+ try {
+ ;[result] = await clickhouse
+ .query(query, { params: { ...params, pid } })
+ .toPromise()
+ } catch (reason) {
+ console.error('validateEIDs - clickhouse request error')
+ console.error(reason)
+ throw new InternalServerErrorException(
+ 'Error occured while validating error IDs',
+ )
+ }
+
+ if (result.count !== _size(eids)) {
+ throw new UnprocessableEntityException(
+ 'Some of the error IDs you provided are not valid (do not exist in our database)',
+ )
+ }
+ }
+
+ async updateEIDStatus(
+ eids: string[],
+ status: 'resolved' | 'active',
+ pid: string,
+ ) {
+ const params = _reduce(
+ eids,
+ (acc, curr, index) => ({
+ ...acc,
+ [`e_${index}`]: curr,
+ }),
+ {},
+ )
+
+ const query = `INSERT INTO error_statuses (eid, pid, status) VALUES ${_map(
+ params,
+ (val, key) =>
+ `({${key}:FixedString(32)}, {pid:FixedString(12)}, '${status}')`,
+ ).join(', ')}`
+
+ try {
+ await clickhouse.query(query, { params: { ...params, pid } }).toPromise()
+ } catch (reason) {
+ console.error('Error at PATCH error-status:')
+ console.error(reason)
+ throw new InternalServerErrorException(
+ 'Error occured while updating error status',
+ )
+ }
+ }
+
async getOnlineCountByProjectId(projectId: string) {
// @ts-ignore
return redis.countKeysByPattern(`hb:${projectId}:*`)
@@ -2942,19 +3305,28 @@ export class AnalyticsService {
}
}
- async getGeneralStats(): Promise<any> {
- const trafficQuery = 'SELECT count(*) FROM analytics'
- const customEVQuery = 'SELECT count(*) FROM customEV'
- const performanceQuery = 'SELECT count(*) FROM performance'
- const captchaQuery = 'SELECT count(*) FROM captcha'
+ async getGeneralStats(): Promise<{
+ users: number
+ projects: number
+ events: number
+ }> {
+ const query = `
+ SELECT 'traffic' AS type, count(*) AS count FROM analytics
+ UNION ALL
+ SELECT 'customEV' AS type, count(*) AS count FROM customEV
+ UNION ALL
+ SELECT 'performance' AS type, count(*) AS count FROM performance
+ UNION ALL
+ SELECT 'captcha' AS type, count(*) AS count FROM captcha
+ UNION ALL
+ SELECT 'errors' AS type, count(*) AS count FROM errors
+ `
const users = await this.userService.count()
const projects = await this.projectService.count()
- const events =
- (await clickhouse.query(trafficQuery).toPromise())[0]['count()'] +
- (await clickhouse.query(customEVQuery).toPromise())[0]['count()'] +
- (await clickhouse.query(performanceQuery).toPromise())[0]['count()'] +
- (await clickhouse.query(captchaQuery).toPromise())[0]['count()']
+ const results = await clickhouse.query(query).toPromise()
+ // @ts-expect-error
+ const events = results.reduce((total, row) => total + row.count, 0)
await redis.set(REDIS_USERS_COUNT_KEY, users, 'EX', 630)
await redis.set(REDIS_PROJECTS_COUNT_KEY, projects, 'EX', 630)
diff --git a/apps/production/src/analytics/dto/error.dto.ts b/apps/production/src/analytics/dto/error.dto.ts
new file mode 100644
index 000000000..efa35255e
--- /dev/null
+++ b/apps/production/src/analytics/dto/error.dto.ts
@@ -0,0 +1,91 @@
+import { ApiProperty } from '@nestjs/swagger'
+import {
+ IsNotEmpty,
+ IsNumber,
+ IsOptional,
+ IsString,
+ MaxLength,
+ Matches,
+} from 'class-validator'
+import { PID_REGEX } from '../../common/constants'
+
+export class ErrorDTO {
+ @ApiProperty({
+ example: 'aUn1quEid-3',
+ required: true,
+ description: 'The project ID',
+ })
+ @IsNotEmpty()
+ @Matches(PID_REGEX, {
+ message: 'Incorrect project ID format.',
+ })
+ pid: string
+
+ // Tracking metrics
+ @ApiProperty({
+ example: 'Europe/Kiev',
+ description: "User's timezone",
+ })
+ @IsOptional()
+ @IsString()
+ tz?: string
+
+ @ApiProperty({
+ example: '/articles/my-awesome-article-1',
+ description: 'A page that user sent data from',
+ })
+ @IsOptional()
+ @IsString()
+ pg?: string
+
+ @ApiProperty({
+ example: 'en-GB',
+ description: "User's locale",
+ })
+ @IsOptional()
+ @IsString()
+ lc?: string
+
+ // Error metrics
+ @ApiProperty({
+ example: 'ParseError',
+ required: true,
+ maxLength: 200,
+ })
+ @IsNotEmpty()
+ @IsString()
+ @MaxLength(200)
+ name: string
+
+ @ApiProperty({
+ example: 'Malformed input',
+ maxLength: 2000,
+ })
+ @IsOptional()
+ @IsString()
+ @MaxLength(2000)
+ message?: string
+
+ @ApiProperty({
+ example: 12,
+ })
+ @IsOptional()
+ @IsNumber()
+ lineno?: number
+
+ @ApiProperty({
+ example: 510,
+ })
+ @IsOptional()
+ @IsNumber()
+ colno?: number
+
+ @ApiProperty({
+ example: 'https://example.com/assets/convert.js',
+ maxLength: 1000,
+ })
+ @IsOptional()
+ @IsString()
+ @MaxLength(1000)
+ filename?: string
+}
diff --git a/apps/production/src/analytics/dto/get-error.dto.ts b/apps/production/src/analytics/dto/get-error.dto.ts
new file mode 100644
index 000000000..ee8e21b6e
--- /dev/null
+++ b/apps/production/src/analytics/dto/get-error.dto.ts
@@ -0,0 +1,15 @@
+import { ApiProperty, PickType } from '@nestjs/swagger'
+import { IsNotEmpty } from 'class-validator'
+import { AnalyticsGET_DTO } from './getData.dto'
+
+export class GetErrorDTO extends PickType(AnalyticsGET_DTO, [
+ 'pid',
+ 'period',
+ 'from',
+ 'to',
+ 'timezone',
+] as const) {
+ @ApiProperty()
+ @IsNotEmpty()
+ eid: string
+}
diff --git a/apps/production/src/analytics/dto/get-errors.dto.ts b/apps/production/src/analytics/dto/get-errors.dto.ts
new file mode 100644
index 000000000..6d85cac2d
--- /dev/null
+++ b/apps/production/src/analytics/dto/get-errors.dto.ts
@@ -0,0 +1,23 @@
+import { PickType, ApiProperty } from '@nestjs/swagger'
+import { AnalyticsGET_DTO } from './getData.dto'
+
+export class GetErrorsDto extends PickType(AnalyticsGET_DTO, [
+ 'pid',
+ 'period',
+ 'from',
+ 'to',
+ 'filters',
+ 'timezone',
+] as const) {
+ take: number
+
+ skip: number
+
+ @ApiProperty({
+ example: {
+ showResolved: true,
+ },
+ description: 'Errors list options',
+ })
+ options?: string
+}
diff --git a/apps/production/src/analytics/dto/patch-status.dto.ts b/apps/production/src/analytics/dto/patch-status.dto.ts
new file mode 100644
index 000000000..2da498d93
--- /dev/null
+++ b/apps/production/src/analytics/dto/patch-status.dto.ts
@@ -0,0 +1,52 @@
+import { ApiProperty } from '@nestjs/swagger'
+import {
+ IsNotEmpty,
+ IsIn,
+ ValidateIf,
+ IsString,
+ IsArray,
+ Matches,
+} from 'class-validator'
+import { PID_REGEX } from '../../common/constants'
+
+export class PatchStatusDTO {
+ @ApiProperty({
+ example: 'aUn1quEid-3',
+ required: true,
+ description: 'The project ID',
+ })
+ @IsNotEmpty()
+ @Matches(PID_REGEX, {
+ message: 'Incorrect project ID format.',
+ })
+ pid: string
+
+ @ApiProperty({
+ example: 'c138c2b1826d9fec65d230e471ab2c25',
+ description: 'Error id',
+ })
+ @ValidateIf(o => !o.eids || o.eid)
+ @IsNotEmpty()
+ @IsString()
+ eid: string
+
+ @ApiProperty({
+ example: [
+ 'c138c2b1826d9fec65d230e471ab2c25',
+ 'b2d12867d59d1d4f9986b52311a97cd5',
+ ],
+ description: 'Error id list',
+ })
+ @ValidateIf(o => o.eids || !o.eid)
+ @IsNotEmpty()
+ @IsArray()
+ @IsString({ each: true })
+ eids: string
+
+ @ApiProperty({
+ enum: ['resolved', 'active'],
+ required: true,
+ })
+ @IsIn(['resolved', 'active'])
+ status: 'resolved' | 'active'
+}
diff --git a/apps/production/src/common/constants.ts b/apps/production/src/common/constants.ts
index 2487caaff..ef06c109e 100644
--- a/apps/production/src/common/constants.ts
+++ b/apps/production/src/common/constants.ts
@@ -79,6 +79,7 @@ const REDIS_LOG_DATA_CACHE_KEY = 'log_cache'
const REDIS_LOG_CAPTCHA_CACHE_KEY = 'log:captcha'
const REDIS_LOG_PERF_CACHE_KEY = 'perf_cache'
const REDIS_LOG_CUSTOM_CACHE_KEY = 'log_custom_cache_v3'
+const REDIS_LOG_ERROR_CACHE_KEY = 'log_error_cache'
const REDIS_SESSION_SALT_KEY = 'log_salt' // is updated every 24 hours
const REDIS_USERS_COUNT_KEY = 'stats:users_count'
const REDIS_PROJECTS_COUNT_KEY = 'stats:projects_count'
@@ -145,6 +146,8 @@ const TRAFFIC_COLUMNS = [
'ca',
]
+const ERROR_COLUMNS = ['pg', 'dv', 'br', 'os', 'lc', 'cc', 'rg', 'ct']
+
const ALL_COLUMNS = [...TRAFFIC_COLUMNS, 'ev']
const CAPTCHA_COLUMNS = ['cc', 'br', 'os', 'dv']
@@ -230,6 +233,7 @@ export {
redisUserUsageinfoCacheTimeout,
TRAFFIC_COLUMNS,
CAPTCHA_COLUMNS,
+ ERROR_COLUMNS,
PERFORMANCE_COLUMNS,
sentryIgnoreErrors,
isProxiedByCloudflare,
@@ -243,4 +247,6 @@ export {
BLOG_POSTS_ROOT,
TRAFFIC_SPIKE_ALLOWED_PERCENTAGE,
AFFILIATE_CUT,
+ REDIS_LOG_ERROR_CACHE_KEY,
+ PID_REGEX,
}
diff --git a/apps/production/src/project/project.controller.ts b/apps/production/src/project/project.controller.ts
index 8a78e06c0..d5e47d112 100644
--- a/apps/production/src/project/project.controller.ts
+++ b/apps/production/src/project/project.controller.ts
@@ -173,11 +173,17 @@ export class ProjectController {
_map(paginated.results, ({ id }) => id),
)
+ const pidsWithErrorData =
+ await this.projectService.getPIDsWhereErrorsDataExists(
+ _map(paginated.results, ({ id }) => id),
+ )
+
paginated.results = _map(paginated.results, p => ({
...p,
isOwner: true,
isLocked: !!user?.dashboardBlockReason,
isDataExists: _includes(pidsWithData, p?.id),
+ isErrorDataExists: _includes(pidsWithErrorData, p?.id),
}))
return {
@@ -250,6 +256,11 @@ export class ProjectController {
_map(paginated.results, ({ project }) => project.id),
)
+ const pidsWithErrorData =
+ await this.projectService.getPIDsWhereErrorsDataExists(
+ _map(paginated.results, ({ project }) => project.id),
+ )
+
paginated.results = _map(paginated.results, share => {
const project = processProjectUser(share.project)
@@ -261,6 +272,7 @@ export class ProjectController {
passwordHash: undefined,
isLocked: !!share?.project?.admin?.dashboardBlockReason,
isDataExists: _includes(pidsWithData, share?.project?.id),
+ isErrorDataExists: _includes(pidsWithErrorData, share?.project?.id),
},
}
})
@@ -1840,11 +1852,16 @@ export class ProjectController {
await this.projectService.getPIDsWhereAnalyticsDataExists([id]),
)
+ const isErrorDataExists = !_isEmpty(
+ await this.projectService.getPIDsWhereErrorsDataExists([id]),
+ )
+
return {
..._omit(project, ['admin', 'passwordHash', 'share']),
isOwner: uid === project.admin?.id,
isLocked: !!project.admin?.dashboardBlockReason,
isDataExists,
+ isErrorDataExists,
}
}
}
diff --git a/apps/production/src/project/project.service.ts b/apps/production/src/project/project.service.ts
index 097ab761c..2cd1cd0ab 100644
--- a/apps/production/src/project/project.service.ts
+++ b/apps/production/src/project/project.service.ts
@@ -888,6 +888,50 @@ export class ProjectService {
return _map(result, ({ pid }) => pid)
}
+ async getPIDsWhereErrorsDataExists(projectIds: string[]) {
+ if (_isEmpty(projectIds)) {
+ return {}
+ }
+
+ const params = _reduce(
+ projectIds,
+ (acc, curr, index) => ({
+ ...acc,
+ [`pid_${index}`]: curr,
+ }),
+ {},
+ )
+
+ const pids = _join(
+ _map(params, (val, key) => `{${key}:FixedString(12)}`),
+ ',',
+ )
+
+ const query = `
+ SELECT
+ pid,
+ CASE
+ WHEN EXISTS (
+ SELECT 1
+ FROM errors
+ WHERE pid IN (${pids})
+ )
+ THEN 1
+ ELSE 0
+ END AS exists
+ FROM
+ (
+ SELECT DISTINCT pid
+ FROM errors
+ WHERE pid IN (${pids})
+ );
+ `
+
+ const result: any = await clickhouse.query(query, { params }).toPromise()
+
+ return _map(result, ({ pid }) => pid)
+ }
+
async getSubscriberByEmail(projectId: string, email: string) {
return this.projectSubscriberRepository.findOne({
where: { projectId, email },
diff --git a/apps/production/src/task-manager/task-manager.service.ts b/apps/production/src/task-manager/task-manager.service.ts
index 683e6ce06..1fbe21e5d 100644
--- a/apps/production/src/task-manager/task-manager.service.ts
+++ b/apps/production/src/task-manager/task-manager.service.ts
@@ -57,6 +57,7 @@ import {
PAYPAL_CLIENT_SECRET,
TRAFFIC_SPIKE_ALLOWED_PERCENTAGE,
isDevelopment,
+ REDIS_LOG_ERROR_CACHE_KEY,
} from '../common/constants'
import { CHPlanUsage } from './interfaces'
import { getRandomTip } from '../common/utils'
@@ -412,6 +413,7 @@ export class TaskManagerService {
const customData = await redis.lrange(REDIS_LOG_CUSTOM_CACHE_KEY, 0, -1)
const perfData = await redis.lrange(REDIS_LOG_PERF_CACHE_KEY, 0, -1)
const captchaData = await redis.lrange(REDIS_LOG_CAPTCHA_CACHE_KEY, 0, -1)
+ const errorData = await redis.lrange(REDIS_LOG_ERROR_CACHE_KEY, 0, -1)
if (!_isEmpty(data)) {
await redis.del(REDIS_LOG_DATA_CACHE_KEY)
@@ -448,6 +450,19 @@ export class TaskManagerService {
}
}
+ if (!_isEmpty(errorData)) {
+ await redis.del(REDIS_LOG_ERROR_CACHE_KEY)
+ const query = `INSERT INTO errors (*) VALUES ${_join(errorData, ',')}`
+
+ try {
+ await clickhouse.query(query).toPromise()
+ } catch (e) {
+ console.error(
+ `[CRON WORKER] Error whilst saving error events data: ${e}`,
+ )
+ }
+ }
+
if (!_isEmpty(perfData)) {
await redis.del(REDIS_LOG_PERF_CACHE_KEY)
diff --git a/apps/selfhosted/src/analytics/analytics.controller.ts b/apps/selfhosted/src/analytics/analytics.controller.ts
index 1bc62af34..a125666ba 100644
--- a/apps/selfhosted/src/analytics/analytics.controller.ts
+++ b/apps/selfhosted/src/analytics/analytics.controller.ts
@@ -21,6 +21,7 @@ import {
UseGuards,
Get,
Post,
+ Patch,
Headers,
BadRequestException,
InternalServerErrorException,
@@ -64,8 +65,13 @@ import {
MIN_PAGES_IN_FUNNEL,
MAX_PAGES_IN_FUNNEL,
clickhouse,
+ REDIS_LOG_ERROR_CACHE_KEY,
} from '../common/constants'
-import { getGeoDetails, getIPFromHeaders } from '../common/utils'
+import {
+ checkRateLimit,
+ getGeoDetails,
+ getIPFromHeaders,
+} from '../common/utils'
import { BotDetection } from '../common/decorators/bot-detection.decorator'
import { BotDetectionGuard } from '../common/guards/bot-detection.guard'
import { GetCustomEventsDto } from './dto/get-custom-events.dto'
@@ -78,6 +84,10 @@ import {
} from './interfaces'
import { GetSessionsDto } from './dto/get-sessions.dto'
import { GetSessionDto } from './dto/get-session.dto'
+import { ErrorDTO } from './dto/error.dto'
+import { GetErrorsDto } from './dto/get-errors.dto'
+import { GetErrorDTO } from './dto/get-error.dto'
+import { PatchStatusDTO } from './dto/patch-status.dto'
// eslint-disable-next-line @typescript-eslint/no-var-requires
const mysql = require('mysql2')
@@ -216,6 +226,43 @@ const customLogDTO = (
]
}
+const errorLogDTO = (
+ eid: string,
+ pid: string,
+ pg: string,
+ dv: string,
+ br: string,
+ os: string,
+ lc: string,
+ cc: string,
+ rg: string,
+ ct: string,
+ name: string,
+ message: string,
+ lineno: number,
+ colno: number,
+ filename: string,
+): Array<string | number | string[]> => {
+ return [
+ eid,
+ pid,
+ pg,
+ dv,
+ br,
+ os,
+ lc,
+ cc,
+ rg,
+ ct,
+ name,
+ message,
+ lineno,
+ colno,
+ filename,
+ dayjs.utc().format('YYYY-MM-DD HH:mm:ss'),
+ ]
+}
+
const getElValue = el => {
if (_isArray(el)) {
return `[${_map(el, getElValue).join(',')}]`
@@ -280,6 +327,27 @@ const getPIDsArray = (pids, pid) => {
return pids
}
+const getEIDsArray = (eids, eid) => {
+ const eidsEmpty = _isEmpty(eids)
+ const eidEmpty = _isEmpty(eid)
+ if (eidsEmpty && eidEmpty) {
+ throw new BadRequestException(
+ "An array of Error ID's (eids) or a Error ID (eid) has to be provided",
+ )
+ }
+ if (!eidsEmpty && !eidEmpty) {
+ throw new BadRequestException(
+ "Please provide either an array of Error ID's (eids) or a Error ID (eid), but not both",
+ )
+ }
+
+ if (!eidsEmpty) {
+ return eids
+ }
+
+ return [eid]
+}
+
const getPagesArray = (rawPages: string): string[] => {
try {
const pages = JSON.parse(rawPages)
@@ -1431,4 +1499,253 @@ export class AnalyticsController {
timeBucket: timeBucketForAllTime,
}
}
+
+ @Get('errors-filters')
+ @Auth([], true, true)
+ async getErrorsFilters(
+ @Query() data: GetFiltersDto,
+ @CurrentUserId() uid: string,
+ ): Promise<any> {
+ const { pid, type } = data
+ this.analyticsService.validatePID(pid)
+
+ await this.analyticsService.checkProjectAccess(pid, uid)
+
+ return this.analyticsService.getErrorsFilters(pid, type)
+ }
+
+ // Log error event
+ @Post('error')
+ @UseGuards(BotDetectionGuard)
+ @BotDetection()
+ @Public()
+ async logError(
+ @Body() errorDTO: ErrorDTO,
+ @Headers() headers,
+ @Ip() reqIP,
+ ): Promise<any> {
+ const { 'user-agent': userAgent, origin } = headers
+
+ const ip = getIPFromHeaders(headers) || reqIP || ''
+
+ await this.analyticsService.validate(errorDTO, origin, 'error', ip)
+
+ const {
+ city = 'NULL',
+ region = 'NULL',
+ country = 'NULL',
+ } = getGeoDetails(ip, errorDTO.tz)
+
+ const ua = UAParser(userAgent)
+ const dv = ua.device.type || 'desktop'
+ const br = ua.browser.name
+ const os = ua.os.name
+
+ const { name, message, lineno, colno, filename } = errorDTO
+
+ const dto = errorLogDTO(
+ this.analyticsService.getErrorID(errorDTO),
+ errorDTO.pid,
+ errorDTO.pg,
+ dv,
+ br,
+ os,
+ errorDTO.lc,
+ country,
+ region,
+ city,
+ name,
+ message,
+ lineno,
+ colno,
+ filename,
+ )
+
+ try {
+ const values = `(${dto.map(getElValue).join(',')})`
+ await redis.rpush(REDIS_LOG_ERROR_CACHE_KEY, values)
+ } catch (e) {
+ this.logger.error(e)
+ throw new InternalServerErrorException(
+ 'Error occured while saving the custom event',
+ )
+ }
+ }
+
+ // Update error(s) status
+ @Patch('error-status')
+ @Auth([], true, true)
+ async patchStatus(
+ @Body() statusDTO: PatchStatusDTO,
+ @CurrentUserId() uid: string,
+ @Headers() headers,
+ @Ip() reqIP,
+ ): Promise<any> {
+ const { pid, eid, eids: unprocessedEids, status } = statusDTO
+ const ip = getIPFromHeaders(headers) || reqIP || ''
+
+ await checkRateLimit(ip, 'error-status', 100, 1800)
+
+ this.analyticsService.validatePID(pid)
+ await this.analyticsService.checkProjectAccess(pid, uid)
+
+ const eids = getEIDsArray(unprocessedEids, eid)
+
+ await this.analyticsService.validateEIDs(eids, pid)
+
+ return this.analyticsService.updateEIDStatus(eids, status, pid)
+ }
+
+ @Get('errors')
+ @Auth([], true, true)
+ async getErrors(
+ @Query() data: GetErrorsDto,
+ @CurrentUserId() uid: string,
+ ): Promise<any> {
+ const {
+ pid,
+ period,
+ from,
+ to,
+ filters,
+ timezone = DEFAULT_TIMEZONE,
+ options,
+ } = data
+ this.analyticsService.validatePID(pid)
+
+ if (!_isEmpty(period)) {
+ this.analyticsService.validatePeriod(period)
+ }
+
+ await this.analyticsService.checkProjectAccess(pid, uid)
+
+ const take = this.analyticsService.getSafeNumber(data.take, 30)
+ const skip = this.analyticsService.getSafeNumber(data.skip, 0)
+
+ if (take > 150) {
+ throw new BadRequestException(
+ 'The maximum number of errors to return is 150',
+ )
+ }
+
+ let timeBucket
+ let diff
+
+ if (period === 'all') {
+ const res = await this.analyticsService.getTimeBucketForAllTime(
+ pid,
+ period,
+ timezone,
+ )
+
+ // eslint-disable-next-line prefer-destructuring
+ timeBucket = res.timeBucket[0]
+ diff = res.diff
+ } else {
+ timeBucket = getLowestPossibleTimeBucket(period, from, to)
+ }
+
+ this.analyticsService.validateTimebucket(timeBucket)
+ const [filtersQuery, filtersParams, appliedFilters] =
+ this.analyticsService.getFiltersQuery(filters, DataType.ANALYTICS)
+
+ const safeTimezone = this.analyticsService.getSafeTimezone(timezone)
+ const { groupFrom, groupTo } = this.analyticsService.getGroupFromTo(
+ from,
+ to,
+ timeBucket,
+ period,
+ safeTimezone,
+ diff,
+ )
+
+ const paramsData = {
+ params: {
+ pid,
+ groupFrom,
+ groupTo,
+ ...filtersParams,
+ },
+ }
+
+ const errors = await this.analyticsService.getErrorsList(
+ options,
+ filtersQuery,
+ paramsData,
+ safeTimezone,
+ take,
+ skip,
+ )
+
+ return {
+ errors,
+ appliedFilters,
+ take,
+ skip,
+ }
+ }
+
+ @Get('get-error')
+ @Auth([], true, true)
+ async getError(
+ @Query() data: GetErrorDTO,
+ @CurrentUserId() uid: string,
+ ): Promise<any> {
+ const {
+ pid,
+ timezone = DEFAULT_TIMEZONE,
+ eid,
+ period,
+ from,
+ to,
+ //
+ } = data
+ this.analyticsService.validatePID(pid)
+
+ if (!_isEmpty(period)) {
+ this.analyticsService.validatePeriod(period)
+ }
+
+ await this.analyticsService.checkProjectAccess(pid, uid)
+
+ let timeBucket
+ let diff
+
+ if (period === 'all') {
+ const res = await this.analyticsService.getTimeBucketForAllTime(
+ pid,
+ period,
+ timezone,
+ )
+
+ // eslint-disable-next-line prefer-destructuring
+ timeBucket = res.timeBucket[0]
+ diff = res.diff
+ } else {
+ timeBucket = getLowestPossibleTimeBucket(period, from, to)
+ }
+
+ this.analyticsService.validateTimebucket(timeBucket)
+
+ const safeTimezone = this.analyticsService.getSafeTimezone(timezone)
+ const { groupFrom, groupTo } = this.analyticsService.getGroupFromTo(
+ from,
+ to,
+ timeBucket,
+ period,
+ safeTimezone,
+ diff,
+ )
+
+ const result = await this.analyticsService.getErrorDetails(
+ pid,
+ eid,
+ safeTimezone,
+ groupFrom,
+ groupTo,
+ timeBucket,
+ )
+
+ return result
+ }
}
diff --git a/apps/selfhosted/src/analytics/analytics.service.ts b/apps/selfhosted/src/analytics/analytics.service.ts
index 51b7f76c2..073c162b7 100644
--- a/apps/selfhosted/src/analytics/analytics.service.ts
+++ b/apps/selfhosted/src/analytics/analytics.service.ts
@@ -45,6 +45,7 @@ import {
REDIS_SESSION_SALT_KEY,
TRAFFIC_COLUMNS,
PERFORMANCE_COLUMNS,
+ ERROR_COLUMNS,
} from '../common/constants'
import { millisecondsToSeconds, sumArrays } from '../common/utils'
import { PageviewsDTO } from './dto/pageviews.dto'
@@ -76,6 +77,7 @@ import {
IPageflow,
PerfMeasure,
} from './interfaces'
+import { ErrorDTO } from './dto/error.dto'
dayjs.extend(utc)
dayjs.extend(dayjsTimezone)
@@ -253,7 +255,7 @@ const generateParamsQuery = (
subQuery: string,
customEVFilterApplied: boolean,
isPageInclusiveFilterSet: boolean,
- isPerformance?: boolean,
+ type: 'traffic' | 'performance' | 'errors',
measure?: PerfMeasure,
): string => {
let columns = [`${col} as name`]
@@ -266,7 +268,7 @@ const generateParamsQuery = (
const columnsQuery = columns.join(', ')
- if (isPerformance) {
+ if (type === 'performance') {
const processedMeasure = measure === 'quantiles' ? 'median' : measure
const fn = MEASURES_MAP[processedMeasure]
@@ -278,6 +280,10 @@ const generateParamsQuery = (
return `SELECT ${columnsQuery}, round(divide(${fn}(pageLoad), 1000), 2) as count ${subQuery} AND ${col} IS NOT NULL GROUP BY ${columnsQuery}`
}
+ if (type === 'errors') {
+ return `SELECT ${columnsQuery}, count(*) as count ${subQuery} AND ${col} IS NOT NULL GROUP BY ${columnsQuery}`
+ }
+
if (customEVFilterApplied) {
return `SELECT ${columnsQuery}, count(*) as count ${subQuery} AND ${col} IS NOT NULL GROUP BY ${columnsQuery}`
}
@@ -379,9 +385,9 @@ export class AnalyticsService {
}
async validate(
- logDTO: PageviewsDTO | EventsDTO,
+ logDTO: PageviewsDTO | EventsDTO | ErrorDTO,
origin: string,
- type: 'custom' | 'log' = 'log',
+ type: 'custom' | 'log' | 'error' = 'log',
ip?: string,
): Promise<string | null> {
if (_isEmpty(logDTO)) {
@@ -439,6 +445,22 @@ export class AnalyticsService {
return null
}
+ async getErrorsFilters(pid: string, type: string): Promise<Array<string>> {
+ if (!_includes(ERROR_COLUMNS, type)) {
+ throw new UnprocessableEntityException(
+ `The provided type (${type}) is incorrect`,
+ )
+ }
+
+ const query = `SELECT ${type} FROM errors WHERE pid={pid:FixedString(12)} AND ${type} IS NOT NULL GROUP BY ${type}`
+
+ const results = await clickhouse
+ .query(query, { params: { pid } })
+ .toPromise()
+
+ return _map(results, type)
+ }
+
getDataTypeColumns(dataType: DataType): string[] {
if (dataType === DataType.ANALYTICS) {
return TRAFFIC_COLUMNS
@@ -1483,34 +1505,39 @@ export class AnalyticsService {
subQuery: string,
customEVFilterApplied: boolean,
paramsData: any,
- isPerformance: boolean,
+ type: 'traffic' | 'performance' | 'errors',
measure?: PerfMeasure,
): Promise<any> {
const params = {}
// We need this to display all the pageview related data (e.g. country, browser) when user applies an inclusive filter on the Page column
- const isPageInclusiveFilterSet = isPerformance
- ? false
- : !_isEmpty(
- _find(
- parsedFilters,
- filter => filter.column === 'pg' && !filter.isExclusive,
- ),
- )
+ const isPageInclusiveFilterSet =
+ type === 'performance'
+ ? false
+ : !_isEmpty(
+ _find(
+ parsedFilters,
+ filter => filter.column === 'pg' && !filter.isExclusive,
+ ),
+ )
let columns = TRAFFIC_COLUMNS
- if (isPerformance) {
+ if (type === 'performance') {
columns = PERFORMANCE_COLUMNS
}
+ if (type === 'errors') {
+ columns = ERROR_COLUMNS
+ }
+
const paramsPromises = _map(columns, async col => {
const query = generateParamsQuery(
col,
subQuery,
customEVFilterApplied,
isPageInclusiveFilterSet,
- isPerformance,
+ type,
measure,
)
const res = await clickhouse.query(query, paramsData).toPromise()
@@ -1909,6 +1936,47 @@ export class AnalyticsService {
}
}
+ generateErrorsAggregationQuery(
+ timeBucket: TimeBucketType,
+ filtersQuery: string,
+ safeTimezone: string,
+ mode: ChartRenderMode,
+ ): string {
+ const timeBucketFunc = timeBucketConversion[timeBucket]
+ const [selector, groupBy] = this.getGroupSubquery(timeBucket)
+ const tzFromDate = `toTimeZone(parseDateTimeBestEffort({groupFrom:String}), '${safeTimezone}')`
+ const tzToDate = `toTimeZone(parseDateTimeBestEffort({groupTo:String}), '${safeTimezone}')`
+
+ const baseQuery = `
+ SELECT
+ ${selector},
+ count() as count
+ FROM (
+ SELECT pg, dv, br, os, lc, cc, rg, ct,
+ ${timeBucketFunc}(toTimeZone(created, '${safeTimezone}')) as tz_created
+ FROM errors
+ WHERE
+ pid = {pid:FixedString(12)}
+ AND created BETWEEN ${tzFromDate} AND ${tzToDate}
+ ${filtersQuery}
+ ) as subquery
+ GROUP BY ${groupBy}
+ ORDER BY ${groupBy}
+ `
+
+ if (mode === ChartRenderMode.CUMULATIVE) {
+ return `
+ SELECT
+ *,
+ sum(pageviews) OVER (ORDER BY ${groupBy}) as pageviews,
+ sum(uniques) OVER (ORDER BY ${groupBy}) as uniques
+ FROM (${baseQuery})
+ `
+ }
+
+ return baseQuery
+ }
+
async groupByTimeBucket(
timeBucket: TimeBucketType,
from: string,
@@ -1936,7 +2004,7 @@ export class AnalyticsService {
if (!_some(_values(params), val => !_isEmpty(val))) {
throw new BadRequestException(
- 'The are no parameters for the specified time frames',
+ 'There are no parameters for the specified time frames',
)
}
})(),
@@ -1978,7 +2046,7 @@ export class AnalyticsService {
subQuery,
customEVFilterApplied,
paramsData,
- false,
+ 'traffic',
)
}
@@ -2045,6 +2113,91 @@ export class AnalyticsService {
})
}
+ extractErrorsChartData(result, x: string[]): any {
+ const count = Array(x.length).fill(0)
+
+ for (let row = 0; row < _size(result); ++row) {
+ const dateString = this.generateDateString(result[row])
+
+ const index = x.indexOf(dateString)
+
+ if (index !== -1) {
+ count[index] = result[row].count
+ }
+ }
+
+ return {
+ count,
+ }
+ }
+
+ async groupErrorsByTimeBucket(
+ timeBucket: TimeBucketType,
+ from: string,
+ to: string,
+ subQuery: string,
+ filtersQuery: string,
+ paramsData: object,
+ safeTimezone: string,
+ mode: ChartRenderMode,
+ ): Promise<object | void> {
+ let params: unknown = {}
+ let chart: unknown = {}
+
+ const promises = [
+ // Params
+ (async () => {
+ params = await this.generateParams(
+ null,
+ subQuery,
+ false,
+ paramsData,
+ 'errors',
+ )
+
+ if (!_some(_values(params), val => !_isEmpty(val))) {
+ throw new BadRequestException(
+ 'There are no error details for specified time frame',
+ )
+ }
+ })(),
+
+ // Chart data
+ (async () => {
+ const { xShifted } = this.generateXAxis(
+ timeBucket,
+ from,
+ to,
+ safeTimezone,
+ )
+
+ const query = this.generateErrorsAggregationQuery(
+ timeBucket,
+ filtersQuery,
+ safeTimezone,
+ mode,
+ )
+
+ const result = <Array<TrafficCEFilterCHResponse>>(
+ await clickhouse.query(query, paramsData).toPromise()
+ )
+ const { count } = this.extractErrorsChartData(result, xShifted)
+
+ chart = {
+ x: xShifted,
+ occurrences: count,
+ }
+ })(),
+ ]
+
+ await Promise.all(promises)
+
+ return Promise.resolve({
+ params,
+ chart,
+ })
+ }
+
extractPerformanceChartData(result, x: string[]): any {
const dns = Array(x.length).fill(0)
const tls = Array(x.length).fill(0)
@@ -2145,13 +2298,13 @@ export class AnalyticsService {
subQuery,
false,
paramsData,
- true,
+ 'performance',
measure,
)
if (!_some(_values(params), val => !_isEmpty(val))) {
throw new BadRequestException(
- 'The are no parameters for the specified time frames',
+ 'There are no parameters for the specified time frames',
)
}
})(),
@@ -2526,6 +2679,236 @@ export class AnalyticsService {
return result
}
+ async getErrorsList(
+ options: string,
+ filtersQuery: string,
+ paramsData: object,
+ safeTimezone: string,
+ take = 30,
+ skip = 0,
+ ): Promise<object | void> {
+ const tzFromDate = `toTimeZone(parseDateTimeBestEffort({groupFrom:String}), '${safeTimezone}')`
+ const tzToDate = `toTimeZone(parseDateTimeBestEffort({groupTo:String}), '${safeTimezone}')`
+
+ let parsedOptions: {
+ showResolved?: boolean
+ } = {}
+
+ try {
+ parsedOptions = JSON.parse(options)
+ } catch (reason) {
+ console.error('[getErrorsList] Failed to parse options:', options)
+ }
+
+ const query = `
+ SELECT
+ eid,
+ any(name) as name,
+ any(message) as message,
+ any(filename) as filename,
+ count(*) as count,
+ max(created) as last_seen,
+ status.status
+ FROM (
+ SELECT eid, name, message, filename, created
+ FROM errors
+ WHERE pid = {pid:FixedString(12)}
+ AND created BETWEEN ${tzFromDate} AND ${tzToDate}
+ ${filtersQuery}
+ ) AS errors
+ LEFT JOIN (
+ SELECT
+ eid,
+ argMax(status, updated) AS status
+ FROM error_statuses
+ WHERE pid = {pid:FixedString(12)}
+ GROUP BY eid
+ ) AS status ON errors.eid = status.eid
+ ${
+ parsedOptions?.showResolved
+ ? ''
+ : "WHERE status.status = 'active' OR status.status = 'regressed'"
+ }
+ GROUP BY errors.eid, status.status
+ ORDER BY last_seen DESC
+ LIMIT ${take}
+ OFFSET ${skip};
+ `
+
+ const result = await clickhouse.query(query, paramsData).toPromise()
+
+ return result
+ }
+
+ async getErrorDetails(
+ pid: string,
+ eid: string,
+ safeTimezone: string,
+ groupFrom: string,
+ groupTo: string,
+ timeBucket: any,
+ ): Promise<any> {
+ const tzFromDate = `toTimeZone(parseDateTimeBestEffort({groupFrom:String}), '${safeTimezone}')`
+ const tzToDate = `toTimeZone(parseDateTimeBestEffort({groupTo:String}), '${safeTimezone}')`
+
+ const queryErrorDetails = `
+ SELECT
+ subquery.eid,
+ any(subquery.name) AS name,
+ any(subquery.message) AS message,
+ any(subquery.filename) AS filename,
+ any(subquery.colno) AS colno,
+ any(subquery.lineno) AS lineno,
+ count(*) AS count,
+ status.status
+ FROM (
+ SELECT
+ eid,
+ name,
+ message,
+ filename,
+ colno,
+ lineno
+ FROM errors
+ WHERE pid = {pid:FixedString(12)}
+ AND eid = {eid:FixedString(32)}
+ AND created BETWEEN ${tzFromDate} AND ${tzToDate}
+ ) AS subquery
+ LEFT JOIN (
+ SELECT
+ eid,
+ argMax(status, updated) AS status
+ FROM error_statuses
+ WHERE pid = {pid:FixedString(12)}
+ AND eid = {eid:FixedString(32)}
+ GROUP BY eid
+ ) AS status ON subquery.eid = status.eid
+ GROUP BY subquery.eid, status.status;
+ `
+
+ const queryFirstLastSeen = `
+ SELECT
+ max(created) AS last_seen,
+ min(created) AS first_seen
+ FROM errors
+ WHERE
+ pid = {pid:FixedString(12)}
+ AND eid = {eid:FixedString(32)};
+ `
+
+ const paramsData = {
+ params: {
+ pid,
+ eid,
+ groupFrom,
+ groupTo,
+ },
+ }
+
+ const details = (
+ await clickhouse.query(queryErrorDetails, paramsData).toPromise()
+ )[0]
+
+ const occurenceDetails = (
+ await clickhouse.query(queryFirstLastSeen, paramsData).toPromise()
+ )[0]
+
+ const groupedChart = await this.groupErrorsByTimeBucket(
+ timeBucket,
+ groupFrom,
+ groupTo,
+ `FROM errors WHERE eid = {eid:FixedString(32)} AND created BETWEEN {groupFrom:String} AND {groupTo:String}`,
+ 'AND eid = {eid:FixedString(32)}',
+ paramsData,
+ safeTimezone,
+ ChartRenderMode.PERIODICAL,
+ )
+
+ return {
+ details: {
+ ...details,
+ ...occurenceDetails,
+ },
+ ...groupedChart,
+ timeBucket,
+ }
+ }
+
+ getErrorID(errorDTO: ErrorDTO): string {
+ const { name, message, colno, lineno, filename } = errorDTO
+
+ return hash(`${name}${message}${colno}${lineno}${filename}`, {
+ length: 16,
+ }).toString('hex')
+ }
+
+ async validateEIDs(eids: string[], pid: string) {
+ const params = _reduce(
+ eids,
+ (acc, curr, index) => ({
+ ...acc,
+ [`e_${index}`]: curr,
+ }),
+ {},
+ )
+
+ const query = `SELECT count(DISTINCT eid) as count FROM errors WHERE pid = {pid:FixedString(12)} AND eid IN (${_map(
+ params,
+ (val, key) => `{${key}:FixedString(32)}`,
+ ).join(', ')})`
+
+ let result
+
+ try {
+ ;[result] = await clickhouse
+ .query(query, { params: { ...params, pid } })
+ .toPromise()
+ } catch (reason) {
+ console.error('validateEIDs - clickhouse request error')
+ console.error(reason)
+ throw new InternalServerErrorException(
+ 'Error occured while validating error IDs',
+ )
+ }
+
+ if (result.count !== _size(eids)) {
+ throw new UnprocessableEntityException(
+ 'Some of the error IDs you provided are not valid (do not exist in our database)',
+ )
+ }
+ }
+
+ async updateEIDStatus(
+ eids: string[],
+ status: 'resolved' | 'active',
+ pid: string,
+ ) {
+ const params = _reduce(
+ eids,
+ (acc, curr, index) => ({
+ ...acc,
+ [`e_${index}`]: curr,
+ }),
+ {},
+ )
+
+ const query = `INSERT INTO error_statuses (eid, pid, status) VALUES ${_map(
+ params,
+ (val, key) =>
+ `({${key}:FixedString(32)}, {pid:FixedString(12)}, '${status}')`,
+ ).join(', ')}`
+
+ try {
+ await clickhouse.query(query, { params: { ...params, pid } }).toPromise()
+ } catch (reason) {
+ console.error('Error at PATCH error-status:')
+ console.error(reason)
+ throw new InternalServerErrorException(
+ 'Error occured while updating error status',
+ )
+ }
+ }
+
async getOnlineCountByProjectId(projectId: string) {
// @ts-ignore
return redis.countKeysByPattern(`hb:${projectId}:*`)
diff --git a/apps/selfhosted/src/analytics/dto/error.dto.ts b/apps/selfhosted/src/analytics/dto/error.dto.ts
new file mode 100644
index 000000000..efa35255e
--- /dev/null
+++ b/apps/selfhosted/src/analytics/dto/error.dto.ts
@@ -0,0 +1,91 @@
+import { ApiProperty } from '@nestjs/swagger'
+import {
+ IsNotEmpty,
+ IsNumber,
+ IsOptional,
+ IsString,
+ MaxLength,
+ Matches,
+} from 'class-validator'
+import { PID_REGEX } from '../../common/constants'
+
+export class ErrorDTO {
+ @ApiProperty({
+ example: 'aUn1quEid-3',
+ required: true,
+ description: 'The project ID',
+ })
+ @IsNotEmpty()
+ @Matches(PID_REGEX, {
+ message: 'Incorrect project ID format.',
+ })
+ pid: string
+
+ // Tracking metrics
+ @ApiProperty({
+ example: 'Europe/Kiev',
+ description: "User's timezone",
+ })
+ @IsOptional()
+ @IsString()
+ tz?: string
+
+ @ApiProperty({
+ example: '/articles/my-awesome-article-1',
+ description: 'A page that user sent data from',
+ })
+ @IsOptional()
+ @IsString()
+ pg?: string
+
+ @ApiProperty({
+ example: 'en-GB',
+ description: "User's locale",
+ })
+ @IsOptional()
+ @IsString()
+ lc?: string
+
+ // Error metrics
+ @ApiProperty({
+ example: 'ParseError',
+ required: true,
+ maxLength: 200,
+ })
+ @IsNotEmpty()
+ @IsString()
+ @MaxLength(200)
+ name: string
+
+ @ApiProperty({
+ example: 'Malformed input',
+ maxLength: 2000,
+ })
+ @IsOptional()
+ @IsString()
+ @MaxLength(2000)
+ message?: string
+
+ @ApiProperty({
+ example: 12,
+ })
+ @IsOptional()
+ @IsNumber()
+ lineno?: number
+
+ @ApiProperty({
+ example: 510,
+ })
+ @IsOptional()
+ @IsNumber()
+ colno?: number
+
+ @ApiProperty({
+ example: 'https://example.com/assets/convert.js',
+ maxLength: 1000,
+ })
+ @IsOptional()
+ @IsString()
+ @MaxLength(1000)
+ filename?: string
+}
diff --git a/apps/selfhosted/src/analytics/dto/get-error.dto.ts b/apps/selfhosted/src/analytics/dto/get-error.dto.ts
new file mode 100644
index 000000000..ee8e21b6e
--- /dev/null
+++ b/apps/selfhosted/src/analytics/dto/get-error.dto.ts
@@ -0,0 +1,15 @@
+import { ApiProperty, PickType } from '@nestjs/swagger'
+import { IsNotEmpty } from 'class-validator'
+import { AnalyticsGET_DTO } from './getData.dto'
+
+export class GetErrorDTO extends PickType(AnalyticsGET_DTO, [
+ 'pid',
+ 'period',
+ 'from',
+ 'to',
+ 'timezone',
+] as const) {
+ @ApiProperty()
+ @IsNotEmpty()
+ eid: string
+}
diff --git a/apps/selfhosted/src/analytics/dto/get-errors.dto.ts b/apps/selfhosted/src/analytics/dto/get-errors.dto.ts
new file mode 100644
index 000000000..6d85cac2d
--- /dev/null
+++ b/apps/selfhosted/src/analytics/dto/get-errors.dto.ts
@@ -0,0 +1,23 @@
+import { PickType, ApiProperty } from '@nestjs/swagger'
+import { AnalyticsGET_DTO } from './getData.dto'
+
+export class GetErrorsDto extends PickType(AnalyticsGET_DTO, [
+ 'pid',
+ 'period',
+ 'from',
+ 'to',
+ 'filters',
+ 'timezone',
+] as const) {
+ take: number
+
+ skip: number
+
+ @ApiProperty({
+ example: {
+ showResolved: true,
+ },
+ description: 'Errors list options',
+ })
+ options?: string
+}
diff --git a/apps/selfhosted/src/analytics/dto/patch-status.dto.ts b/apps/selfhosted/src/analytics/dto/patch-status.dto.ts
new file mode 100644
index 000000000..2da498d93
--- /dev/null
+++ b/apps/selfhosted/src/analytics/dto/patch-status.dto.ts
@@ -0,0 +1,52 @@
+import { ApiProperty } from '@nestjs/swagger'
+import {
+ IsNotEmpty,
+ IsIn,
+ ValidateIf,
+ IsString,
+ IsArray,
+ Matches,
+} from 'class-validator'
+import { PID_REGEX } from '../../common/constants'
+
+export class PatchStatusDTO {
+ @ApiProperty({
+ example: 'aUn1quEid-3',
+ required: true,
+ description: 'The project ID',
+ })
+ @IsNotEmpty()
+ @Matches(PID_REGEX, {
+ message: 'Incorrect project ID format.',
+ })
+ pid: string
+
+ @ApiProperty({
+ example: 'c138c2b1826d9fec65d230e471ab2c25',
+ description: 'Error id',
+ })
+ @ValidateIf(o => !o.eids || o.eid)
+ @IsNotEmpty()
+ @IsString()
+ eid: string
+
+ @ApiProperty({
+ example: [
+ 'c138c2b1826d9fec65d230e471ab2c25',
+ 'b2d12867d59d1d4f9986b52311a97cd5',
+ ],
+ description: 'Error id list',
+ })
+ @ValidateIf(o => o.eids || !o.eid)
+ @IsNotEmpty()
+ @IsArray()
+ @IsString({ each: true })
+ eids: string
+
+ @ApiProperty({
+ enum: ['resolved', 'active'],
+ required: true,
+ })
+ @IsIn(['resolved', 'active'])
+ status: 'resolved' | 'active'
+}
diff --git a/apps/selfhosted/src/common/constants.ts b/apps/selfhosted/src/common/constants.ts
index aa8470201..89220bfc3 100644
--- a/apps/selfhosted/src/common/constants.ts
+++ b/apps/selfhosted/src/common/constants.ts
@@ -78,6 +78,7 @@ const REDIS_LOG_CUSTOM_CACHE_KEY = 'log_custom_cache_v3'
const REDIS_SESSION_SALT_KEY = 'log_salt' // is updated every 24 hours
const REDIS_USERS_COUNT_KEY = 'stats:users_count'
const REDIS_PROJECTS_COUNT_KEY = 'stats:projects_count'
+const REDIS_LOG_ERROR_CACHE_KEY = 'log_error_cache'
// 3600 sec -> 1 hour
const redisProjectCacheTimeout = 3600
@@ -108,6 +109,7 @@ const TRAFFIC_COLUMNS = [
]
const PERFORMANCE_COLUMNS = ['cc', 'rg', 'ct', 'pg', 'dv', 'br']
+const ERROR_COLUMNS = ['pg', 'dv', 'br', 'os', 'lc', 'cc', 'rg', 'ct']
const NUMBER_JWT_REFRESH_TOKEN_LIFETIME = Number(JWT_REFRESH_TOKEN_LIFETIME)
const NUMBER_JWT_ACCESS_TOKEN_LIFETIME = Number(JWT_ACCESS_TOKEN_LIFETIME)
@@ -143,4 +145,7 @@ export {
isProxiedByCloudflare,
MIN_PAGES_IN_FUNNEL,
MAX_PAGES_IN_FUNNEL,
+ ERROR_COLUMNS,
+ REDIS_LOG_ERROR_CACHE_KEY,
+ PID_REGEX,
}
diff --git a/apps/selfhosted/src/project/project.controller.ts b/apps/selfhosted/src/project/project.controller.ts
index 4c0ff4cb7..75f86f2ff 100644
--- a/apps/selfhosted/src/project/project.controller.ts
+++ b/apps/selfhosted/src/project/project.controller.ts
@@ -76,11 +76,17 @@ export class ProjectController {
_map(formatted, ({ id }) => id),
)
+ const pidsWithErrorData =
+ await this.projectService.getPIDsWhereErrorsDataExists(
+ _map(formatted, ({ id }) => id),
+ )
+
const results = _map(formatted, p => ({
...p,
isOwner: true,
isLocked: false,
isDataExists: _includes(pidsWithData, p?.id),
+ isErrorDataExists: _includes(pidsWithErrorData, p?.id),
}))
return {
@@ -306,9 +312,14 @@ export class ProjectController {
await this.projectService.getPIDsWhereAnalyticsDataExists([id]),
)
+ const isErrorDataExists = !_isEmpty(
+ await this.projectService.getPIDsWhereErrorsDataExists([id]),
+ )
+
return this.projectService.formatFromClickhouse({
...project,
isDataExists,
+ isErrorDataExists,
})
}
}
diff --git a/apps/selfhosted/src/project/project.service.ts b/apps/selfhosted/src/project/project.service.ts
index 3f078f9c6..e845cffbe 100644
--- a/apps/selfhosted/src/project/project.service.ts
+++ b/apps/selfhosted/src/project/project.service.ts
@@ -156,6 +156,50 @@ export class ProjectService {
return _map(result, ({ pid }) => pid)
}
+ async getPIDsWhereErrorsDataExists(projectIds: string[]) {
+ if (_isEmpty(projectIds)) {
+ return {}
+ }
+
+ const params = _reduce(
+ projectIds,
+ (acc, curr, index) => ({
+ ...acc,
+ [`pid_${index}`]: curr,
+ }),
+ {},
+ )
+
+ const pids = _join(
+ _map(params, (val, key) => `{${key}:FixedString(12)}`),
+ ',',
+ )
+
+ const query = `
+ SELECT
+ pid,
+ CASE
+ WHEN EXISTS (
+ SELECT 1
+ FROM errors
+ WHERE pid IN (${pids})
+ )
+ THEN 1
+ ELSE 0
+ END AS exists
+ FROM
+ (
+ SELECT DISTINCT pid
+ FROM errors
+ WHERE pid IN (${pids})
+ );
+ `
+
+ const result: any = await clickhouse.query(query, { params }).toPromise()
+
+ return _map(result, ({ pid }) => pid)
+ }
+
async removeDataFromClickhouse(
pid: string,
from: string,
diff --git a/apps/selfhosted/src/task-manager/task-manager.service.ts b/apps/selfhosted/src/task-manager/task-manager.service.ts
index 18f624f8f..2a19640bc 100644
--- a/apps/selfhosted/src/task-manager/task-manager.service.ts
+++ b/apps/selfhosted/src/task-manager/task-manager.service.ts
@@ -17,6 +17,7 @@ import {
REDIS_SESSION_SALT_KEY,
REDIS_LOG_PERF_CACHE_KEY,
REDIS_LOG_CAPTCHA_CACHE_KEY,
+ REDIS_LOG_ERROR_CACHE_KEY,
} from '../common/constants'
import { AppLoggerService } from '../logger/logger.service'
@@ -32,6 +33,7 @@ export class TaskManagerService {
const customData = await redis.lrange(REDIS_LOG_CUSTOM_CACHE_KEY, 0, -1)
const perfData = await redis.lrange(REDIS_LOG_PERF_CACHE_KEY, 0, -1)
const captchaData = await redis.lrange(REDIS_LOG_CAPTCHA_CACHE_KEY, 0, -1)
+ const errorData = await redis.lrange(REDIS_LOG_ERROR_CACHE_KEY, 0, -1)
if (!_isEmpty(data)) {
await redis.del(REDIS_LOG_DATA_CACHE_KEY)
@@ -68,6 +70,19 @@ export class TaskManagerService {
}
}
+ if (!_isEmpty(errorData)) {
+ await redis.del(REDIS_LOG_ERROR_CACHE_KEY)
+ const query = `INSERT INTO errors (*) VALUES ${_join(errorData, ',')}`
+
+ try {
+ await clickhouse.query(query).toPromise()
+ } catch (e) {
+ console.error(
+ `[CRON WORKER] Error whilst saving error events data: ${e}`,
+ )
+ }
+ }
+
if (!_isEmpty(perfData)) {
await redis.del(REDIS_LOG_PERF_CACHE_KEY)
diff --git a/migrations/clickhouse/2024_03_30.js b/migrations/clickhouse/2024_03_30.js
new file mode 100644
index 000000000..01ea5daeb
--- /dev/null
+++ b/migrations/clickhouse/2024_03_30.js
@@ -0,0 +1,39 @@
+const { queriesRunner, dbName } = require('./setup')
+
+const queries = [
+ // Error events table
+ `CREATE TABLE IF NOT EXISTS ${dbName}.errors
+ (
+ eid FixedString(32),
+ pid FixedString(12),
+ pg Nullable(String),
+ dv LowCardinality(Nullable(String)),
+ br LowCardinality(Nullable(String)),
+ os LowCardinality(Nullable(String)),
+ lc LowCardinality(Nullable(String)),
+ cc LowCardinality(Nullable(FixedString(2))),
+ rg LowCardinality(Nullable(String)),
+ ct Nullable(String),
+ name String,
+ message Nullable(String),
+ lineno Nullable(UInt32),
+ colno Nullable(UInt32),
+ filename Nullable(String),
+ created DateTime('UTC')
+ )
+ ENGINE = MergeTree()
+ PARTITION BY toYYYYMM(created)
+ ORDER BY (pid, created);`,
+
+ // Error events status table
+ `CREATE TABLE IF NOT EXISTS ${dbName}.error_statuses (
+ eid FixedString(32),
+ pid FixedString(12),
+ status Enum8('active', 'regressed', 'resolved'),
+ updated DateTime('UTC') DEFAULT now()
+ )
+ ENGINE = ReplacingMergeTree()
+ PRIMARY KEY (eid, pid);`,
+]
+
+queriesRunner(queries)
diff --git a/migrations/clickhouse/initialise_database.js b/migrations/clickhouse/initialise_database.js
index 4172594a1..c41928733 100644
--- a/migrations/clickhouse/initialise_database.js
+++ b/migrations/clickhouse/initialise_database.js
@@ -1,9 +1,7 @@
// This script initialises the Swetrix cloud edition database and tables if they're absent
const { queriesRunner, dbName, databaselessQueriesRunner } = require('./setup')
-const CLICKHOUSE_DB_INIT_QUERIES = [
- `CREATE DATABASE IF NOT EXISTS ${dbName}`,
-]
+const CLICKHOUSE_DB_INIT_QUERIES = [`CREATE DATABASE IF NOT EXISTS ${dbName}`]
const CLICKHOUSE_INIT_QUERIES = [
// The traffic data table
@@ -86,6 +84,40 @@ const CLICKHOUSE_INIT_QUERIES = [
PARTITION BY toYYYYMM(created)
ORDER BY (pid, created);`,
+ // Error events table
+ `CREATE TABLE IF NOT EXISTS ${dbName}.errors
+ (
+ eid FixedString(32),
+ pid FixedString(12),
+ pg Nullable(String),
+ dv LowCardinality(Nullable(String)),
+ br LowCardinality(Nullable(String)),
+ os LowCardinality(Nullable(String)),
+ lc LowCardinality(Nullable(String)),
+ cc LowCardinality(Nullable(FixedString(2))),
+ rg LowCardinality(Nullable(String)),
+ ct Nullable(String),
+ name String,
+ message Nullable(String),
+ lineno Nullable(UInt32),
+ colno Nullable(UInt32),
+ filename Nullable(String),
+ created DateTime('UTC')
+ )
+ ENGINE = MergeTree()
+ PARTITION BY toYYYYMM(created)
+ ORDER BY (pid, created);`,
+
+ // Error events status table
+ `CREATE TABLE IF NOT EXISTS ${dbName}.error_statuses (
+ eid FixedString(32),
+ pid FixedString(12),
+ status Enum8('active', 'regressed', 'resolved'),
+ updated DateTime('UTC') DEFAULT now()
+ )
+ ENGINE = ReplacingMergeTree()
+ PRIMARY KEY (eid, pid);`,
+
// The CAPTCHA data table
`CREATE TABLE IF NOT EXISTS ${dbName}.captcha
(
@@ -99,7 +131,7 @@ const CLICKHOUSE_INIT_QUERIES = [
)
ENGINE = MergeTree()
PARTITION BY toYYYYMM(created)
- ORDER BY (pid, created);`
+ ORDER BY (pid, created);`,
]
const initialiseDatabase = async () => {
@@ -107,7 +139,9 @@ const initialiseDatabase = async () => {
await databaselessQueriesRunner(CLICKHOUSE_DB_INIT_QUERIES)
await queriesRunner(CLICKHOUSE_INIT_QUERIES)
} catch (reason) {
- console.error(`[ERROR] Error occured whilst initialising the database: ${reason}`)
+ console.error(
+ `[ERROR] Error occured whilst initialising the database: ${reason}`,
+ )
}
}