📈 Open source, privacy-first web analytics — a cookie-less Google Analytics alternative with built-in error tracking and performance monitoring. Self-host it or use our cloud. Get started for free!
TypeScript chartsclickhousedockergdprgoogle-analyticsmetricsmonitoringmysqlnestjsprivacyredisself-hostedselfhostedstatisticstailwindcssweb-analyticswebanalyticswebsite
(feature) Error tracking (#224)
Andrii Romasiund7b6c73e0f75
diff --git a/apps/production/src/analytics/analytics.controller.ts b/apps/production/src/analytics/analytics.controller.tsindex eb7f9fc4f..f26e8e506 100644--- a/apps/production/src/analytics/analytics.controller.ts+++ b/apps/production/src/analytics/analytics.controller.ts@@ -19,6 +19,7 @@ import {UseGuards,Get,Post,+ Patch,Headers,BadRequestException,InternalServerErrorException,@@ -63,8 +64,13 @@ import {REDIS_EVENTS_COUNT_KEY,REDIS_SESSION_SALT_KEY,clickhouse,+ REDIS_LOG_ERROR_CACHE_KEY,} from '../common/constants'-import { getGeoDetails, getIPFromHeaders } from '../common/utils'+import {+ checkRateLimit,+ getGeoDetails,+ getIPFromHeaders,+} from '../common/utils'import { BotDetection } from '../common/decorators/bot-detection.decorator'import { BotDetectionGuard } from '../common/guards/bot-detection.guard'import { GetCustomEventsDto } from './dto/get-custom-events.dto'@@ -78,6 +84,10 @@ import {} from './interfaces'import { GetSessionsDto } from './dto/get-sessions.dto'import { GetSessionDto } from './dto/get-session.dto'+import { ErrorDTO } from './dto/error.dto'+import { GetErrorsDto } from './dto/get-errors.dto'+import { GetErrorDTO } from './dto/get-error.dto'+import { PatchStatusDTO } from './dto/patch-status.dto'// eslint-disable-next-line @typescript-eslint/no-var-requiresconst mysql = require('mysql2')@@ -216,6 +226,43 @@ const customLogDTO = (]}+const errorLogDTO = (+ eid: string,+ pid: string,+ pg: string,+ dv: string,+ br: string,+ os: string,+ lc: string,+ cc: string,+ rg: string,+ ct: string,+ name: string,+ message: string,+ lineno: number,+ colno: number,+ filename: string,+): Array<string | number | string[]> => {+ return [+ eid,+ pid,+ pg,+ dv,+ br,+ os,+ lc,+ cc,+ rg,+ ct,+ name,+ message,+ lineno,+ colno,+ filename,+ dayjs.utc().format('YYYY-MM-DD HH:mm:ss'),+ ]+}+export const getElValue = el => {if (_isArray(el)) {return `[${_map(el, getElValue).join(',')}]`@@ -280,6 +327,27 @@ const getPIDsArray = (pids, pid) => {return pids}+const getEIDsArray = (eids, eid) => {+ const eidsEmpty = _isEmpty(eids)+ const eidEmpty = _isEmpty(eid)+ if (eidsEmpty && eidEmpty) {+ throw new BadRequestException(+ "An array of Error ID's (eids) or a Error ID (eid) has to be provided",+ )+ }+ if (!eidsEmpty && !eidEmpty) {+ throw new BadRequestException(+ "Please provide either an array of Error ID's (eids) or a Error ID (eid), but not both",+ )+ }++ if (!eidsEmpty) {+ return eids+ }++ return [eid]+}+// needed for serving 1x1 px GIFconst TRANSPARENT_GIF_BUFFER = Buffer.from('R0lGODlhAQABAIAAAP///wAAACwAAAAAAQABAAACAkQBADs=',@@ -555,6 +623,27 @@ export class AnalyticsController {return this.analyticsService.getFilters(pid, type)}+ @Get('errors-filters')+ @Auth([], true, true)+ async getErrorsFilters(+ @Query() data: GetFiltersDto,+ @CurrentUserId() uid: string,+ @Headers() headers: { 'x-password'?: string },+ ): Promise<any> {+ const { pid, type } = data+ this.analyticsService.validatePID(pid)++ await this.analyticsService.checkProjectAccess(+ pid,+ uid,+ headers['x-password'],+ )++ await this.analyticsService.checkBillingAccess(pid)++ return this.analyticsService.getErrorsFilters(pid, type)+ }+@Get('chart')@Auth([], true, true)async getChartData(@@ -1088,6 +1177,89 @@ export class AnalyticsController {return processed}+ // Log error event+ @Post('error')+ @UseGuards(BotDetectionGuard)+ @BotDetection()+ @Public()+ async logError(+ @Body() errorDTO: ErrorDTO,+ @Headers() headers,+ @Ip() reqIP,+ ): Promise<any> {+ const { 'user-agent': userAgent, origin } = headers++ const ip = getIPFromHeaders(headers, true) || reqIP || ''++ await this.analyticsService.validate(errorDTO, origin, 'error', ip)++ const {+ city = 'NULL',+ region = 'NULL',+ country = 'NULL',+ } = getGeoDetails(ip, errorDTO.tz)++ const ua = UAParser(userAgent)+ const dv = ua.device.type || 'desktop'+ const br = ua.browser.name+ const os = ua.os.name++ const { name, message, lineno, colno, filename } = errorDTO++ const dto = errorLogDTO(+ this.analyticsService.getErrorID(errorDTO),+ errorDTO.pid,+ errorDTO.pg,+ dv,+ br,+ os,+ errorDTO.lc,+ country,+ region,+ city,+ name,+ message,+ lineno,+ colno,+ filename,+ )++ try {+ const values = `(${dto.map(getElValue).join(',')})`+ await redis.rpush(REDIS_LOG_ERROR_CACHE_KEY, values)+ } catch (e) {+ this.logger.error(e)+ throw new InternalServerErrorException(+ 'Error occured while saving the custom event',+ )+ }+ }++ // Update error(s) status+ @Patch('error-status')+ @Auth([], true, true)+ async patchStatus(+ @Body() statusDTO: PatchStatusDTO,+ @CurrentUserId() uid: string,+ @Headers() headers,+ @Ip() reqIP,+ ): Promise<any> {+ const { pid, eid, eids: unprocessedEids, status } = statusDTO+ const ip = getIPFromHeaders(headers) || reqIP || ''++ await checkRateLimit(ip, 'error-status', 100, 1800)++ this.analyticsService.validatePID(pid)+ await this.analyticsService.checkManageAccess(pid, uid)+ await this.analyticsService.checkBillingAccess(pid)++ const eids = getEIDsArray(unprocessedEids, eid)++ await this.analyticsService.validateEIDs(eids, pid)++ return this.analyticsService.updateEIDStatus(eids, status, pid)+ }+// Log custom event@Post('custom')@UseGuards(BotDetectionGuard)@@ -1469,6 +1641,173 @@ export class AnalyticsController {}}+ @Get('errors')+ @Auth([], true, true)+ async getErrors(+ @Query() data: GetErrorsDto,+ @CurrentUserId() uid: string,+ @Headers() headers: { 'x-password'?: string },+ ): Promise<any> {+ const {+ pid,+ period,+ from,+ to,+ filters,+ timezone = DEFAULT_TIMEZONE,+ options,+ } = data+ this.analyticsService.validatePID(pid)++ if (!_isEmpty(period)) {+ this.analyticsService.validatePeriod(period)+ }++ await this.analyticsService.checkProjectAccess(+ pid,+ uid,+ headers['x-password'],+ )++ await this.analyticsService.checkBillingAccess(pid)++ const take = this.analyticsService.getSafeNumber(data.take, 30)+ const skip = this.analyticsService.getSafeNumber(data.skip, 0)++ if (take > 150) {+ throw new BadRequestException(+ 'The maximum number of errors to return is 150',+ )+ }++ let timeBucket+ let diff++ if (period === 'all') {+ const res = await this.analyticsService.getTimeBucketForAllTime(+ pid,+ period,+ timezone,+ )++ // eslint-disable-next-line prefer-destructuring+ timeBucket = res.timeBucket[0]+ diff = res.diff+ } else {+ timeBucket = getLowestPossibleTimeBucket(period, from, to)+ }++ this.analyticsService.validateTimebucket(timeBucket)+ const [filtersQuery, filtersParams, appliedFilters] =+ this.analyticsService.getFiltersQuery(filters, DataType.ANALYTICS)++ const safeTimezone = this.analyticsService.getSafeTimezone(timezone)+ const { groupFrom, groupTo } = this.analyticsService.getGroupFromTo(+ from,+ to,+ timeBucket,+ period,+ safeTimezone,+ diff,+ )++ const paramsData = {+ params: {+ pid,+ groupFrom,+ groupTo,+ ...filtersParams,+ },+ }++ const errors = await this.analyticsService.getErrorsList(+ options,+ filtersQuery,+ paramsData,+ safeTimezone,+ take,+ skip,+ )++ return {+ errors,+ appliedFilters,+ take,+ skip,+ }+ }++ @Get('get-error')+ @Auth([], true, true)+ async getError(+ @Query() data: GetErrorDTO,+ @CurrentUserId() uid: string,+ @Headers() headers: { 'x-password'?: string },+ ): Promise<any> {+ const {+ pid,+ timezone = DEFAULT_TIMEZONE,+ eid,+ period,+ from,+ to,+ //+ } = data+ this.analyticsService.validatePID(pid)++ if (!_isEmpty(period)) {+ this.analyticsService.validatePeriod(period)+ }++ await this.analyticsService.checkProjectAccess(+ pid,+ uid,+ headers['x-password'],+ )++ await this.analyticsService.checkBillingAccess(pid)++ let timeBucket+ let diff++ if (period === 'all') {+ const res = await this.analyticsService.getTimeBucketForAllTime(+ pid,+ period,+ timezone,+ )++ // eslint-disable-next-line prefer-destructuring+ timeBucket = res.timeBucket[0]+ diff = res.diff+ } else {+ timeBucket = getLowestPossibleTimeBucket(period, from, to)+ }++ this.analyticsService.validateTimebucket(timeBucket)++ const safeTimezone = this.analyticsService.getSafeTimezone(timezone)+ const { groupFrom, groupTo } = this.analyticsService.getGroupFromTo(+ from,+ to,+ timeBucket,+ period,+ safeTimezone,+ diff,+ )++ const result = await this.analyticsService.getErrorDetails(+ pid,+ eid,+ safeTimezone,+ groupFrom,+ groupTo,+ timeBucket,+ )++ return result+ }+@Get('session')@Auth([], true, true)async getSession(diff --git a/apps/production/src/analytics/analytics.service.ts b/apps/production/src/analytics/analytics.service.tsindex e0f301325..e15b152c8 100644--- a/apps/production/src/analytics/analytics.service.ts+++ b/apps/production/src/analytics/analytics.service.ts@@ -48,6 +48,7 @@ import {TRAFFIC_COLUMNS,ALL_COLUMNS,CAPTCHA_COLUMNS,+ ERROR_COLUMNS,PERFORMANCE_COLUMNS,MIN_PAGES_IN_FUNNEL,MAX_PAGES_IN_FUNNEL,@@ -91,6 +92,7 @@ import {IPageflow,PerfMeasure,} from './interfaces'+import { ErrorDTO } from './dto/error.dto'dayjs.extend(utc)dayjs.extend(dayjsTimezone)@@ -268,8 +270,7 @@ const generateParamsQuery = (subQuery: string,customEVFilterApplied: boolean,isPageInclusiveFilterSet: boolean,- isCaptcha?: boolean,- isPerformance?: boolean,+ type: 'traffic' | 'performance' | 'captcha' | 'errors',measure?: PerfMeasure,): string => {let columns = [`${col} as name`]@@ -282,7 +283,7 @@ const generateParamsQuery = (const columnsQuery = columns.join(', ')- if (isPerformance) {+ if (type === 'performance') {const processedMeasure = measure === 'quantiles' ? 'median' : measureconst fn = MEASURES_MAP[processedMeasure]@@ -294,7 +295,11 @@ const generateParamsQuery = (return `SELECT ${columnsQuery}, round(divide(${fn}(pageLoad), 1000), 2) as count ${subQuery} AND ${col} IS NOT NULL GROUP BY ${columnsQuery}`}- if (isCaptcha) {+ if (type === 'errors') {+ return `SELECT ${columnsQuery}, count(*) as count ${subQuery} AND ${col} IS NOT NULL GROUP BY ${columnsQuery}`+ }++ if (type === 'captcha') {return `SELECT ${columnsQuery}, count(*) as count ${subQuery} AND ${col} IS NOT NULL GROUP BY ${col}`}@@ -359,6 +364,11 @@ export class AnalyticsService {this.projectService.allowedToView(project, uid, password)}+ async checkManageAccess(pid: string, uid: string | null): Promise<void> {+ const project = await this.projectService.getRedisProject(pid)+ this.projectService.allowedToManage(project, uid)+ }+async checkBillingAccess(pid: string) {const project = await this.projectService.getRedisProject(pid)@@ -384,6 +394,7 @@ export class AnalyticsService {}} else {const { hostname } = new URL(origin)+if (!isValidOrigin(origins, hostname)) {throw new BadRequestException("This origin is prohibited by the project's origins policy",@@ -418,9 +429,9 @@ export class AnalyticsService {}async validate(- logDTO: PageviewsDTO | EventsDTO,+ logDTO: PageviewsDTO | EventsDTO | ErrorDTO,origin: string,- type: 'custom' | 'log' = 'log',+ type: 'custom' | 'log' | 'error' = 'log',ip?: string,): Promise<string | null> {if (_isEmpty(logDTO)) {@@ -1747,35 +1758,53 @@ export class AnalyticsService {return _map(results, type)}+ async getErrorsFilters(pid: string, type: string): Promise<Array<string>> {+ if (!_includes(ERROR_COLUMNS, type)) {+ throw new UnprocessableEntityException(+ `The provided type (${type}) is incorrect`,+ )+ }++ const query = `SELECT ${type} FROM errors WHERE pid={pid:FixedString(12)} AND ${type} IS NOT NULL GROUP BY ${type}`++ const results = await clickhouse+ .query(query, { params: { pid } })+ .toPromise()++ return _map(results, type)+ }+async generateParams(parsedFilters: Array<{ [key: string]: string }>,subQuery: string,customEVFilterApplied: boolean,paramsData: any,- isCaptcha: boolean,- isPerformance: boolean,+ type: 'traffic' | 'performance' | 'captcha' | 'errors',measure?: PerfMeasure,): Promise<any> {const params = {}// We need this to display all the pageview related data (e.g. country, browser) when user applies an inclusive filter on the Page column- const isPageInclusiveFilterSet =- isCaptcha || isPerformance- ? false- : !_isEmpty(- _find(- parsedFilters,- filter => filter.column === 'pg' && !filter.isExclusive,- ),- )+ const isPageInclusiveFilterSet = ['captcha', 'performance'].includes(type)+ ? false+ : !_isEmpty(+ _find(+ parsedFilters,+ filter => filter.column === 'pg' && !filter.isExclusive,+ ),+ )let columns = TRAFFIC_COLUMNS- if (isCaptcha) {+ if (type === 'captcha') {columns = CAPTCHA_COLUMNS}- if (isPerformance) {+ if (type === 'errors') {+ columns = ERROR_COLUMNS+ }++ if (type === 'performance') {columns = PERFORMANCE_COLUMNS}@@ -1785,8 +1814,7 @@ export class AnalyticsService {subQuery,customEVFilterApplied,isPageInclusiveFilterSet,- isCaptcha,- isPerformance,+ type,measure,)const res = await clickhouse.query(query, paramsData).toPromise()@@ -2189,6 +2217,47 @@ export class AnalyticsService {return baseQuery}+ generateErrorsAggregationQuery(+ timeBucket: TimeBucketType,+ filtersQuery: string,+ safeTimezone: string,+ mode: ChartRenderMode,+ ): string {+ const timeBucketFunc = timeBucketConversion[timeBucket]+ const [selector, groupBy] = this.getGroupSubquery(timeBucket)+ const tzFromDate = `toTimeZone(parseDateTimeBestEffort({groupFrom:String}), '${safeTimezone}')`+ const tzToDate = `toTimeZone(parseDateTimeBestEffort({groupTo:String}), '${safeTimezone}')`++ const baseQuery = `+ SELECT+ ${selector},+ count() as count+ FROM (+ SELECT pg, dv, br, os, lc, cc, rg, ct,+ ${timeBucketFunc}(toTimeZone(created, '${safeTimezone}')) as tz_created+ FROM errors+ WHERE+ pid = {pid:FixedString(12)}+ AND created BETWEEN ${tzFromDate} AND ${tzToDate}+ ${filtersQuery}+ ) as subquery+ GROUP BY ${groupBy}+ ORDER BY ${groupBy}+ `++ if (mode === ChartRenderMode.CUMULATIVE) {+ return `+ SELECT+ *,+ sum(pageviews) OVER (ORDER BY ${groupBy}) as pageviews,+ sum(uniques) OVER (ORDER BY ${groupBy}) as uniques+ FROM (${baseQuery})+ `+ }++ return baseQuery+ }+async groupByTimeBucket(timeBucket: TimeBucketType,from: string,@@ -2216,7 +2285,7 @@ export class AnalyticsService {if (!_some(_values(params), val => !_isEmpty(val))) {throw new BadRequestException(- 'The are no parameters for the specified time frames',+ 'There are no parameters for the specified time frames',)}})(),@@ -2258,8 +2327,7 @@ export class AnalyticsService {subQuery,customEVFilterApplied,paramsData,- false,- false,+ 'traffic',)}@@ -2365,13 +2433,12 @@ export class AnalyticsService {subQuery,false,paramsData,- true,- false,+ 'captcha',)if (!_some(_values(params), val => !_isEmpty(val))) {throw new BadRequestException(- 'The are no parameters for the specified time frames',+ 'There are no parameters for the specified time frames',)}})(),@@ -2412,6 +2479,73 @@ export class AnalyticsService {})}+ async groupErrorsByTimeBucket(+ timeBucket: TimeBucketType,+ from: string,+ to: string,+ subQuery: string,+ filtersQuery: string,+ paramsData: object,+ safeTimezone: string,+ mode: ChartRenderMode,+ ): Promise<object | void> {+ let params: unknown = {}+ let chart: unknown = {}++ const promises = [+ // Params+ (async () => {+ params = await this.generateParams(+ null,+ subQuery,+ false,+ paramsData,+ 'errors',+ )++ if (!_some(_values(params), val => !_isEmpty(val))) {+ throw new BadRequestException(+ 'There are no error details for specified time frame',+ )+ }+ })(),++ // Chart data+ (async () => {+ const { xShifted } = this.generateXAxis(+ timeBucket,+ from,+ to,+ safeTimezone,+ )++ const query = this.generateErrorsAggregationQuery(+ timeBucket,+ filtersQuery,+ safeTimezone,+ mode,+ )++ const result = <Array<TrafficCEFilterCHResponse>>(+ await clickhouse.query(query, paramsData).toPromise()+ )+ const { count } = this.extractCaptchaChartData(result, xShifted)++ chart = {+ x: xShifted,+ occurrences: count,+ }+ })(),+ ]++ await Promise.all(promises)++ return Promise.resolve({+ params,+ chart,+ })+ }+extractPerformanceChartData(result, x: string[]): any {const dns = Array(x.length).fill(0)const tls = Array(x.length).fill(0)@@ -2545,14 +2679,13 @@ export class AnalyticsService {subQuery,false,paramsData,- false,- true,+ 'performance',measure,)if (!_some(_values(params), val => !_isEmpty(val))) {throw new BadRequestException(- 'The are no parameters for the specified time frames',+ 'There are no parameters for the specified time frames',)}})(),@@ -2927,6 +3060,236 @@ export class AnalyticsService {return result}+ async getErrorsList(+ options: string,+ filtersQuery: string,+ paramsData: object,+ safeTimezone: string,+ take = 30,+ skip = 0,+ ): Promise<object | void> {+ const tzFromDate = `toTimeZone(parseDateTimeBestEffort({groupFrom:String}), '${safeTimezone}')`+ const tzToDate = `toTimeZone(parseDateTimeBestEffort({groupTo:String}), '${safeTimezone}')`++ let parsedOptions: {+ showResolved?: boolean+ } = {}++ try {+ parsedOptions = JSON.parse(options)+ } catch (reason) {+ console.error('[getErrorsList] Failed to parse options:', options)+ }++ const query = `+ SELECT+ eid,+ any(name) as name,+ any(message) as message,+ any(filename) as filename,+ count(*) as count,+ max(created) as last_seen,+ status.status+ FROM (+ SELECT eid, name, message, filename, created+ FROM errors+ WHERE pid = {pid:FixedString(12)}+ AND created BETWEEN ${tzFromDate} AND ${tzToDate}+ ${filtersQuery}+ ) AS errors+ LEFT JOIN (+ SELECT+ eid,+ argMax(status, updated) AS status+ FROM error_statuses+ WHERE pid = {pid:FixedString(12)}+ GROUP BY eid+ ) AS status ON errors.eid = status.eid+ ${+ parsedOptions?.showResolved+ ? ''+ : "WHERE status.status = 'active' OR status.status = 'regressed'"+ }+ GROUP BY errors.eid, status.status+ ORDER BY last_seen DESC+ LIMIT ${take}+ OFFSET ${skip};+ `++ const result = await clickhouse.query(query, paramsData).toPromise()++ return result+ }++ async getErrorDetails(+ pid: string,+ eid: string,+ safeTimezone: string,+ groupFrom: string,+ groupTo: string,+ timeBucket: any,+ ): Promise<any> {+ const tzFromDate = `toTimeZone(parseDateTimeBestEffort({groupFrom:String}), '${safeTimezone}')`+ const tzToDate = `toTimeZone(parseDateTimeBestEffort({groupTo:String}), '${safeTimezone}')`++ const queryErrorDetails = `+ SELECT+ subquery.eid,+ any(subquery.name) AS name,+ any(subquery.message) AS message,+ any(subquery.filename) AS filename,+ any(subquery.colno) AS colno,+ any(subquery.lineno) AS lineno,+ count(*) AS count,+ status.status+ FROM (+ SELECT+ eid,+ name,+ message,+ filename,+ colno,+ lineno+ FROM errors+ WHERE pid = {pid:FixedString(12)}+ AND eid = {eid:FixedString(32)}+ AND created BETWEEN ${tzFromDate} AND ${tzToDate}+ ) AS subquery+ LEFT JOIN (+ SELECT+ eid,+ argMax(status, updated) AS status+ FROM error_statuses+ WHERE pid = {pid:FixedString(12)}+ AND eid = {eid:FixedString(32)}+ GROUP BY eid+ ) AS status ON subquery.eid = status.eid+ GROUP BY subquery.eid, status.status;+ `++ const queryFirstLastSeen = `+ SELECT+ max(created) AS last_seen,+ min(created) AS first_seen+ FROM errors+ WHERE+ pid = {pid:FixedString(12)}+ AND eid = {eid:FixedString(32)};+ `++ const paramsData = {+ params: {+ pid,+ eid,+ groupFrom,+ groupTo,+ },+ }++ const details = (+ await clickhouse.query(queryErrorDetails, paramsData).toPromise()+ )[0]++ const occurenceDetails = (+ await clickhouse.query(queryFirstLastSeen, paramsData).toPromise()+ )[0]++ const groupedChart = await this.groupErrorsByTimeBucket(+ timeBucket,+ groupFrom,+ groupTo,+ `FROM errors WHERE eid = {eid:FixedString(32)} AND created BETWEEN {groupFrom:String} AND {groupTo:String}`,+ 'AND eid = {eid:FixedString(32)}',+ paramsData,+ safeTimezone,+ ChartRenderMode.PERIODICAL,+ )++ return {+ details: {+ ...details,+ ...occurenceDetails,+ },+ ...groupedChart,+ timeBucket,+ }+ }++ getErrorID(errorDTO: ErrorDTO): string {+ const { name, message, colno, lineno, filename } = errorDTO++ return hash(`${name}${message}${colno}${lineno}${filename}`, {+ length: 16,+ }).toString('hex')+ }++ async validateEIDs(eids: string[], pid: string) {+ const params = _reduce(+ eids,+ (acc, curr, index) => ({+ ...acc,+ [`e_${index}`]: curr,+ }),+ {},+ )++ const query = `SELECT count(DISTINCT eid) as count FROM errors WHERE pid = {pid:FixedString(12)} AND eid IN (${_map(+ params,+ (val, key) => `{${key}:FixedString(32)}`,+ ).join(', ')})`++ let result++ try {+ ;[result] = await clickhouse+ .query(query, { params: { ...params, pid } })+ .toPromise()+ } catch (reason) {+ console.error('validateEIDs - clickhouse request error')+ console.error(reason)+ throw new InternalServerErrorException(+ 'Error occured while validating error IDs',+ )+ }++ if (result.count !== _size(eids)) {+ throw new UnprocessableEntityException(+ 'Some of the error IDs you provided are not valid (do not exist in our database)',+ )+ }+ }++ async updateEIDStatus(+ eids: string[],+ status: 'resolved' | 'active',+ pid: string,+ ) {+ const params = _reduce(+ eids,+ (acc, curr, index) => ({+ ...acc,+ [`e_${index}`]: curr,+ }),+ {},+ )++ const query = `INSERT INTO error_statuses (eid, pid, status) VALUES ${_map(+ params,+ (val, key) =>+ `({${key}:FixedString(32)}, {pid:FixedString(12)}, '${status}')`,+ ).join(', ')}`++ try {+ await clickhouse.query(query, { params: { ...params, pid } }).toPromise()+ } catch (reason) {+ console.error('Error at PATCH error-status:')+ console.error(reason)+ throw new InternalServerErrorException(+ 'Error occured while updating error status',+ )+ }+ }+async getOnlineCountByProjectId(projectId: string) {// @ts-ignorereturn redis.countKeysByPattern(`hb:${projectId}:*`)@@ -2942,19 +3305,28 @@ export class AnalyticsService {}}- async getGeneralStats(): Promise<any> {- const trafficQuery = 'SELECT count(*) FROM analytics'- const customEVQuery = 'SELECT count(*) FROM customEV'- const performanceQuery = 'SELECT count(*) FROM performance'- const captchaQuery = 'SELECT count(*) FROM captcha'+ async getGeneralStats(): Promise<{+ users: number+ projects: number+ events: number+ }> {+ const query = `+ SELECT 'traffic' AS type, count(*) AS count FROM analytics+ UNION ALL+ SELECT 'customEV' AS type, count(*) AS count FROM customEV+ UNION ALL+ SELECT 'performance' AS type, count(*) AS count FROM performance+ UNION ALL+ SELECT 'captcha' AS type, count(*) AS count FROM captcha+ UNION ALL+ SELECT 'errors' AS type, count(*) AS count FROM errors+ `const users = await this.userService.count()const projects = await this.projectService.count()- const events =- (await clickhouse.query(trafficQuery).toPromise())[0]['count()'] +- (await clickhouse.query(customEVQuery).toPromise())[0]['count()'] +- (await clickhouse.query(performanceQuery).toPromise())[0]['count()'] +- (await clickhouse.query(captchaQuery).toPromise())[0]['count()']+ const results = await clickhouse.query(query).toPromise()+ // @ts-expect-error+ const events = results.reduce((total, row) => total + row.count, 0)await redis.set(REDIS_USERS_COUNT_KEY, users, 'EX', 630)await redis.set(REDIS_PROJECTS_COUNT_KEY, projects, 'EX', 630)diff --git a/apps/production/src/analytics/dto/error.dto.ts b/apps/production/src/analytics/dto/error.dto.tsnew file mode 100644index 000000000..efa35255e--- /dev/null+++ b/apps/production/src/analytics/dto/error.dto.ts@@ -0,0 +1,91 @@+import { ApiProperty } from '@nestjs/swagger'+import {+ IsNotEmpty,+ IsNumber,+ IsOptional,+ IsString,+ MaxLength,+ Matches,+} from 'class-validator'+import { PID_REGEX } from '../../common/constants'++export class ErrorDTO {+ @ApiProperty({+ example: 'aUn1quEid-3',+ required: true,+ description: 'The project ID',+ })+ @IsNotEmpty()+ @Matches(PID_REGEX, {+ message: 'Incorrect project ID format.',+ })+ pid: string++ // Tracking metrics+ @ApiProperty({+ example: 'Europe/Kiev',+ description: "User's timezone",+ })+ @IsOptional()+ @IsString()+ tz?: string++ @ApiProperty({+ example: '/articles/my-awesome-article-1',+ description: 'A page that user sent data from',+ })+ @IsOptional()+ @IsString()+ pg?: string++ @ApiProperty({+ example: 'en-GB',+ description: "User's locale",+ })+ @IsOptional()+ @IsString()+ lc?: string++ // Error metrics+ @ApiProperty({+ example: 'ParseError',+ required: true,+ maxLength: 200,+ })+ @IsNotEmpty()+ @IsString()+ @MaxLength(200)+ name: string++ @ApiProperty({+ example: 'Malformed input',+ maxLength: 2000,+ })+ @IsOptional()+ @IsString()+ @MaxLength(2000)+ message?: string++ @ApiProperty({+ example: 12,+ })+ @IsOptional()+ @IsNumber()+ lineno?: number++ @ApiProperty({+ example: 510,+ })+ @IsOptional()+ @IsNumber()+ colno?: number++ @ApiProperty({+ example: 'https://example.com/assets/convert.js',+ maxLength: 1000,+ })+ @IsOptional()+ @IsString()+ @MaxLength(1000)+ filename?: string+}diff --git a/apps/production/src/analytics/dto/get-error.dto.ts b/apps/production/src/analytics/dto/get-error.dto.tsnew file mode 100644index 000000000..ee8e21b6e--- /dev/null+++ b/apps/production/src/analytics/dto/get-error.dto.ts@@ -0,0 +1,15 @@+import { ApiProperty, PickType } from '@nestjs/swagger'+import { IsNotEmpty } from 'class-validator'+import { AnalyticsGET_DTO } from './getData.dto'++export class GetErrorDTO extends PickType(AnalyticsGET_DTO, [+ 'pid',+ 'period',+ 'from',+ 'to',+ 'timezone',+] as const) {+ @ApiProperty()+ @IsNotEmpty()+ eid: string+}diff --git a/apps/production/src/analytics/dto/get-errors.dto.ts b/apps/production/src/analytics/dto/get-errors.dto.tsnew file mode 100644index 000000000..6d85cac2d--- /dev/null+++ b/apps/production/src/analytics/dto/get-errors.dto.ts@@ -0,0 +1,23 @@+import { PickType, ApiProperty } from '@nestjs/swagger'+import { AnalyticsGET_DTO } from './getData.dto'++export class GetErrorsDto extends PickType(AnalyticsGET_DTO, [+ 'pid',+ 'period',+ 'from',+ 'to',+ 'filters',+ 'timezone',+] as const) {+ take: number++ skip: number++ @ApiProperty({+ example: {+ showResolved: true,+ },+ description: 'Errors list options',+ })+ options?: string+}diff --git a/apps/production/src/analytics/dto/patch-status.dto.ts b/apps/production/src/analytics/dto/patch-status.dto.tsnew file mode 100644index 000000000..2da498d93--- /dev/null+++ b/apps/production/src/analytics/dto/patch-status.dto.ts@@ -0,0 +1,52 @@+import { ApiProperty } from '@nestjs/swagger'+import {+ IsNotEmpty,+ IsIn,+ ValidateIf,+ IsString,+ IsArray,+ Matches,+} from 'class-validator'+import { PID_REGEX } from '../../common/constants'++export class PatchStatusDTO {+ @ApiProperty({+ example: 'aUn1quEid-3',+ required: true,+ description: 'The project ID',+ })+ @IsNotEmpty()+ @Matches(PID_REGEX, {+ message: 'Incorrect project ID format.',+ })+ pid: string++ @ApiProperty({+ example: 'c138c2b1826d9fec65d230e471ab2c25',+ description: 'Error id',+ })+ @ValidateIf(o => !o.eids || o.eid)+ @IsNotEmpty()+ @IsString()+ eid: string++ @ApiProperty({+ example: [+ 'c138c2b1826d9fec65d230e471ab2c25',+ 'b2d12867d59d1d4f9986b52311a97cd5',+ ],+ description: 'Error id list',+ })+ @ValidateIf(o => o.eids || !o.eid)+ @IsNotEmpty()+ @IsArray()+ @IsString({ each: true })+ eids: string++ @ApiProperty({+ enum: ['resolved', 'active'],+ required: true,+ })+ @IsIn(['resolved', 'active'])+ status: 'resolved' | 'active'+}diff --git a/apps/production/src/common/constants.ts b/apps/production/src/common/constants.tsindex 2487caaff..ef06c109e 100644--- a/apps/production/src/common/constants.ts+++ b/apps/production/src/common/constants.ts@@ -79,6 +79,7 @@ const REDIS_LOG_DATA_CACHE_KEY = 'log_cache'const REDIS_LOG_CAPTCHA_CACHE_KEY = 'log:captcha'const REDIS_LOG_PERF_CACHE_KEY = 'perf_cache'const REDIS_LOG_CUSTOM_CACHE_KEY = 'log_custom_cache_v3'+const REDIS_LOG_ERROR_CACHE_KEY = 'log_error_cache'const REDIS_SESSION_SALT_KEY = 'log_salt' // is updated every 24 hoursconst REDIS_USERS_COUNT_KEY = 'stats:users_count'const REDIS_PROJECTS_COUNT_KEY = 'stats:projects_count'@@ -145,6 +146,8 @@ const TRAFFIC_COLUMNS = ['ca',]+const ERROR_COLUMNS = ['pg', 'dv', 'br', 'os', 'lc', 'cc', 'rg', 'ct']+const ALL_COLUMNS = [...TRAFFIC_COLUMNS, 'ev']const CAPTCHA_COLUMNS = ['cc', 'br', 'os', 'dv']@@ -230,6 +233,7 @@ export {redisUserUsageinfoCacheTimeout,TRAFFIC_COLUMNS,CAPTCHA_COLUMNS,+ ERROR_COLUMNS,PERFORMANCE_COLUMNS,sentryIgnoreErrors,isProxiedByCloudflare,@@ -243,4 +247,6 @@ export {BLOG_POSTS_ROOT,TRAFFIC_SPIKE_ALLOWED_PERCENTAGE,AFFILIATE_CUT,+ REDIS_LOG_ERROR_CACHE_KEY,+ PID_REGEX,}diff --git a/apps/production/src/project/project.controller.ts b/apps/production/src/project/project.controller.tsindex 8a78e06c0..d5e47d112 100644--- a/apps/production/src/project/project.controller.ts+++ b/apps/production/src/project/project.controller.ts@@ -173,11 +173,17 @@ export class ProjectController {_map(paginated.results, ({ id }) => id),)+ const pidsWithErrorData =+ await this.projectService.getPIDsWhereErrorsDataExists(+ _map(paginated.results, ({ id }) => id),+ )+paginated.results = _map(paginated.results, p => ({...p,isOwner: true,isLocked: !!user?.dashboardBlockReason,isDataExists: _includes(pidsWithData, p?.id),+ isErrorDataExists: _includes(pidsWithErrorData, p?.id),}))return {@@ -250,6 +256,11 @@ export class ProjectController {_map(paginated.results, ({ project }) => project.id),)+ const pidsWithErrorData =+ await this.projectService.getPIDsWhereErrorsDataExists(+ _map(paginated.results, ({ project }) => project.id),+ )+paginated.results = _map(paginated.results, share => {const project = processProjectUser(share.project)@@ -261,6 +272,7 @@ export class ProjectController {passwordHash: undefined,isLocked: !!share?.project?.admin?.dashboardBlockReason,isDataExists: _includes(pidsWithData, share?.project?.id),+ isErrorDataExists: _includes(pidsWithErrorData, share?.project?.id),},}})@@ -1840,11 +1852,16 @@ export class ProjectController {await this.projectService.getPIDsWhereAnalyticsDataExists([id]),)+ const isErrorDataExists = !_isEmpty(+ await this.projectService.getPIDsWhereErrorsDataExists([id]),+ )+return {..._omit(project, ['admin', 'passwordHash', 'share']),isOwner: uid === project.admin?.id,isLocked: !!project.admin?.dashboardBlockReason,isDataExists,+ isErrorDataExists,}}}diff --git a/apps/production/src/project/project.service.ts b/apps/production/src/project/project.service.tsindex 097ab761c..2cd1cd0ab 100644--- a/apps/production/src/project/project.service.ts+++ b/apps/production/src/project/project.service.ts@@ -888,6 +888,50 @@ export class ProjectService {return _map(result, ({ pid }) => pid)}+ async getPIDsWhereErrorsDataExists(projectIds: string[]) {+ if (_isEmpty(projectIds)) {+ return {}+ }++ const params = _reduce(+ projectIds,+ (acc, curr, index) => ({+ ...acc,+ [`pid_${index}`]: curr,+ }),+ {},+ )++ const pids = _join(+ _map(params, (val, key) => `{${key}:FixedString(12)}`),+ ',',+ )++ const query = `+ SELECT+ pid,+ CASE+ WHEN EXISTS (+ SELECT 1+ FROM errors+ WHERE pid IN (${pids})+ )+ THEN 1+ ELSE 0+ END AS exists+ FROM+ (+ SELECT DISTINCT pid+ FROM errors+ WHERE pid IN (${pids})+ );+ `++ const result: any = await clickhouse.query(query, { params }).toPromise()++ return _map(result, ({ pid }) => pid)+ }+async getSubscriberByEmail(projectId: string, email: string) {return this.projectSubscriberRepository.findOne({where: { projectId, email },diff --git a/apps/production/src/task-manager/task-manager.service.ts b/apps/production/src/task-manager/task-manager.service.tsindex 683e6ce06..1fbe21e5d 100644--- a/apps/production/src/task-manager/task-manager.service.ts+++ b/apps/production/src/task-manager/task-manager.service.ts@@ -57,6 +57,7 @@ import {PAYPAL_CLIENT_SECRET,TRAFFIC_SPIKE_ALLOWED_PERCENTAGE,isDevelopment,+ REDIS_LOG_ERROR_CACHE_KEY,} from '../common/constants'import { CHPlanUsage } from './interfaces'import { getRandomTip } from '../common/utils'@@ -412,6 +413,7 @@ export class TaskManagerService {const customData = await redis.lrange(REDIS_LOG_CUSTOM_CACHE_KEY, 0, -1)const perfData = await redis.lrange(REDIS_LOG_PERF_CACHE_KEY, 0, -1)const captchaData = await redis.lrange(REDIS_LOG_CAPTCHA_CACHE_KEY, 0, -1)+ const errorData = await redis.lrange(REDIS_LOG_ERROR_CACHE_KEY, 0, -1)if (!_isEmpty(data)) {await redis.del(REDIS_LOG_DATA_CACHE_KEY)@@ -448,6 +450,19 @@ export class TaskManagerService {}}+ if (!_isEmpty(errorData)) {+ await redis.del(REDIS_LOG_ERROR_CACHE_KEY)+ const query = `INSERT INTO errors (*) VALUES ${_join(errorData, ',')}`++ try {+ await clickhouse.query(query).toPromise()+ } catch (e) {+ console.error(+ `[CRON WORKER] Error whilst saving error events data: ${e}`,+ )+ }+ }+if (!_isEmpty(perfData)) {await redis.del(REDIS_LOG_PERF_CACHE_KEY)diff --git a/apps/selfhosted/src/analytics/analytics.controller.ts b/apps/selfhosted/src/analytics/analytics.controller.tsindex 1bc62af34..a125666ba 100644--- a/apps/selfhosted/src/analytics/analytics.controller.ts+++ b/apps/selfhosted/src/analytics/analytics.controller.ts@@ -21,6 +21,7 @@ import {UseGuards,Get,Post,+ Patch,Headers,BadRequestException,InternalServerErrorException,@@ -64,8 +65,13 @@ import {MIN_PAGES_IN_FUNNEL,MAX_PAGES_IN_FUNNEL,clickhouse,+ REDIS_LOG_ERROR_CACHE_KEY,} from '../common/constants'-import { getGeoDetails, getIPFromHeaders } from '../common/utils'+import {+ checkRateLimit,+ getGeoDetails,+ getIPFromHeaders,+} from '../common/utils'import { BotDetection } from '../common/decorators/bot-detection.decorator'import { BotDetectionGuard } from '../common/guards/bot-detection.guard'import { GetCustomEventsDto } from './dto/get-custom-events.dto'@@ -78,6 +84,10 @@ import {} from './interfaces'import { GetSessionsDto } from './dto/get-sessions.dto'import { GetSessionDto } from './dto/get-session.dto'+import { ErrorDTO } from './dto/error.dto'+import { GetErrorsDto } from './dto/get-errors.dto'+import { GetErrorDTO } from './dto/get-error.dto'+import { PatchStatusDTO } from './dto/patch-status.dto'// eslint-disable-next-line @typescript-eslint/no-var-requiresconst mysql = require('mysql2')@@ -216,6 +226,43 @@ const customLogDTO = (]}+const errorLogDTO = (+ eid: string,+ pid: string,+ pg: string,+ dv: string,+ br: string,+ os: string,+ lc: string,+ cc: string,+ rg: string,+ ct: string,+ name: string,+ message: string,+ lineno: number,+ colno: number,+ filename: string,+): Array<string | number | string[]> => {+ return [+ eid,+ pid,+ pg,+ dv,+ br,+ os,+ lc,+ cc,+ rg,+ ct,+ name,+ message,+ lineno,+ colno,+ filename,+ dayjs.utc().format('YYYY-MM-DD HH:mm:ss'),+ ]+}+const getElValue = el => {if (_isArray(el)) {return `[${_map(el, getElValue).join(',')}]`@@ -280,6 +327,27 @@ const getPIDsArray = (pids, pid) => {return pids}+const getEIDsArray = (eids, eid) => {+ const eidsEmpty = _isEmpty(eids)+ const eidEmpty = _isEmpty(eid)+ if (eidsEmpty && eidEmpty) {+ throw new BadRequestException(+ "An array of Error ID's (eids) or a Error ID (eid) has to be provided",+ )+ }+ if (!eidsEmpty && !eidEmpty) {+ throw new BadRequestException(+ "Please provide either an array of Error ID's (eids) or a Error ID (eid), but not both",+ )+ }++ if (!eidsEmpty) {+ return eids+ }++ return [eid]+}+const getPagesArray = (rawPages: string): string[] => {try {const pages = JSON.parse(rawPages)@@ -1431,4 +1499,253 @@ export class AnalyticsController {timeBucket: timeBucketForAllTime,}}++ @Get('errors-filters')+ @Auth([], true, true)+ async getErrorsFilters(+ @Query() data: GetFiltersDto,+ @CurrentUserId() uid: string,+ ): Promise<any> {+ const { pid, type } = data+ this.analyticsService.validatePID(pid)++ await this.analyticsService.checkProjectAccess(pid, uid)++ return this.analyticsService.getErrorsFilters(pid, type)+ }++ // Log error event+ @Post('error')+ @UseGuards(BotDetectionGuard)+ @BotDetection()+ @Public()+ async logError(+ @Body() errorDTO: ErrorDTO,+ @Headers() headers,+ @Ip() reqIP,+ ): Promise<any> {+ const { 'user-agent': userAgent, origin } = headers++ const ip = getIPFromHeaders(headers) || reqIP || ''++ await this.analyticsService.validate(errorDTO, origin, 'error', ip)++ const {+ city = 'NULL',+ region = 'NULL',+ country = 'NULL',+ } = getGeoDetails(ip, errorDTO.tz)++ const ua = UAParser(userAgent)+ const dv = ua.device.type || 'desktop'+ const br = ua.browser.name+ const os = ua.os.name++ const { name, message, lineno, colno, filename } = errorDTO++ const dto = errorLogDTO(+ this.analyticsService.getErrorID(errorDTO),+ errorDTO.pid,+ errorDTO.pg,+ dv,+ br,+ os,+ errorDTO.lc,+ country,+ region,+ city,+ name,+ message,+ lineno,+ colno,+ filename,+ )++ try {+ const values = `(${dto.map(getElValue).join(',')})`+ await redis.rpush(REDIS_LOG_ERROR_CACHE_KEY, values)+ } catch (e) {+ this.logger.error(e)+ throw new InternalServerErrorException(+ 'Error occured while saving the custom event',+ )+ }+ }++ // Update error(s) status+ @Patch('error-status')+ @Auth([], true, true)+ async patchStatus(+ @Body() statusDTO: PatchStatusDTO,+ @CurrentUserId() uid: string,+ @Headers() headers,+ @Ip() reqIP,+ ): Promise<any> {+ const { pid, eid, eids: unprocessedEids, status } = statusDTO+ const ip = getIPFromHeaders(headers) || reqIP || ''++ await checkRateLimit(ip, 'error-status', 100, 1800)++ this.analyticsService.validatePID(pid)+ await this.analyticsService.checkProjectAccess(pid, uid)++ const eids = getEIDsArray(unprocessedEids, eid)++ await this.analyticsService.validateEIDs(eids, pid)++ return this.analyticsService.updateEIDStatus(eids, status, pid)+ }++ @Get('errors')+ @Auth([], true, true)+ async getErrors(+ @Query() data: GetErrorsDto,+ @CurrentUserId() uid: string,+ ): Promise<any> {+ const {+ pid,+ period,+ from,+ to,+ filters,+ timezone = DEFAULT_TIMEZONE,+ options,+ } = data+ this.analyticsService.validatePID(pid)++ if (!_isEmpty(period)) {+ this.analyticsService.validatePeriod(period)+ }++ await this.analyticsService.checkProjectAccess(pid, uid)++ const take = this.analyticsService.getSafeNumber(data.take, 30)+ const skip = this.analyticsService.getSafeNumber(data.skip, 0)++ if (take > 150) {+ throw new BadRequestException(+ 'The maximum number of errors to return is 150',+ )+ }++ let timeBucket+ let diff++ if (period === 'all') {+ const res = await this.analyticsService.getTimeBucketForAllTime(+ pid,+ period,+ timezone,+ )++ // eslint-disable-next-line prefer-destructuring+ timeBucket = res.timeBucket[0]+ diff = res.diff+ } else {+ timeBucket = getLowestPossibleTimeBucket(period, from, to)+ }++ this.analyticsService.validateTimebucket(timeBucket)+ const [filtersQuery, filtersParams, appliedFilters] =+ this.analyticsService.getFiltersQuery(filters, DataType.ANALYTICS)++ const safeTimezone = this.analyticsService.getSafeTimezone(timezone)+ const { groupFrom, groupTo } = this.analyticsService.getGroupFromTo(+ from,+ to,+ timeBucket,+ period,+ safeTimezone,+ diff,+ )++ const paramsData = {+ params: {+ pid,+ groupFrom,+ groupTo,+ ...filtersParams,+ },+ }++ const errors = await this.analyticsService.getErrorsList(+ options,+ filtersQuery,+ paramsData,+ safeTimezone,+ take,+ skip,+ )++ return {+ errors,+ appliedFilters,+ take,+ skip,+ }+ }++ @Get('get-error')+ @Auth([], true, true)+ async getError(+ @Query() data: GetErrorDTO,+ @CurrentUserId() uid: string,+ ): Promise<any> {+ const {+ pid,+ timezone = DEFAULT_TIMEZONE,+ eid,+ period,+ from,+ to,+ //+ } = data+ this.analyticsService.validatePID(pid)++ if (!_isEmpty(period)) {+ this.analyticsService.validatePeriod(period)+ }++ await this.analyticsService.checkProjectAccess(pid, uid)++ let timeBucket+ let diff++ if (period === 'all') {+ const res = await this.analyticsService.getTimeBucketForAllTime(+ pid,+ period,+ timezone,+ )++ // eslint-disable-next-line prefer-destructuring+ timeBucket = res.timeBucket[0]+ diff = res.diff+ } else {+ timeBucket = getLowestPossibleTimeBucket(period, from, to)+ }++ this.analyticsService.validateTimebucket(timeBucket)++ const safeTimezone = this.analyticsService.getSafeTimezone(timezone)+ const { groupFrom, groupTo } = this.analyticsService.getGroupFromTo(+ from,+ to,+ timeBucket,+ period,+ safeTimezone,+ diff,+ )++ const result = await this.analyticsService.getErrorDetails(+ pid,+ eid,+ safeTimezone,+ groupFrom,+ groupTo,+ timeBucket,+ )++ return result+ }}diff --git a/apps/selfhosted/src/analytics/analytics.service.ts b/apps/selfhosted/src/analytics/analytics.service.tsindex 51b7f76c2..073c162b7 100644--- a/apps/selfhosted/src/analytics/analytics.service.ts+++ b/apps/selfhosted/src/analytics/analytics.service.ts@@ -45,6 +45,7 @@ import {REDIS_SESSION_SALT_KEY,TRAFFIC_COLUMNS,PERFORMANCE_COLUMNS,+ ERROR_COLUMNS,} from '../common/constants'import { millisecondsToSeconds, sumArrays } from '../common/utils'import { PageviewsDTO } from './dto/pageviews.dto'@@ -76,6 +77,7 @@ import {IPageflow,PerfMeasure,} from './interfaces'+import { ErrorDTO } from './dto/error.dto'dayjs.extend(utc)dayjs.extend(dayjsTimezone)@@ -253,7 +255,7 @@ const generateParamsQuery = (subQuery: string,customEVFilterApplied: boolean,isPageInclusiveFilterSet: boolean,- isPerformance?: boolean,+ type: 'traffic' | 'performance' | 'errors',measure?: PerfMeasure,): string => {let columns = [`${col} as name`]@@ -266,7 +268,7 @@ const generateParamsQuery = (const columnsQuery = columns.join(', ')- if (isPerformance) {+ if (type === 'performance') {const processedMeasure = measure === 'quantiles' ? 'median' : measureconst fn = MEASURES_MAP[processedMeasure]@@ -278,6 +280,10 @@ const generateParamsQuery = (return `SELECT ${columnsQuery}, round(divide(${fn}(pageLoad), 1000), 2) as count ${subQuery} AND ${col} IS NOT NULL GROUP BY ${columnsQuery}`}+ if (type === 'errors') {+ return `SELECT ${columnsQuery}, count(*) as count ${subQuery} AND ${col} IS NOT NULL GROUP BY ${columnsQuery}`+ }+if (customEVFilterApplied) {return `SELECT ${columnsQuery}, count(*) as count ${subQuery} AND ${col} IS NOT NULL GROUP BY ${columnsQuery}`}@@ -379,9 +385,9 @@ export class AnalyticsService {}async validate(- logDTO: PageviewsDTO | EventsDTO,+ logDTO: PageviewsDTO | EventsDTO | ErrorDTO,origin: string,- type: 'custom' | 'log' = 'log',+ type: 'custom' | 'log' | 'error' = 'log',ip?: string,): Promise<string | null> {if (_isEmpty(logDTO)) {@@ -439,6 +445,22 @@ export class AnalyticsService {return null}+ async getErrorsFilters(pid: string, type: string): Promise<Array<string>> {+ if (!_includes(ERROR_COLUMNS, type)) {+ throw new UnprocessableEntityException(+ `The provided type (${type}) is incorrect`,+ )+ }++ const query = `SELECT ${type} FROM errors WHERE pid={pid:FixedString(12)} AND ${type} IS NOT NULL GROUP BY ${type}`++ const results = await clickhouse+ .query(query, { params: { pid } })+ .toPromise()++ return _map(results, type)+ }+getDataTypeColumns(dataType: DataType): string[] {if (dataType === DataType.ANALYTICS) {return TRAFFIC_COLUMNS@@ -1483,34 +1505,39 @@ export class AnalyticsService {subQuery: string,customEVFilterApplied: boolean,paramsData: any,- isPerformance: boolean,+ type: 'traffic' | 'performance' | 'errors',measure?: PerfMeasure,): Promise<any> {const params = {}// We need this to display all the pageview related data (e.g. country, browser) when user applies an inclusive filter on the Page column- const isPageInclusiveFilterSet = isPerformance- ? false- : !_isEmpty(- _find(- parsedFilters,- filter => filter.column === 'pg' && !filter.isExclusive,- ),- )+ const isPageInclusiveFilterSet =+ type === 'performance'+ ? false+ : !_isEmpty(+ _find(+ parsedFilters,+ filter => filter.column === 'pg' && !filter.isExclusive,+ ),+ )let columns = TRAFFIC_COLUMNS- if (isPerformance) {+ if (type === 'performance') {columns = PERFORMANCE_COLUMNS}+ if (type === 'errors') {+ columns = ERROR_COLUMNS+ }+const paramsPromises = _map(columns, async col => {const query = generateParamsQuery(col,subQuery,customEVFilterApplied,isPageInclusiveFilterSet,- isPerformance,+ type,measure,)const res = await clickhouse.query(query, paramsData).toPromise()@@ -1909,6 +1936,47 @@ export class AnalyticsService {}}+ generateErrorsAggregationQuery(+ timeBucket: TimeBucketType,+ filtersQuery: string,+ safeTimezone: string,+ mode: ChartRenderMode,+ ): string {+ const timeBucketFunc = timeBucketConversion[timeBucket]+ const [selector, groupBy] = this.getGroupSubquery(timeBucket)+ const tzFromDate = `toTimeZone(parseDateTimeBestEffort({groupFrom:String}), '${safeTimezone}')`+ const tzToDate = `toTimeZone(parseDateTimeBestEffort({groupTo:String}), '${safeTimezone}')`++ const baseQuery = `+ SELECT+ ${selector},+ count() as count+ FROM (+ SELECT pg, dv, br, os, lc, cc, rg, ct,+ ${timeBucketFunc}(toTimeZone(created, '${safeTimezone}')) as tz_created+ FROM errors+ WHERE+ pid = {pid:FixedString(12)}+ AND created BETWEEN ${tzFromDate} AND ${tzToDate}+ ${filtersQuery}+ ) as subquery+ GROUP BY ${groupBy}+ ORDER BY ${groupBy}+ `++ if (mode === ChartRenderMode.CUMULATIVE) {+ return `+ SELECT+ *,+ sum(pageviews) OVER (ORDER BY ${groupBy}) as pageviews,+ sum(uniques) OVER (ORDER BY ${groupBy}) as uniques+ FROM (${baseQuery})+ `+ }++ return baseQuery+ }+async groupByTimeBucket(timeBucket: TimeBucketType,from: string,@@ -1936,7 +2004,7 @@ export class AnalyticsService {if (!_some(_values(params), val => !_isEmpty(val))) {throw new BadRequestException(- 'The are no parameters for the specified time frames',+ 'There are no parameters for the specified time frames',)}})(),@@ -1978,7 +2046,7 @@ export class AnalyticsService {subQuery,customEVFilterApplied,paramsData,- false,+ 'traffic',)}@@ -2045,6 +2113,91 @@ export class AnalyticsService {})}+ extractErrorsChartData(result, x: string[]): any {+ const count = Array(x.length).fill(0)++ for (let row = 0; row < _size(result); ++row) {+ const dateString = this.generateDateString(result[row])++ const index = x.indexOf(dateString)++ if (index !== -1) {+ count[index] = result[row].count+ }+ }++ return {+ count,+ }+ }++ async groupErrorsByTimeBucket(+ timeBucket: TimeBucketType,+ from: string,+ to: string,+ subQuery: string,+ filtersQuery: string,+ paramsData: object,+ safeTimezone: string,+ mode: ChartRenderMode,+ ): Promise<object | void> {+ let params: unknown = {}+ let chart: unknown = {}++ const promises = [+ // Params+ (async () => {+ params = await this.generateParams(+ null,+ subQuery,+ false,+ paramsData,+ 'errors',+ )++ if (!_some(_values(params), val => !_isEmpty(val))) {+ throw new BadRequestException(+ 'There are no error details for specified time frame',+ )+ }+ })(),++ // Chart data+ (async () => {+ const { xShifted } = this.generateXAxis(+ timeBucket,+ from,+ to,+ safeTimezone,+ )++ const query = this.generateErrorsAggregationQuery(+ timeBucket,+ filtersQuery,+ safeTimezone,+ mode,+ )++ const result = <Array<TrafficCEFilterCHResponse>>(+ await clickhouse.query(query, paramsData).toPromise()+ )+ const { count } = this.extractErrorsChartData(result, xShifted)++ chart = {+ x: xShifted,+ occurrences: count,+ }+ })(),+ ]++ await Promise.all(promises)++ return Promise.resolve({+ params,+ chart,+ })+ }+extractPerformanceChartData(result, x: string[]): any {const dns = Array(x.length).fill(0)const tls = Array(x.length).fill(0)@@ -2145,13 +2298,13 @@ export class AnalyticsService {subQuery,false,paramsData,- true,+ 'performance',measure,)if (!_some(_values(params), val => !_isEmpty(val))) {throw new BadRequestException(- 'The are no parameters for the specified time frames',+ 'There are no parameters for the specified time frames',)}})(),@@ -2526,6 +2679,236 @@ export class AnalyticsService {return result}+ async getErrorsList(+ options: string,+ filtersQuery: string,+ paramsData: object,+ safeTimezone: string,+ take = 30,+ skip = 0,+ ): Promise<object | void> {+ const tzFromDate = `toTimeZone(parseDateTimeBestEffort({groupFrom:String}), '${safeTimezone}')`+ const tzToDate = `toTimeZone(parseDateTimeBestEffort({groupTo:String}), '${safeTimezone}')`++ let parsedOptions: {+ showResolved?: boolean+ } = {}++ try {+ parsedOptions = JSON.parse(options)+ } catch (reason) {+ console.error('[getErrorsList] Failed to parse options:', options)+ }++ const query = `+ SELECT+ eid,+ any(name) as name,+ any(message) as message,+ any(filename) as filename,+ count(*) as count,+ max(created) as last_seen,+ status.status+ FROM (+ SELECT eid, name, message, filename, created+ FROM errors+ WHERE pid = {pid:FixedString(12)}+ AND created BETWEEN ${tzFromDate} AND ${tzToDate}+ ${filtersQuery}+ ) AS errors+ LEFT JOIN (+ SELECT+ eid,+ argMax(status, updated) AS status+ FROM error_statuses+ WHERE pid = {pid:FixedString(12)}+ GROUP BY eid+ ) AS status ON errors.eid = status.eid+ ${+ parsedOptions?.showResolved+ ? ''+ : "WHERE status.status = 'active' OR status.status = 'regressed'"+ }+ GROUP BY errors.eid, status.status+ ORDER BY last_seen DESC+ LIMIT ${take}+ OFFSET ${skip};+ `++ const result = await clickhouse.query(query, paramsData).toPromise()++ return result+ }++ async getErrorDetails(+ pid: string,+ eid: string,+ safeTimezone: string,+ groupFrom: string,+ groupTo: string,+ timeBucket: any,+ ): Promise<any> {+ const tzFromDate = `toTimeZone(parseDateTimeBestEffort({groupFrom:String}), '${safeTimezone}')`+ const tzToDate = `toTimeZone(parseDateTimeBestEffort({groupTo:String}), '${safeTimezone}')`++ const queryErrorDetails = `+ SELECT+ subquery.eid,+ any(subquery.name) AS name,+ any(subquery.message) AS message,+ any(subquery.filename) AS filename,+ any(subquery.colno) AS colno,+ any(subquery.lineno) AS lineno,+ count(*) AS count,+ status.status+ FROM (+ SELECT+ eid,+ name,+ message,+ filename,+ colno,+ lineno+ FROM errors+ WHERE pid = {pid:FixedString(12)}+ AND eid = {eid:FixedString(32)}+ AND created BETWEEN ${tzFromDate} AND ${tzToDate}+ ) AS subquery+ LEFT JOIN (+ SELECT+ eid,+ argMax(status, updated) AS status+ FROM error_statuses+ WHERE pid = {pid:FixedString(12)}+ AND eid = {eid:FixedString(32)}+ GROUP BY eid+ ) AS status ON subquery.eid = status.eid+ GROUP BY subquery.eid, status.status;+ `++ const queryFirstLastSeen = `+ SELECT+ max(created) AS last_seen,+ min(created) AS first_seen+ FROM errors+ WHERE+ pid = {pid:FixedString(12)}+ AND eid = {eid:FixedString(32)};+ `++ const paramsData = {+ params: {+ pid,+ eid,+ groupFrom,+ groupTo,+ },+ }++ const details = (+ await clickhouse.query(queryErrorDetails, paramsData).toPromise()+ )[0]++ const occurenceDetails = (+ await clickhouse.query(queryFirstLastSeen, paramsData).toPromise()+ )[0]++ const groupedChart = await this.groupErrorsByTimeBucket(+ timeBucket,+ groupFrom,+ groupTo,+ `FROM errors WHERE eid = {eid:FixedString(32)} AND created BETWEEN {groupFrom:String} AND {groupTo:String}`,+ 'AND eid = {eid:FixedString(32)}',+ paramsData,+ safeTimezone,+ ChartRenderMode.PERIODICAL,+ )++ return {+ details: {+ ...details,+ ...occurenceDetails,+ },+ ...groupedChart,+ timeBucket,+ }+ }++ getErrorID(errorDTO: ErrorDTO): string {+ const { name, message, colno, lineno, filename } = errorDTO++ return hash(`${name}${message}${colno}${lineno}${filename}`, {+ length: 16,+ }).toString('hex')+ }++ async validateEIDs(eids: string[], pid: string) {+ const params = _reduce(+ eids,+ (acc, curr, index) => ({+ ...acc,+ [`e_${index}`]: curr,+ }),+ {},+ )++ const query = `SELECT count(DISTINCT eid) as count FROM errors WHERE pid = {pid:FixedString(12)} AND eid IN (${_map(+ params,+ (val, key) => `{${key}:FixedString(32)}`,+ ).join(', ')})`++ let result++ try {+ ;[result] = await clickhouse+ .query(query, { params: { ...params, pid } })+ .toPromise()+ } catch (reason) {+ console.error('validateEIDs - clickhouse request error')+ console.error(reason)+ throw new InternalServerErrorException(+ 'Error occured while validating error IDs',+ )+ }++ if (result.count !== _size(eids)) {+ throw new UnprocessableEntityException(+ 'Some of the error IDs you provided are not valid (do not exist in our database)',+ )+ }+ }++ async updateEIDStatus(+ eids: string[],+ status: 'resolved' | 'active',+ pid: string,+ ) {+ const params = _reduce(+ eids,+ (acc, curr, index) => ({+ ...acc,+ [`e_${index}`]: curr,+ }),+ {},+ )++ const query = `INSERT INTO error_statuses (eid, pid, status) VALUES ${_map(+ params,+ (val, key) =>+ `({${key}:FixedString(32)}, {pid:FixedString(12)}, '${status}')`,+ ).join(', ')}`++ try {+ await clickhouse.query(query, { params: { ...params, pid } }).toPromise()+ } catch (reason) {+ console.error('Error at PATCH error-status:')+ console.error(reason)+ throw new InternalServerErrorException(+ 'Error occured while updating error status',+ )+ }+ }+async getOnlineCountByProjectId(projectId: string) {// @ts-ignorereturn redis.countKeysByPattern(`hb:${projectId}:*`)diff --git a/apps/selfhosted/src/analytics/dto/error.dto.ts b/apps/selfhosted/src/analytics/dto/error.dto.tsnew file mode 100644index 000000000..efa35255e--- /dev/null+++ b/apps/selfhosted/src/analytics/dto/error.dto.ts@@ -0,0 +1,91 @@+import { ApiProperty } from '@nestjs/swagger'+import {+ IsNotEmpty,+ IsNumber,+ IsOptional,+ IsString,+ MaxLength,+ Matches,+} from 'class-validator'+import { PID_REGEX } from '../../common/constants'++export class ErrorDTO {+ @ApiProperty({+ example: 'aUn1quEid-3',+ required: true,+ description: 'The project ID',+ })+ @IsNotEmpty()+ @Matches(PID_REGEX, {+ message: 'Incorrect project ID format.',+ })+ pid: string++ // Tracking metrics+ @ApiProperty({+ example: 'Europe/Kiev',+ description: "User's timezone",+ })+ @IsOptional()+ @IsString()+ tz?: string++ @ApiProperty({+ example: '/articles/my-awesome-article-1',+ description: 'A page that user sent data from',+ })+ @IsOptional()+ @IsString()+ pg?: string++ @ApiProperty({+ example: 'en-GB',+ description: "User's locale",+ })+ @IsOptional()+ @IsString()+ lc?: string++ // Error metrics+ @ApiProperty({+ example: 'ParseError',+ required: true,+ maxLength: 200,+ })+ @IsNotEmpty()+ @IsString()+ @MaxLength(200)+ name: string++ @ApiProperty({+ example: 'Malformed input',+ maxLength: 2000,+ })+ @IsOptional()+ @IsString()+ @MaxLength(2000)+ message?: string++ @ApiProperty({+ example: 12,+ })+ @IsOptional()+ @IsNumber()+ lineno?: number++ @ApiProperty({+ example: 510,+ })+ @IsOptional()+ @IsNumber()+ colno?: number++ @ApiProperty({+ example: 'https://example.com/assets/convert.js',+ maxLength: 1000,+ })+ @IsOptional()+ @IsString()+ @MaxLength(1000)+ filename?: string+}diff --git a/apps/selfhosted/src/analytics/dto/get-error.dto.ts b/apps/selfhosted/src/analytics/dto/get-error.dto.tsnew file mode 100644index 000000000..ee8e21b6e--- /dev/null+++ b/apps/selfhosted/src/analytics/dto/get-error.dto.ts@@ -0,0 +1,15 @@+import { ApiProperty, PickType } from '@nestjs/swagger'+import { IsNotEmpty } from 'class-validator'+import { AnalyticsGET_DTO } from './getData.dto'++export class GetErrorDTO extends PickType(AnalyticsGET_DTO, [+ 'pid',+ 'period',+ 'from',+ 'to',+ 'timezone',+] as const) {+ @ApiProperty()+ @IsNotEmpty()+ eid: string+}diff --git a/apps/selfhosted/src/analytics/dto/get-errors.dto.ts b/apps/selfhosted/src/analytics/dto/get-errors.dto.tsnew file mode 100644index 000000000..6d85cac2d--- /dev/null+++ b/apps/selfhosted/src/analytics/dto/get-errors.dto.ts@@ -0,0 +1,23 @@+import { PickType, ApiProperty } from '@nestjs/swagger'+import { AnalyticsGET_DTO } from './getData.dto'++export class GetErrorsDto extends PickType(AnalyticsGET_DTO, [+ 'pid',+ 'period',+ 'from',+ 'to',+ 'filters',+ 'timezone',+] as const) {+ take: number++ skip: number++ @ApiProperty({+ example: {+ showResolved: true,+ },+ description: 'Errors list options',+ })+ options?: string+}diff --git a/apps/selfhosted/src/analytics/dto/patch-status.dto.ts b/apps/selfhosted/src/analytics/dto/patch-status.dto.tsnew file mode 100644index 000000000..2da498d93--- /dev/null+++ b/apps/selfhosted/src/analytics/dto/patch-status.dto.ts@@ -0,0 +1,52 @@+import { ApiProperty } from '@nestjs/swagger'+import {+ IsNotEmpty,+ IsIn,+ ValidateIf,+ IsString,+ IsArray,+ Matches,+} from 'class-validator'+import { PID_REGEX } from '../../common/constants'++export class PatchStatusDTO {+ @ApiProperty({+ example: 'aUn1quEid-3',+ required: true,+ description: 'The project ID',+ })+ @IsNotEmpty()+ @Matches(PID_REGEX, {+ message: 'Incorrect project ID format.',+ })+ pid: string++ @ApiProperty({+ example: 'c138c2b1826d9fec65d230e471ab2c25',+ description: 'Error id',+ })+ @ValidateIf(o => !o.eids || o.eid)+ @IsNotEmpty()+ @IsString()+ eid: string++ @ApiProperty({+ example: [+ 'c138c2b1826d9fec65d230e471ab2c25',+ 'b2d12867d59d1d4f9986b52311a97cd5',+ ],+ description: 'Error id list',+ })+ @ValidateIf(o => o.eids || !o.eid)+ @IsNotEmpty()+ @IsArray()+ @IsString({ each: true })+ eids: string++ @ApiProperty({+ enum: ['resolved', 'active'],+ required: true,+ })+ @IsIn(['resolved', 'active'])+ status: 'resolved' | 'active'+}diff --git a/apps/selfhosted/src/common/constants.ts b/apps/selfhosted/src/common/constants.tsindex aa8470201..89220bfc3 100644--- a/apps/selfhosted/src/common/constants.ts+++ b/apps/selfhosted/src/common/constants.ts@@ -78,6 +78,7 @@ const REDIS_LOG_CUSTOM_CACHE_KEY = 'log_custom_cache_v3'const REDIS_SESSION_SALT_KEY = 'log_salt' // is updated every 24 hoursconst REDIS_USERS_COUNT_KEY = 'stats:users_count'const REDIS_PROJECTS_COUNT_KEY = 'stats:projects_count'+const REDIS_LOG_ERROR_CACHE_KEY = 'log_error_cache'// 3600 sec -> 1 hourconst redisProjectCacheTimeout = 3600@@ -108,6 +109,7 @@ const TRAFFIC_COLUMNS = []const PERFORMANCE_COLUMNS = ['cc', 'rg', 'ct', 'pg', 'dv', 'br']+const ERROR_COLUMNS = ['pg', 'dv', 'br', 'os', 'lc', 'cc', 'rg', 'ct']const NUMBER_JWT_REFRESH_TOKEN_LIFETIME = Number(JWT_REFRESH_TOKEN_LIFETIME)const NUMBER_JWT_ACCESS_TOKEN_LIFETIME = Number(JWT_ACCESS_TOKEN_LIFETIME)@@ -143,4 +145,7 @@ export {isProxiedByCloudflare,MIN_PAGES_IN_FUNNEL,MAX_PAGES_IN_FUNNEL,+ ERROR_COLUMNS,+ REDIS_LOG_ERROR_CACHE_KEY,+ PID_REGEX,}diff --git a/apps/selfhosted/src/project/project.controller.ts b/apps/selfhosted/src/project/project.controller.tsindex 4c0ff4cb7..75f86f2ff 100644--- a/apps/selfhosted/src/project/project.controller.ts+++ b/apps/selfhosted/src/project/project.controller.ts@@ -76,11 +76,17 @@ export class ProjectController {_map(formatted, ({ id }) => id),)+ const pidsWithErrorData =+ await this.projectService.getPIDsWhereErrorsDataExists(+ _map(formatted, ({ id }) => id),+ )+const results = _map(formatted, p => ({...p,isOwner: true,isLocked: false,isDataExists: _includes(pidsWithData, p?.id),+ isErrorDataExists: _includes(pidsWithErrorData, p?.id),}))return {@@ -306,9 +312,14 @@ export class ProjectController {await this.projectService.getPIDsWhereAnalyticsDataExists([id]),)+ const isErrorDataExists = !_isEmpty(+ await this.projectService.getPIDsWhereErrorsDataExists([id]),+ )+return this.projectService.formatFromClickhouse({...project,isDataExists,+ isErrorDataExists,})}}diff --git a/apps/selfhosted/src/project/project.service.ts b/apps/selfhosted/src/project/project.service.tsindex 3f078f9c6..e845cffbe 100644--- a/apps/selfhosted/src/project/project.service.ts+++ b/apps/selfhosted/src/project/project.service.ts@@ -156,6 +156,50 @@ export class ProjectService {return _map(result, ({ pid }) => pid)}+ async getPIDsWhereErrorsDataExists(projectIds: string[]) {+ if (_isEmpty(projectIds)) {+ return {}+ }++ const params = _reduce(+ projectIds,+ (acc, curr, index) => ({+ ...acc,+ [`pid_${index}`]: curr,+ }),+ {},+ )++ const pids = _join(+ _map(params, (val, key) => `{${key}:FixedString(12)}`),+ ',',+ )++ const query = `+ SELECT+ pid,+ CASE+ WHEN EXISTS (+ SELECT 1+ FROM errors+ WHERE pid IN (${pids})+ )+ THEN 1+ ELSE 0+ END AS exists+ FROM+ (+ SELECT DISTINCT pid+ FROM errors+ WHERE pid IN (${pids})+ );+ `++ const result: any = await clickhouse.query(query, { params }).toPromise()++ return _map(result, ({ pid }) => pid)+ }+async removeDataFromClickhouse(pid: string,from: string,diff --git a/apps/selfhosted/src/task-manager/task-manager.service.ts b/apps/selfhosted/src/task-manager/task-manager.service.tsindex 18f624f8f..2a19640bc 100644--- a/apps/selfhosted/src/task-manager/task-manager.service.ts+++ b/apps/selfhosted/src/task-manager/task-manager.service.ts@@ -17,6 +17,7 @@ import {REDIS_SESSION_SALT_KEY,REDIS_LOG_PERF_CACHE_KEY,REDIS_LOG_CAPTCHA_CACHE_KEY,+ REDIS_LOG_ERROR_CACHE_KEY,} from '../common/constants'import { AppLoggerService } from '../logger/logger.service'@@ -32,6 +33,7 @@ export class TaskManagerService {const customData = await redis.lrange(REDIS_LOG_CUSTOM_CACHE_KEY, 0, -1)const perfData = await redis.lrange(REDIS_LOG_PERF_CACHE_KEY, 0, -1)const captchaData = await redis.lrange(REDIS_LOG_CAPTCHA_CACHE_KEY, 0, -1)+ const errorData = await redis.lrange(REDIS_LOG_ERROR_CACHE_KEY, 0, -1)if (!_isEmpty(data)) {await redis.del(REDIS_LOG_DATA_CACHE_KEY)@@ -68,6 +70,19 @@ export class TaskManagerService {}}+ if (!_isEmpty(errorData)) {+ await redis.del(REDIS_LOG_ERROR_CACHE_KEY)+ const query = `INSERT INTO errors (*) VALUES ${_join(errorData, ',')}`++ try {+ await clickhouse.query(query).toPromise()+ } catch (e) {+ console.error(+ `[CRON WORKER] Error whilst saving error events data: ${e}`,+ )+ }+ }+if (!_isEmpty(perfData)) {await redis.del(REDIS_LOG_PERF_CACHE_KEY)diff --git a/migrations/clickhouse/2024_03_30.js b/migrations/clickhouse/2024_03_30.jsnew file mode 100644index 000000000..01ea5daeb--- /dev/null+++ b/migrations/clickhouse/2024_03_30.js@@ -0,0 +1,39 @@+const { queriesRunner, dbName } = require('./setup')++const queries = [+ // Error events table+ `CREATE TABLE IF NOT EXISTS ${dbName}.errors+ (+ eid FixedString(32),+ pid FixedString(12),+ pg Nullable(String),+ dv LowCardinality(Nullable(String)),+ br LowCardinality(Nullable(String)),+ os LowCardinality(Nullable(String)),+ lc LowCardinality(Nullable(String)),+ cc LowCardinality(Nullable(FixedString(2))),+ rg LowCardinality(Nullable(String)),+ ct Nullable(String),+ name String,+ message Nullable(String),+ lineno Nullable(UInt32),+ colno Nullable(UInt32),+ filename Nullable(String),+ created DateTime('UTC')+ )+ ENGINE = MergeTree()+ PARTITION BY toYYYYMM(created)+ ORDER BY (pid, created);`,++ // Error events status table+ `CREATE TABLE IF NOT EXISTS ${dbName}.error_statuses (+ eid FixedString(32),+ pid FixedString(12),+ status Enum8('active', 'regressed', 'resolved'),+ updated DateTime('UTC') DEFAULT now()+ )+ ENGINE = ReplacingMergeTree()+ PRIMARY KEY (eid, pid);`,+]++queriesRunner(queries)diff --git a/migrations/clickhouse/initialise_database.js b/migrations/clickhouse/initialise_database.jsindex 4172594a1..c41928733 100644--- a/migrations/clickhouse/initialise_database.js+++ b/migrations/clickhouse/initialise_database.js@@ -1,9 +1,7 @@// This script initialises the Swetrix cloud edition database and tables if they're absentconst { queriesRunner, dbName, databaselessQueriesRunner } = require('./setup')-const CLICKHOUSE_DB_INIT_QUERIES = [- `CREATE DATABASE IF NOT EXISTS ${dbName}`,-]+const CLICKHOUSE_DB_INIT_QUERIES = [`CREATE DATABASE IF NOT EXISTS ${dbName}`]const CLICKHOUSE_INIT_QUERIES = [// The traffic data table@@ -86,6 +84,40 @@ const CLICKHOUSE_INIT_QUERIES = [PARTITION BY toYYYYMM(created)ORDER BY (pid, created);`,+ // Error events table+ `CREATE TABLE IF NOT EXISTS ${dbName}.errors+ (+ eid FixedString(32),+ pid FixedString(12),+ pg Nullable(String),+ dv LowCardinality(Nullable(String)),+ br LowCardinality(Nullable(String)),+ os LowCardinality(Nullable(String)),+ lc LowCardinality(Nullable(String)),+ cc LowCardinality(Nullable(FixedString(2))),+ rg LowCardinality(Nullable(String)),+ ct Nullable(String),+ name String,+ message Nullable(String),+ lineno Nullable(UInt32),+ colno Nullable(UInt32),+ filename Nullable(String),+ created DateTime('UTC')+ )+ ENGINE = MergeTree()+ PARTITION BY toYYYYMM(created)+ ORDER BY (pid, created);`,++ // Error events status table+ `CREATE TABLE IF NOT EXISTS ${dbName}.error_statuses (+ eid FixedString(32),+ pid FixedString(12),+ status Enum8('active', 'regressed', 'resolved'),+ updated DateTime('UTC') DEFAULT now()+ )+ ENGINE = ReplacingMergeTree()+ PRIMARY KEY (eid, pid);`,+// The CAPTCHA data table`CREATE TABLE IF NOT EXISTS ${dbName}.captcha(@@ -99,7 +131,7 @@ const CLICKHOUSE_INIT_QUERIES = [)ENGINE = MergeTree()PARTITION BY toYYYYMM(created)- ORDER BY (pid, created);`+ ORDER BY (pid, created);`,]const initialiseDatabase = async () => {@@ -107,7 +139,9 @@ const initialiseDatabase = async () => {await databaselessQueriesRunner(CLICKHOUSE_DB_INIT_QUERIES)await queriesRunner(CLICKHOUSE_INIT_QUERIES)} catch (reason) {- console.error(`[ERROR] Error occured whilst initialising the database: ${reason}`)+ console.error(+ `[ERROR] Error occured whilst initialising the database: ${reason}`,+ )}}