📈 Open source, privacy-first web analytics — a cookie-less Google Analytics alternative with built-in error tracking and performance monitoring. Self-host it or use our cloud. Get started for free!
TypeScript chartsclickhousedockergdprgoogle-analyticsmetricsmonitoringmysqlnestjsprivacyredisself-hostedselfhostedstatisticstailwindcssweb-analyticswebanalyticswebsite
Added filters, 'created' checks and previous value support
Blue Moused2ca2a27ecfa
diff --git a/apps/production/src/analytics/analytics.controller.ts b/apps/production/src/analytics/analytics.controller.tsindex a3cd88807..ee8b18ed5 100644--- a/apps/production/src/analytics/analytics.controller.ts+++ b/apps/production/src/analytics/analytics.controller.ts@@ -3,6 +3,7 @@ import * as _isArray from 'lodash/isArray'import * as _toNumber from 'lodash/toNumber'import * as _pick from 'lodash/pick'import * as _includes from 'lodash/includes'+import * as _size from 'lodash/size'import * as _map from 'lodash/map'import * as _uniqBy from 'lodash/uniqBy'import * as dayjs from 'dayjs'@@ -25,8 +26,6 @@ import {ForbiddenException,Response,Header,- ConflictException,- NotFoundException,} from '@nestjs/common'import { ApiBearerAuth, ApiTags } from '@nestjs/swagger'import * as UAParser from 'ua-parser-js'@@ -90,6 +89,10 @@ import {performanceTransformer,trafficTransformer,} from './utils/transformers'+import {+ MAX_METRICS_IN_VIEW,+ ProjectViewCustomEventDto,+} from '../project/dto/create-project-view.dto'dayjs.extend(utc)dayjs.extend(dayjsTimezone)@@ -229,6 +232,18 @@ export class AnalyticsController {await this.analyticsService.checkBillingAccess(pid)+ let parsedMetrics: ProjectViewCustomEventDto[]++ if (!isCaptcha) {+ parsedMetrics = this.analyticsService.parseMetrics(metrics)++ if (_size(parsedMetrics) > MAX_METRICS_IN_VIEW) {+ throw new BadRequestException(+ `The maximum number of metrics within one request is ${MAX_METRICS_IN_VIEW}`,+ )+ }+ }+let newTimebucket = timeBucketlet allowedTumebucketForPeriodAll@@ -334,21 +349,19 @@ export class AnalyticsController {)}- let meta: Awaited<ReturnType<typeof this.analyticsService.getMetaResult>>-- if (!isCaptcha) {- const parsedMetrics = this.analyticsService.parseMetrics(metrics)-- console.log('parsedMetrics:', parsedMetrics)+ let meta: Awaited<ReturnType<typeof this.analyticsService.getMetaResults>>- if (!_isEmpty(parsedMetrics)) {- meta = await this.analyticsService.getMetaResult(- pid,- parsedMetrics,- filtersQuery,- paramsData,- )- }+ if (!_isEmpty(parsedMetrics)) {+ meta = await this.analyticsService.getMetaResults(+ pid,+ parsedMetrics,+ filtersQuery,+ paramsData,+ timezone,+ period,+ from,+ to,+ )}return {diff --git a/apps/production/src/analytics/analytics.service.ts b/apps/production/src/analytics/analytics.service.tsindex 47ef901df..41d6649aa 100644--- a/apps/production/src/analytics/analytics.service.ts+++ b/apps/production/src/analytics/analytics.service.ts@@ -3806,87 +3806,202 @@ export class AnalyticsService {}}- async getMetaResult(+ async getMetaResults(pid: string,customEvents: ProjectViewCustomEventDto[],filtersQuery: string,paramsData: any,+ timezone: string,+ period?: string,+ from?: string,+ to?: string,) {- const nonStringCustomEvents = _filter(- customEvents,- event =>- event.metaValueType !== ProjectViewCustomEventMetaValueType.STRING,+ // eslint-disable-next-line+ let _from = from+ // eslint-disable-next-line+ let _to = to++ if (_isEmpty(period) || ['today', 'yesterday', 'custom'].includes(period)) {+ const safeTimezone = this.getSafeTimezone(timezone)++ const { groupFrom, groupTo } = this.getGroupFromTo(+ from,+ to,+ ['today', 'yesterday'].includes(period) ? TimeBucketType.HOUR : null,+ period,+ safeTimezone,+ )++ _from = groupFrom+ _to = groupTo+ }++ const results = []++ await Promise.all(+ customEvents.map(async event => {+ const result = await this.getMetaResult(+ pid,+ event,+ filtersQuery,+ paramsData,+ period,+ _from,+ _to,+ )+ results.push(result)+ }),)- const metaKeys = nonStringCustomEvents.map(event => event.metaKey)- const params = {- pid,- ..._reduce(- metaKeys,- (acc, key, index) => ({ ...acc, [`metaKey_${index}`]: key }),- {},- ),- ...paramsData.params,+ return results.filter(r => !!r)+ }++ async getMetaResult(+ pid: string,+ metric: ProjectViewCustomEventDto,+ filtersQuery: string,+ paramsData: any,+ period?: string,+ from?: string,+ to?: string,+ ) {+ if (metric.metaValueType === ProjectViewCustomEventMetaValueType.STRING) {+ return undefined}- const casesSum = nonStringCustomEvents- .map(- (event, index) => `- WHEN key = {metaKey_${index}:String} THEN ${- event.metaValueType === ProjectViewCustomEventMetaValueType.INTEGER- ? 'toInt32OrZero(value)'- : 'toFloat32OrZero(value)'- }- `,- )- .join(' ')-- const casesAvg = nonStringCustomEvents- .map(- (event, index) => `- WHEN key = {metaKey_${index}:String} THEN ${- event.metaValueType === ProjectViewCustomEventMetaValueType.INTEGER- ? 'toInt32OrZero(value)'- : 'toFloat32OrZero(value)'- }- `,- )- .join(' ')+ const paramsKeys = ['customEventName', 'metaKey', 'metaValue', 'metricKey']- const metaKeysParams = metaKeys- .map((_, index) => `{metaKey_${index}:String}`)- .join(', ')+ const aggFunction =+ metric.metaValueType === ProjectViewCustomEventMetaValueType.INTEGER+ ? 'toInt32OrZero(value)'+ : 'toFloat32OrZero(value)'- /*- query += `indexOf(meta.key, {${keyParam}:String}) > 0 AND meta.value[indexOf(meta.key, {${keyParam}:String})] ${- isExclusive ? '!= ' : '='- } {${param}:String}`- */+ let kvQuery = ''- const query = `+ if (metric.metaKey && metric.metaValue) {+ kvQuery = `AND (+ indexOf(meta.key, {metaKey:String}) > 0+ AND meta.value[indexOf(meta.key, {metaKey:String})] = {metaValue:String}+ )`+ } else if (metric.metaKey) {+ kvQuery = `AND indexOf(meta.key, {metaKey:String}) > 0`+ } else if (metric.metaValue) {+ kvQuery = `AND indexOf(meta.value, {metaValue:String}) > 0`+ }++ const queryCurrent = `SELECT+ 1 AS sortOrder,key,- sum(CASE ${casesSum} ELSE 0 END) AS sum,- avg(CASE ${casesAvg} ELSE 0 END) AS avg+ round(sum(${aggFunction}), 2) AS sum,+ round(avg(${aggFunction}), 2) AS avgFROM customEVARRAY JOIN meta.key AS key, meta.value AS valueWHEREpid = {pid:FixedString(12)}- AND key IN (${metaKeysParams})- AND created BETWEEN {groupFrom:String} AND {groupTo:String}+ AND key = {metricKey:String}+ AND ev = {customEventName:String}+ AND created BETWEEN {periodFormatted:String} AND {now:String}+ ${kvQuery}+ ${filtersQuery}+ GROUP BY key+ `++ const queryPrevious = `+ SELECT+ 2 AS sortOrder,+ key,+ round(sum(${aggFunction}), 2) AS sum,+ round(avg(${aggFunction}), 2) AS avg+ FROM customEV+ ARRAY JOIN meta.key AS key, meta.value AS value+ WHERE+ pid = {pid:FixedString(12)}+ AND key = {metricKey:String}+ AND ev = {customEventName:String}+ AND created BETWEEN {periodSubtracted:String} AND {periodFormatted:String}+ ${kvQuery}${filtersQuery}GROUP BY key`+ let now+ let periodFormatted+ let periodSubtracted++ if (period !== 'all') {+ if (from && to) {+ // diff may be 0 (when selecting data for 1 day), so let's make it 1 to grab some data for the prev day as well+ const diff = dayjs(to).diff(dayjs(from), 'days') || 1++ now = to+ periodFormatted = from+ periodSubtracted = dayjs(from)+ .subtract(diff, 'days')+ .format('YYYY-MM-DD HH:mm:ss')+ } else {+ const amountToSubtract = parseInt(period, 10)+ const unit = _replace(period, /[0-9]/g, '')++ now = dayjs.utc().format('YYYY-MM-DD HH:mm:ss')+ const periodRaw = dayjs.utc().subtract(amountToSubtract, unit)+ periodFormatted = periodRaw.format('YYYY-MM-DD HH:mm:ss')+ periodSubtracted = periodRaw+ .subtract(amountToSubtract, unit)+ .format('YYYY-MM-DD HH:mm:ss')+ }+ }++ const params = {+ pid,+ ...paramsKeys.reduce(+ (acc, curr) => ({+ ...acc,+ [curr]: metric[curr],+ }),+ {},+ ),+ now,+ periodFormatted,+ periodSubtracted,+ ...paramsData.params,+ }+try {- const { data } = await clickhouse+ let { data } = await clickhouse.query({- query,+ query: `${queryCurrent} UNION ALL ${queryPrevious}`,query_params: params,})- .then(resultSet => resultSet.json())+ .then(resultSet =>+ resultSet.json<{+ sortOrder: 1 | 2+ key: string+ sum: number+ avg: number+ }>(),+ )- return data+ data = _sortBy(data, 'sortOrder')++ const currentPeriod = data[0]+ const previousPeriod = data[1]++ if (!currentPeriod) {+ return undefined+ }++ return {+ key: currentPeriod.key,+ current: {+ sum: currentPeriod.sum || 0,+ avg: currentPeriod.avg || 0,+ },+ previous: {+ sum: previousPeriod?.sum || 0,+ avg: previousPeriod?.avg || 0,+ },+ }} catch (reason) {console.error('[ERROR] (getMetaResult) - Clickhouse query error:')console.error(reason)