Skip to content

📈 Open source, privacy-first web analytics — a cookie-less Google Analytics alternative with built-in error tracking and performance monitoring. Self-host it or use our cloud. Get started for free!

Added filters, 'created' checks and previous value support

Blue Moused2ca2a27ecfa
diff --git a/apps/production/src/analytics/analytics.controller.ts b/apps/production/src/analytics/analytics.controller.ts
index a3cd88807..ee8b18ed5 100644
--- a/apps/production/src/analytics/analytics.controller.ts
+++ b/apps/production/src/analytics/analytics.controller.ts
@@ -3,6 +3,7 @@ import * as _isArray from 'lodash/isArray'
import * as _toNumber from 'lodash/toNumber'
import * as _pick from 'lodash/pick'
import * as _includes from 'lodash/includes'
+import * as _size from 'lodash/size'
import * as _map from 'lodash/map'
import * as _uniqBy from 'lodash/uniqBy'
import * as dayjs from 'dayjs'
@@ -25,8 +26,6 @@ import {
ForbiddenException,
Response,
Header,
- ConflictException,
- NotFoundException,
} from '@nestjs/common'
import { ApiBearerAuth, ApiTags } from '@nestjs/swagger'
import * as UAParser from 'ua-parser-js'
@@ -90,6 +89,10 @@ import {
performanceTransformer,
trafficTransformer,
} from './utils/transformers'
+import {
+ MAX_METRICS_IN_VIEW,
+ ProjectViewCustomEventDto,
+} from '../project/dto/create-project-view.dto'
dayjs.extend(utc)
dayjs.extend(dayjsTimezone)
@@ -229,6 +232,18 @@ export class AnalyticsController {
await this.analyticsService.checkBillingAccess(pid)
+ let parsedMetrics: ProjectViewCustomEventDto[]
+
+ if (!isCaptcha) {
+ parsedMetrics = this.analyticsService.parseMetrics(metrics)
+
+ if (_size(parsedMetrics) > MAX_METRICS_IN_VIEW) {
+ throw new BadRequestException(
+ `The maximum number of metrics within one request is ${MAX_METRICS_IN_VIEW}`,
+ )
+ }
+ }
+
let newTimebucket = timeBucket
let allowedTumebucketForPeriodAll
@@ -334,21 +349,19 @@ export class AnalyticsController {
)
}
- let meta: Awaited<ReturnType<typeof this.analyticsService.getMetaResult>>
-
- if (!isCaptcha) {
- const parsedMetrics = this.analyticsService.parseMetrics(metrics)
-
- console.log('parsedMetrics:', parsedMetrics)
+ let meta: Awaited<ReturnType<typeof this.analyticsService.getMetaResults>>
- if (!_isEmpty(parsedMetrics)) {
- meta = await this.analyticsService.getMetaResult(
- pid,
- parsedMetrics,
- filtersQuery,
- paramsData,
- )
- }
+ if (!_isEmpty(parsedMetrics)) {
+ meta = await this.analyticsService.getMetaResults(
+ pid,
+ parsedMetrics,
+ filtersQuery,
+ paramsData,
+ timezone,
+ period,
+ from,
+ to,
+ )
}
return {
diff --git a/apps/production/src/analytics/analytics.service.ts b/apps/production/src/analytics/analytics.service.ts
index 47ef901df..41d6649aa 100644
--- a/apps/production/src/analytics/analytics.service.ts
+++ b/apps/production/src/analytics/analytics.service.ts
@@ -3806,87 +3806,202 @@ export class AnalyticsService {
}
}
- async getMetaResult(
+ async getMetaResults(
pid: string,
customEvents: ProjectViewCustomEventDto[],
filtersQuery: string,
paramsData: any,
+ timezone: string,
+ period?: string,
+ from?: string,
+ to?: string,
) {
- const nonStringCustomEvents = _filter(
- customEvents,
- event =>
- event.metaValueType !== ProjectViewCustomEventMetaValueType.STRING,
+ // eslint-disable-next-line
+ let _from = from
+ // eslint-disable-next-line
+ let _to = to
+
+ if (_isEmpty(period) || ['today', 'yesterday', 'custom'].includes(period)) {
+ const safeTimezone = this.getSafeTimezone(timezone)
+
+ const { groupFrom, groupTo } = this.getGroupFromTo(
+ from,
+ to,
+ ['today', 'yesterday'].includes(period) ? TimeBucketType.HOUR : null,
+ period,
+ safeTimezone,
+ )
+
+ _from = groupFrom
+ _to = groupTo
+ }
+
+ const results = []
+
+ await Promise.all(
+ customEvents.map(async event => {
+ const result = await this.getMetaResult(
+ pid,
+ event,
+ filtersQuery,
+ paramsData,
+ period,
+ _from,
+ _to,
+ )
+ results.push(result)
+ }),
)
- const metaKeys = nonStringCustomEvents.map(event => event.metaKey)
- const params = {
- pid,
- ..._reduce(
- metaKeys,
- (acc, key, index) => ({ ...acc, [`metaKey_${index}`]: key }),
- {},
- ),
- ...paramsData.params,
+ return results.filter(r => !!r)
+ }
+
+ async getMetaResult(
+ pid: string,
+ metric: ProjectViewCustomEventDto,
+ filtersQuery: string,
+ paramsData: any,
+ period?: string,
+ from?: string,
+ to?: string,
+ ) {
+ if (metric.metaValueType === ProjectViewCustomEventMetaValueType.STRING) {
+ return undefined
}
- const casesSum = nonStringCustomEvents
- .map(
- (event, index) => `
- WHEN key = {metaKey_${index}:String} THEN ${
- event.metaValueType === ProjectViewCustomEventMetaValueType.INTEGER
- ? 'toInt32OrZero(value)'
- : 'toFloat32OrZero(value)'
- }
- `,
- )
- .join(' ')
-
- const casesAvg = nonStringCustomEvents
- .map(
- (event, index) => `
- WHEN key = {metaKey_${index}:String} THEN ${
- event.metaValueType === ProjectViewCustomEventMetaValueType.INTEGER
- ? 'toInt32OrZero(value)'
- : 'toFloat32OrZero(value)'
- }
- `,
- )
- .join(' ')
+ const paramsKeys = ['customEventName', 'metaKey', 'metaValue', 'metricKey']
- const metaKeysParams = metaKeys
- .map((_, index) => `{metaKey_${index}:String}`)
- .join(', ')
+ const aggFunction =
+ metric.metaValueType === ProjectViewCustomEventMetaValueType.INTEGER
+ ? 'toInt32OrZero(value)'
+ : 'toFloat32OrZero(value)'
- /*
- query += `indexOf(meta.key, {${keyParam}:String}) > 0 AND meta.value[indexOf(meta.key, {${keyParam}:String})] ${
- isExclusive ? '!= ' : '='
- } {${param}:String}`
- */
+ let kvQuery = ''
- const query = `
+ if (metric.metaKey && metric.metaValue) {
+ kvQuery = `AND (
+ indexOf(meta.key, {metaKey:String}) > 0
+ AND meta.value[indexOf(meta.key, {metaKey:String})] = {metaValue:String}
+ )`
+ } else if (metric.metaKey) {
+ kvQuery = `AND indexOf(meta.key, {metaKey:String}) > 0`
+ } else if (metric.metaValue) {
+ kvQuery = `AND indexOf(meta.value, {metaValue:String}) > 0`
+ }
+
+ const queryCurrent = `
SELECT
+ 1 AS sortOrder,
key,
- sum(CASE ${casesSum} ELSE 0 END) AS sum,
- avg(CASE ${casesAvg} ELSE 0 END) AS avg
+ round(sum(${aggFunction}), 2) AS sum,
+ round(avg(${aggFunction}), 2) AS avg
FROM customEV
ARRAY JOIN meta.key AS key, meta.value AS value
WHERE
pid = {pid:FixedString(12)}
- AND key IN (${metaKeysParams})
- AND created BETWEEN {groupFrom:String} AND {groupTo:String}
+ AND key = {metricKey:String}
+ AND ev = {customEventName:String}
+ AND created BETWEEN {periodFormatted:String} AND {now:String}
+ ${kvQuery}
+ ${filtersQuery}
+ GROUP BY key
+ `
+
+ const queryPrevious = `
+ SELECT
+ 2 AS sortOrder,
+ key,
+ round(sum(${aggFunction}), 2) AS sum,
+ round(avg(${aggFunction}), 2) AS avg
+ FROM customEV
+ ARRAY JOIN meta.key AS key, meta.value AS value
+ WHERE
+ pid = {pid:FixedString(12)}
+ AND key = {metricKey:String}
+ AND ev = {customEventName:String}
+ AND created BETWEEN {periodSubtracted:String} AND {periodFormatted:String}
+ ${kvQuery}
${filtersQuery}
GROUP BY key
`
+ let now
+ let periodFormatted
+ let periodSubtracted
+
+ if (period !== 'all') {
+ if (from && to) {
+ // diff may be 0 (when selecting data for 1 day), so let's make it 1 to grab some data for the prev day as well
+ const diff = dayjs(to).diff(dayjs(from), 'days') || 1
+
+ now = to
+ periodFormatted = from
+ periodSubtracted = dayjs(from)
+ .subtract(diff, 'days')
+ .format('YYYY-MM-DD HH:mm:ss')
+ } else {
+ const amountToSubtract = parseInt(period, 10)
+ const unit = _replace(period, /[0-9]/g, '')
+
+ now = dayjs.utc().format('YYYY-MM-DD HH:mm:ss')
+ const periodRaw = dayjs.utc().subtract(amountToSubtract, unit)
+ periodFormatted = periodRaw.format('YYYY-MM-DD HH:mm:ss')
+ periodSubtracted = periodRaw
+ .subtract(amountToSubtract, unit)
+ .format('YYYY-MM-DD HH:mm:ss')
+ }
+ }
+
+ const params = {
+ pid,
+ ...paramsKeys.reduce(
+ (acc, curr) => ({
+ ...acc,
+ [curr]: metric[curr],
+ }),
+ {},
+ ),
+ now,
+ periodFormatted,
+ periodSubtracted,
+ ...paramsData.params,
+ }
+
try {
- const { data } = await clickhouse
+ let { data } = await clickhouse
.query({
- query,
+ query: `${queryCurrent} UNION ALL ${queryPrevious}`,
query_params: params,
})
- .then(resultSet => resultSet.json())
+ .then(resultSet =>
+ resultSet.json<{
+ sortOrder: 1 | 2
+ key: string
+ sum: number
+ avg: number
+ }>(),
+ )
- return data
+ data = _sortBy(data, 'sortOrder')
+
+ const currentPeriod = data[0]
+ const previousPeriod = data[1]
+
+ if (!currentPeriod) {
+ return undefined
+ }
+
+ return {
+ key: currentPeriod.key,
+ current: {
+ sum: currentPeriod.sum || 0,
+ avg: currentPeriod.avg || 0,
+ },
+ previous: {
+ sum: previousPeriod?.sum || 0,
+ avg: previousPeriod?.avg || 0,
+ },
+ }
} catch (reason) {
console.error('[ERROR] (getMetaResult) - Clickhouse query error:')
console.error(reason)