📈 Open source, privacy-first web analytics — a cookie-less Google Analytics alternative with built-in error tracking and performance monitoring. Self-host it or use our cloud. Get started for free!
TypeScript chartsclickhousedockergdprgoogle-analyticsmetricsmonitoringmysqlnestjsprivacyredisself-hostedselfhostedstatisticstailwindcssweb-analyticswebanalyticswebsite
Unify analytics tables into a single 'events' table (#526)
Andrii Romasiun286428ec730c
diff --git a/admin/src/db/clickhouse.ts b/admin/src/db/clickhouse.tsindex 2e4f960b3..3b76d08eb 100644--- a/admin/src/db/clickhouse.ts+++ b/admin/src/db/clickhouse.ts@@ -34,6 +34,18 @@ export interface ClickHouseStats {tables: TableStats[];}+const EVENT_TABLE = "events";+const PROJECT_ACTIVITY_EVENT_TYPES = [+ "pageview",+ "custom_event",+ "error",+ "captcha",+ "performance",+];+const PROJECT_ACTIVITY_TYPE_LIST = PROJECT_ACTIVITY_EVENT_TYPES.map(+ (type) => `'${type}'`+).join(", ");+function formatBytes(bytes: number): string {if (bytes === 0) return "0 B";const k = 1024;@@ -47,20 +59,16 @@ export async function getClickHouseStats(): Promise<ClickHouseStats> {const database = process.env.CLICKHOUSE_DATABASE || "analytics";const tables = [- "analytics",- "customEV",- "performance",- "errors",- "captcha",+ EVENT_TABLE,"sessions","revenue","feature_flag_evaluations","experiment_exposures","error_statuses",+ "bot_blocks",];const tableStats: TableStats[] = [];- let totalEvents = 0;for (const table of tables) {try {@@ -93,8 +101,6 @@ export async function getClickHouseStats(): Promise<ClickHouseStats> {bytes,bytesFormatted: formatBytes(bytes),});-- totalEvents += rows;} catch {// Table might not exist, skip ittableStats.push({@@ -106,6 +112,9 @@ export async function getClickHouseStats(): Promise<ClickHouseStats> {}}+ const totalEvents =+ tableStats.find((table) => table.table === EVENT_TABLE)?.rows || 0;+return {totalEvents,tables: tableStats,@@ -118,7 +127,12 @@ export async function getProjectEventCount(pid: string): Promise<number> {try {const result = await client.query({- query: `SELECT count() as count FROM ${database}.analytics WHERE pid = {pid:String}`,+ query: `+ SELECT count() as count+ FROM ${database}.${EVENT_TABLE}+ WHERE pid = {pid:FixedString(12)}+ AND type IN (${PROJECT_ACTIVITY_TYPE_LIST})+ `,query_params: { pid },format: "JSONEachRow",});@@ -158,8 +172,9 @@ export async function getTopProjectsByEvents(SELECTpid,count() as eventCount- FROM ${database}.analytics+ FROM ${database}.${EVENT_TABLE}WHERE created >= now() - INTERVAL ${days} DAY+ AND type IN (${PROJECT_ACTIVITY_TYPE_LIST})GROUP BY pidORDER BY eventCount DESCLIMIT ${limit}@@ -183,8 +198,9 @@ export async function getProjectsWithRecentEvents(const result = await client.query({query: `SELECT DISTINCT pid- FROM ${database}.analytics+ FROM ${database}.${EVENT_TABLE}WHERE created >= now() - INTERVAL ${days} DAY+ AND type IN (${PROJECT_ACTIVITY_TYPE_LIST})`,format: "JSONEachRow",});diff --git a/backend/apps/cloud/src/ai/ai.service.ts b/backend/apps/cloud/src/ai/ai.service.tsindex 8671e63fd..5414f6c3c 100644--- a/backend/apps/cloud/src/ai/ai.service.ts+++ b/backend/apps/cloud/src/ai/ai.service.ts@@ -1218,8 +1218,9 @@ Filter modifiers:SELECTcount(*) as pageviews,count(DISTINCT psid) as sessions- FROM analytics+ FROM eventsWHERE pid = {pid:FixedString(12)}+ AND type = 'pageview'AND created BETWEEN {groupFrom:String} AND {groupTo:String}${filterConditions.where}`@@ -1242,8 +1243,9 @@ Filter modifiers:${this.getTimeBucketSelect(timeBucket, timezone)} as date,count(*) as pageviews,count(DISTINCT psid) as sessions- FROM analytics+ FROM eventsWHERE pid = {pid:FixedString(12)}+ AND type = 'pageview'AND created BETWEEN {groupFrom:String} AND {groupTo:String}${filterConditions.where}GROUP BY date@@ -1266,8 +1268,9 @@ Filter modifiers:// Get top pagesconst pagesQuery = `SELECT pg as name, count(*) as count- FROM analytics+ FROM eventsWHERE pid = {pid:FixedString(12)}+ AND type = 'pageview'AND created BETWEEN {groupFrom:String} AND {groupTo:String}${filterConditions.where}GROUP BY pg@@ -1285,8 +1288,9 @@ Filter modifiers:// Get top countriesconst countriesQuery = `SELECT cc as name, uniqExact(psid) as count- FROM analytics+ FROM eventsWHERE pid = {pid:FixedString(12)}+ AND type = 'pageview'AND created BETWEEN {groupFrom:String} AND {groupTo:String}AND cc IS NOT NULL AND cc != ''${filterConditions.where}@@ -1305,8 +1309,9 @@ Filter modifiers:// Get top referrersconst referrersQuery = `SELECT ref as name, uniqExact(psid) as count- FROM analytics+ FROM eventsWHERE pid = {pid:FixedString(12)}+ AND type = 'pageview'AND created BETWEEN {groupFrom:String} AND {groupTo:String}AND ref IS NOT NULL AND ref != ''${filterConditions.where}@@ -1325,8 +1330,9 @@ Filter modifiers:// Get browsersconst browsersQuery = `SELECT br as name, uniqExact(psid) as count- FROM analytics+ FROM eventsWHERE pid = {pid:FixedString(12)}+ AND type = 'pageview'AND created BETWEEN {groupFrom:String} AND {groupTo:String}AND br IS NOT NULL AND br != ''${filterConditions.where}@@ -1345,8 +1351,9 @@ Filter modifiers:// Get devicesconst devicesQuery = `SELECT dv as name, uniqExact(psid) as count- FROM analytics+ FROM eventsWHERE pid = {pid:FixedString(12)}+ AND type = 'pageview'AND created BETWEEN {groupFrom:String} AND {groupTo:String}AND dv IS NOT NULL AND dv != ''${filterConditions.where}@@ -1416,8 +1423,9 @@ Filter modifiers:round(${measureFn}(render) / 1000, 2) as render,round(${measureFn}(domLoad) / 1000, 2) as domLoad,round(${measureFn}(ttfb) / 1000, 2) as ttfb- FROM performance+ FROM eventsWHERE pid = {pid:FixedString(12)}+ AND type = 'performance'AND created BETWEEN {groupFrom:String} AND {groupTo:String}${filterConditions.where}`@@ -1440,8 +1448,9 @@ Filter modifiers:${this.getTimeBucketSelect(timeBucket, timezone)} as date,round(${measureFn}(pageLoad) / 1000, 2) as pageLoad,round(${measureFn}(ttfb) / 1000, 2) as ttfb- FROM performance+ FROM eventsWHERE pid = {pid:FixedString(12)}+ AND type = 'performance'AND created BETWEEN {groupFrom:String} AND {groupTo:String}${filterConditions.where}GROUP BY date@@ -1497,8 +1506,9 @@ Filter modifiers:SELECTcount(*) as totalErrors,uniqExact(eid) as uniqueErrors- FROM errors+ FROM eventsWHERE pid = {pid:FixedString(12)}+ AND type = 'error'AND created BETWEEN {groupFrom:String} AND {groupTo:String}${filterConditions.where}`@@ -1512,15 +1522,16 @@ Filter modifiers:const topErrorsQuery = `SELECT- name,- message,+ error_name AS name,+ error_message AS message,count(*) as count,max(created) as lastSeen- FROM errors+ FROM eventsWHERE pid = {pid:FixedString(12)}+ AND type = 'error'AND created BETWEEN {groupFrom:String} AND {groupTo:String}${filterConditions.where}- GROUP BY name, message+ GROUP BY error_name, error_messageORDER BY count DESCLIMIT 10`@@ -1571,22 +1582,24 @@ Filter modifiers:}// Query specific goal stats- const table = goal.type === 'custom_event' ? 'customEV' : 'analytics'+ const goalType =+ goal.type === 'custom_event' ? 'custom_event' : 'pageview'const matchCondition =goal.matchType === 'exact'? goal.type === 'custom_event'- ? `ev = {goalValue:String}`+ ? `event_name = {goalValue:String}`: `pg = {goalValue:String}`: goal.type === 'custom_event'- ? `ev ILIKE concat('%', {goalValue:String}, '%')`+ ? `event_name ILIKE concat('%', {goalValue:String}, '%')`: `pg ILIKE concat('%', {goalValue:String}, '%')`const query = `SELECTcount(*) as conversions,uniqExact(psid) as uniqueSessions- FROM ${table}+ FROM eventsWHERE pid = {pid:FixedString(12)}+ AND type = {goalType:String}AND ${matchCondition}AND created BETWEEN {groupFrom:String} AND {groupTo:String}`@@ -1596,6 +1609,7 @@ Filter modifiers:query,query_params: {pid,+ goalType,goalValue: goal.value || '',groupFrom: groupFromUTC,groupTo: groupToUTC,@@ -1617,20 +1631,22 @@ Filter modifiers:const results = await Promise.all(goals.map(async (goal) => {- const table = goal.type === 'custom_event' ? 'customEV' : 'analytics'+ const goalType =+ goal.type === 'custom_event' ? 'custom_event' : 'pageview'const matchCondition =goal.matchType === 'exact'? goal.type === 'custom_event'- ? `ev = {goalValue:String}`+ ? `event_name = {goalValue:String}`: `pg = {goalValue:String}`: goal.type === 'custom_event'- ? `ev ILIKE concat('%', {goalValue:String}, '%')`+ ? `event_name ILIKE concat('%', {goalValue:String}, '%')`: `pg ILIKE concat('%', {goalValue:String}, '%')`const query = `SELECT count(*) as conversions- FROM ${table}+ FROM eventsWHERE pid = {pid:FixedString(12)}+ AND type = {goalType:String}AND ${matchCondition}AND created BETWEEN {groupFrom:String} AND {groupTo:String}`@@ -1640,6 +1656,7 @@ Filter modifiers:query,query_params: {pid,+ goalType,goalValue: goal.value || '',groupFrom: groupFromUTC,groupTo: groupToUTC,@@ -1747,8 +1764,9 @@ Filter modifiers:const overallQuery = `SELECTcount(*) as total- FROM captcha+ FROM eventsWHERE pid = {pid:FixedString(12)}+ AND type = 'captcha'AND created BETWEEN {groupFrom:String} AND {groupTo:String}${filterConditions.where}`@@ -1757,8 +1775,9 @@ Filter modifiers:SELECT${this.getTimeBucketSelect(timeBucket, timezone)} as date,count(*) as challenges- FROM captcha+ FROM eventsWHERE pid = {pid:FixedString(12)}+ AND type = 'captcha'AND created BETWEEN {groupFrom:String} AND {groupTo:String}${filterConditions.where}GROUP BY date@@ -1767,8 +1786,9 @@ Filter modifiers:const breakdownQuery = (column: string) => `SELECT ${column} as name, count(*) as count- FROM captcha+ FROM eventsWHERE pid = {pid:FixedString(12)}+ AND type = 'captcha'AND created BETWEEN {groupFrom:String} AND {groupTo:String}AND ${column} IS NOT NULL AND ${column} != ''${filterConditions.where}@@ -1844,21 +1864,23 @@ Filter modifiers:const overallQuery = `SELECTcount(*) as totalEvents,- uniqExact(ev) as uniqueEvents,+ uniqExact(event_name) as uniqueEvents,uniqExact(psid) as sessions- FROM customEV+ FROM eventsWHERE pid = {pid:FixedString(12)}+ AND type = 'custom_event'AND created BETWEEN {groupFrom:String} AND {groupTo:String}${filterConditions.where}`const topEventsQuery = `- SELECT ev as name, count(*) as count, uniqExact(psid) as sessions- FROM customEV+ SELECT event_name AS name, count(*) as count, uniqExact(psid) as sessions+ FROM eventsWHERE pid = {pid:FixedString(12)}+ AND type = 'custom_event'AND created BETWEEN {groupFrom:String} AND {groupTo:String}${filterConditions.where}- GROUP BY ev+ GROUP BY event_nameORDER BY count DESCLIMIT 25`@@ -1868,8 +1890,9 @@ Filter modifiers:${this.getTimeBucketSelect(timeBucket, timezone)} as date,count(*) as events,uniqExact(psid) as sessions- FROM customEV+ FROM eventsWHERE pid = {pid:FixedString(12)}+ AND type = 'custom_event'AND created BETWEEN {groupFrom:String} AND {groupTo:String}${filterConditions.where}GROUP BY date@@ -2137,10 +2160,10 @@ Filter modifiers:let conversions: { variantKey: string; conversions: number }[] = []if (experiment.goal) {- const table =- experiment.goal.type === 'custom_event' ? 'customEV' : 'analytics'+ const eventType =+ experiment.goal.type === 'custom_event' ? 'custom_event' : 'pageview'const matchColumn =- experiment.goal.type === 'custom_event' ? 'ev' : 'pg'+ experiment.goal.type === 'custom_event' ? 'event_name' : 'pg'const matchCondition =experiment.goal.matchType === 'exact'? `c.${matchColumn} = {goalValue:String}`@@ -2149,7 +2172,7 @@ Filter modifiers:const conversionsQuery = `SELECT e.variantKey, uniqExact(e.profileId) as conversionsFROM experiment_exposures e- INNER JOIN ${table} c ON e.pid = c.pid AND e.profileId = assumeNotNull(c.profileId)+ INNER JOIN events c ON e.pid = c.pid AND e.profileId = assumeNotNull(c.profileId) AND c.type = '${eventType}'WHERE e.pid = {pid:FixedString(12)}AND e.experimentId = {experimentId:String}AND e.created BETWEEN {groupFrom:String} AND {groupTo:String}@@ -2291,8 +2314,9 @@ Filter modifiers:min(created) as startedAt,max(created) as endedAt,count(*) as pageviews- FROM analytics+ FROM eventsWHERE pid = {pid:FixedString(12)}+ AND type = 'pageview'AND created BETWEEN {groupFrom:String} AND {groupTo:String}AND psid IS NOT NULL${extraWhere.join(' ')}@@ -2356,8 +2380,9 @@ Filter modifiers:uniqExactIf(profileId, profileId LIKE 'usr_%') as identifiedProfiles,uniqExact(psid) as sessions,count(*) as pageviews- FROM analytics+ FROM eventsWHERE pid = {pid:FixedString(12)}+ AND type = 'pageview'AND created BETWEEN {groupFrom:String} AND {groupTo:String}AND profileId IS NOT NULL`@@ -2368,8 +2393,9 @@ Filter modifiers:uniqExact(psid) as sessions,count(*) as pageviews,max(created) as lastSeen- FROM analytics+ FROM eventsWHERE pid = {pid:FixedString(12)}+ AND type = 'pageview'AND created BETWEEN {groupFrom:String} AND {groupTo:String}AND profileId IS NOT NULLGROUP BY profileId@@ -2379,8 +2405,9 @@ Filter modifiers:const topPagesQuery = `SELECT pg as name, uniqExact(profileId) as profiles- FROM analytics+ FROM eventsWHERE pid = {pid:FixedString(12)}+ AND type = 'pageview'AND created BETWEEN {groupFrom:String} AND {groupTo:String}AND profileId IS NOT NULLAND pg IS NOT NULL AND pg != ''diff --git a/backend/apps/cloud/src/analytics/analytics.controller.ts b/backend/apps/cloud/src/analytics/analytics.controller.tsindex 8fa6767c0..4f3063996 100644--- a/backend/apps/cloud/src/analytics/analytics.controller.ts+++ b/backend/apps/cloud/src/analytics/analytics.controller.ts@@ -81,12 +81,7 @@ import {GetErrorOverviewOptions,} from './dto/get-error-overview.dto'import { PatchStatusDto } from './dto/patch-status.dto'-import {- customEventTransformer,- errorEventTransformer,- performanceTransformer,- trafficTransformer,-} from './utils/transformers'+import { eventTransformer } from './utils/transformers'import { enrichTrafficSource } from './utils/clickIdSources'import {MAX_METRICS_IN_VIEW,@@ -263,10 +258,19 @@ export class AnalyticsController {let diff+ const [filtersQuery, filtersParams, appliedFilters, customEVFilterApplied] =+ this.analyticsService.getFiltersQuery(+ filters,+ isCaptcha ? DataType.CAPTCHA : DataType.ANALYTICS,+ )+if (period === 'all') {+ const eventTypes = isCaptcha+ ? 'captcha'+ : (['pageview', 'custom_event', 'error'] as const)const res = await this.analyticsService.calculateTimeBucketForAllTime(pid,- 'analytics',+ eventTypes,)diff = res.diff@@ -277,12 +281,6 @@ export class AnalyticsController {allowedTumebucketForPeriodAll = res.timeBucket}- const [filtersQuery, filtersParams, appliedFilters, customEVFilterApplied] =- this.analyticsService.getFiltersQuery(- filters,- isCaptcha ? DataType.CAPTCHA : DataType.ANALYTICS,- )-const safeTimezone = this.analyticsService.getSafeTimezone(timezone)const { groupFrom, groupTo, groupFromUTC, groupToUTC } =this.analyticsService.getGroupFromTo(@@ -294,13 +292,11 @@ export class AnalyticsController {diff,)- let subQuery = `FROM ${- isCaptcha ? 'captcha' : 'analytics'- } WHERE pid = {pid:FixedString(12)} ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String}`-- if (customEVFilterApplied && !isCaptcha) {- subQuery = `FROM customEV WHERE pid = {pid:FixedString(12)} ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String}`- }+ const subQuery = this.analyticsService.buildAnalyticsEventsSubQuery(+ filtersQuery,+ customEVFilterApplied,+ isCaptcha,+ )const paramsData = {params: {@@ -422,12 +418,15 @@ export class AnalyticsController {let diffif (period === 'all') {- const [analyticsRes, customEVRes] = await Promise.all([- this.analyticsService.calculateTimeBucketForAllTime(pid, 'analytics'),- this.analyticsService.calculateTimeBucketForAllTime(pid, 'customEV'),+ const [pageviewRes, customEventRes] = await Promise.all([+ this.analyticsService.calculateTimeBucketForAllTime(pid, 'pageview'),+ this.analyticsService.calculateTimeBucketForAllTime(+ pid,+ 'custom_event',+ ),])- diff = Math.max(analyticsRes.diff, customEVRes.diff)+ diff = Math.max(pageviewRes.diff, customEventRes.diff)}const safeTimezone = this.analyticsService.getSafeTimezone(timezone)@@ -537,12 +536,15 @@ export class AnalyticsController {let diffif (period === 'all') {- const [analyticsRes, customEVRes] = await Promise.all([- this.analyticsService.calculateTimeBucketForAllTime(pid, 'analytics'),- this.analyticsService.calculateTimeBucketForAllTime(pid, 'customEV'),+ const [pageviewRes, customEventRes] = await Promise.all([+ this.analyticsService.calculateTimeBucketForAllTime(pid, 'pageview'),+ this.analyticsService.calculateTimeBucketForAllTime(+ pid,+ 'custom_event',+ ),])- diff = Math.max(analyticsRes.diff, customEVRes.diff)+ diff = Math.max(pageviewRes.diff, customEventRes.diff)}const safeTimezone = this.analyticsService.getSafeTimezone(timezone)@@ -707,13 +709,14 @@ export class AnalyticsController {this.analyticsService.getFiltersQuery(filters, DataType.ANALYTICS)const safeTimezone = this.analyticsService.getSafeTimezone(timezone)- const { groupFrom, groupTo } = this.analyticsService.getGroupFromTo(- from,- to,- timeBucket,- period,- safeTimezone,- )+ const { groupFrom, groupTo, groupFromUTC, groupToUTC } =+ this.analyticsService.getGroupFromTo(+ from,+ to,+ timeBucket,+ period,+ safeTimezone,+ )await this.analyticsService.checkProjectAccess(pid,uid,@@ -724,7 +727,14 @@ export class AnalyticsController {this.logger.log(`pid: ${pid}, period: ${period}`, 'GET /analytics/chart')- const paramsData = { params: { pid, groupFrom, groupTo, ...filtersParams } }+ const paramsData = {+ params: {+ pid,+ groupFrom: groupFromUTC,+ groupTo: groupToUTC,+ ...filtersParams,+ },+ }const result = await this.analyticsService.groupChartByTimeBucket(timeBucket,@@ -795,18 +805,26 @@ export class AnalyticsController {this.analyticsService.getFiltersQuery(filters, DataType.PERFORMANCE, true)const safeTimezone = this.analyticsService.getSafeTimezone(timezone)- const { groupFrom, groupTo } = this.analyticsService.getGroupFromTo(- from,- to,- newTimeBucket,- period,- safeTimezone,- diff,- )+ const { groupFrom, groupTo, groupFromUTC, groupToUTC } =+ this.analyticsService.getGroupFromTo(+ from,+ to,+ newTimeBucket,+ period,+ safeTimezone,+ diff,+ )- const subQuery = `FROM performance WHERE pid = {pid:FixedString(12)} ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String}`+ const subQuery = `FROM events WHERE pid = {pid:FixedString(12)} AND type = 'performance' ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String}`- const paramsData = { params: { pid, groupFrom, groupTo, ...filtersParams } }+ const paramsData = {+ params: {+ pid,+ groupFrom: groupFromUTC,+ groupTo: groupToUTC,+ ...filtersParams,+ },+ }const result = await this.analyticsService.groupPerfByTimeBucket(newTimeBucket,@@ -850,13 +868,14 @@ export class AnalyticsController {this.analyticsService.getFiltersQuery(filters, DataType.PERFORMANCE, true)const safeTimezone = this.analyticsService.getSafeTimezone(timezone)- const { groupFrom, groupTo } = this.analyticsService.getGroupFromTo(- from,- to,- timeBucket,- period,- safeTimezone,- )+ const { groupFrom, groupTo, groupFromUTC, groupToUTC } =+ this.analyticsService.getGroupFromTo(+ from,+ to,+ timeBucket,+ period,+ safeTimezone,+ )await this.analyticsService.checkProjectAccess(pid,uid,@@ -870,12 +889,19 @@ export class AnalyticsController {'GET /analytics/performance/chart',)- const paramsData = { params: { pid, groupFrom, groupTo, ...filtersParams } }+ const paramsData = {+ params: {+ pid,+ groupFrom: groupFromUTC,+ groupTo: groupToUTC,+ ...filtersParams,+ },+ }const chart = await this.analyticsService.getPerfChartData(timeBucket,- from,- to,+ groupFrom,+ groupTo,filtersQuery,paramsData,safeTimezone,@@ -921,7 +947,7 @@ export class AnalyticsController {if (period === 'all') {const res = await this.analyticsService.calculateTimeBucketForAllTime(pid,- 'analytics',+ 'pageview',)diff = res.diff@@ -1332,32 +1358,12 @@ export class AnalyticsController {any(os) AS os,any(cc) AS cc,toString(psid) AS psid- FROM- (- SELECT- psid,- dv,- br,- os,- cc- FROM analytics- WHERE- pid = {pid:FixedString(12)}- AND created >= {since:DateTime}- AND psid IS NOT NULL- UNION ALL- SELECT- psid,- dv,- br,- os,- cc- FROM customEV- WHERE- pid = {pid:FixedString(12)}- AND created >= {since:DateTime}- AND psid IS NOT NULL- )+ FROM events+ WHERE+ pid = {pid:FixedString(12)}+ AND type IN ('pageview', 'custom_event')+ AND created >= {since:DateTime}+ AND psid IS NOT NULLGROUP BY psid`@@ -1464,7 +1470,8 @@ export class AnalyticsController {const { name, message, lineno, colno, filename, stackTrace, meta } =errorDTO- const transformed = errorEventTransformer({+ const transformed = eventTransformer({+ type: 'error',psid,profileId,eid: this.analyticsService.getErrorID(errorDTO),@@ -1496,7 +1503,7 @@ export class AnalyticsController {try {await clickhouse.insert({- table: 'errors',+ table: 'events',format: 'JSONEachRow',values: [transformed],clickhouse_settings: { async_insert: 1 },@@ -1618,7 +1625,8 @@ export class AnalyticsController {enrichTrafficSource(eventsDTO)- const transformed = customEventTransformer({+ const transformed = eventTransformer({+ type: 'custom_event',psid,profileId,pid: eventsDTO.pid,@@ -1650,7 +1658,7 @@ export class AnalyticsController {try {await clickhouse.insert({- table: 'customEV',+ table: 'events',format: 'JSONEachRow',values: [transformed],clickhouse_settings: { async_insert: 1 },@@ -1789,7 +1797,8 @@ export class AnalyticsController {enrichTrafficSource(logDTO)- const transformed = trafficTransformer({+ const transformed = eventTransformer({+ type: 'pageview',psid,profileId,pid: logDTO.pid,@@ -1832,7 +1841,8 @@ export class AnalyticsController {ttfb,} = logDTO.perf- perfTransformed = performanceTransformer({+ perfTransformed = eventTransformer({+ type: 'performance',pid: logDTO.pid,host: this.analyticsService.getHostFromOrigin(headers.origin),pg: logDTO.pg,@@ -1860,7 +1870,7 @@ export class AnalyticsController {try {await clickhouse.insert({- table: 'analytics',+ table: 'events',format: 'JSONEachRow',values: [transformed],clickhouse_settings: { async_insert: 1 },@@ -1868,7 +1878,7 @@ export class AnalyticsController {if (!_isEmpty(perfTransformed)) {await clickhouse.insert({- table: 'performance',+ table: 'events',format: 'JSONEachRow',values: [perfTransformed],clickhouse_settings: { async_insert: 1 },@@ -1955,7 +1965,8 @@ export class AnalyticsController {const { deviceType, browserName, browserVersion, osName, osVersion } =await this.analyticsService.getRequestInformation(headers)- const transformed = trafficTransformer({+ const transformed = eventTransformer({+ type: 'pageview',psid,profileId,pid: logDTO.pid,@@ -1986,7 +1997,7 @@ export class AnalyticsController {try {await clickhouse.insert({- table: 'analytics',+ table: 'events',format: 'JSONEachRow',values: [transformed],clickhouse_settings: { async_insert: 1 },@@ -2039,7 +2050,7 @@ export class AnalyticsController {if (period === 'all') {const res = await this.analyticsService.calculateTimeBucketForAllTime(pid,- customEVFilterApplied ? 'customEV' : 'analytics',+ ['pageview', 'custom_event', 'error'],)timeBucket = res.timeBucket[0]@@ -2124,7 +2135,7 @@ export class AnalyticsController {if (period === 'all') {const res = await this.analyticsService.calculateTimeBucketForAllTime(pid,- 'errors',+ 'error',)timeBucket = res.timeBucket[0]@@ -2203,7 +2214,7 @@ export class AnalyticsController {if (period === 'all') {const res = await this.analyticsService.calculateTimeBucketForAllTime(pid,- 'errors',+ 'error',)newTimeBucket = res.timeBucket[0]@@ -2281,7 +2292,7 @@ export class AnalyticsController {if (period === 'all') {const res = await this.analyticsService.calculateTimeBucketForAllTime(pid,- 'errors',+ 'error',)newTimeBucket = res.timeBucket[0]@@ -2293,6 +2304,8 @@ export class AnalyticsController {DataType.ERRORS,true,)+ const [sessionFiltersQuery, sessionFiltersParams] =+ this.analyticsService.getFiltersQuery(filters, DataType.ANALYTICS, true)const safeTimezone = this.analyticsService.getSafeTimezone(timezone)const { groupFromUTC, groupToUTC } = this.analyticsService.getGroupFromTo(@@ -2322,6 +2335,8 @@ export class AnalyticsController {groupToUTC,newTimeBucket,parsedOptions.showResolved || false,+ sessionFiltersQuery,+ sessionFiltersParams,)}@@ -2362,7 +2377,7 @@ export class AnalyticsController {if (period === 'all') {const res = await this.analyticsService.calculateTimeBucketForAllTime(pid,- 'errors',+ 'error',)newTimeBucket = res.timeBucket[0]@@ -2467,7 +2482,7 @@ export class AnalyticsController {if (period === 'all') {const res = await this.analyticsService.calculateTimeBucketForAllTime(pid,- customEVFilterApplied ? 'customEV' : 'analytics',+ ['pageview', 'custom_event', 'error'],)timeBucket = res.timeBucket[0]@@ -2546,7 +2561,7 @@ export class AnalyticsController {if (period === 'all') {const res = await this.analyticsService.calculateTimeBucketForAllTime(pid,- 'analytics',+ ['pageview', 'custom_event', 'error'],)timeBucket = res.timeBucket[0]diff = res.diff@@ -2634,7 +2649,7 @@ export class AnalyticsController {if (period === 'all') {const res = await this.analyticsService.calculateTimeBucketForAllTime(pid,- customEVFilterApplied ? 'customEV' : 'analytics',+ ['pageview', 'custom_event', 'error'],)timeBucket = res.timeBucket[0]@@ -2714,7 +2729,7 @@ export class AnalyticsController {if (period === VALID_PERIODS[VALID_PERIODS.length - 1]) {const res = await this.analyticsService.calculateTimeBucketForAllTime(pid,- 'customEV',+ 'custom_event',)newTimeBucket = _includes(res.timeBucket, timeBucket)@@ -2728,16 +2743,24 @@ export class AnalyticsController {this.analyticsService.getFiltersQuery(filters, DataType.ANALYTICS)const safeTimezone = this.analyticsService.getSafeTimezone(timezone)- const { groupFrom, groupTo } = this.analyticsService.getGroupFromTo(- from,- to,- newTimeBucket,- period,- safeTimezone,- diff,- )+ const { groupFrom, groupTo, groupFromUTC, groupToUTC } =+ this.analyticsService.getGroupFromTo(+ from,+ to,+ newTimeBucket,+ period,+ safeTimezone,+ diff,+ )- const paramsData = { params: { pid, groupFrom, groupTo, ...filtersParams } }+ const paramsData = {+ params: {+ pid,+ groupFrom: groupFromUTC,+ groupTo: groupToUTC,+ ...filtersParams,+ },+ }// customEvents comes as a JSON.stringified array from the frontendlet customEventsList: string[] = []diff --git a/backend/apps/cloud/src/analytics/analytics.service.ts b/backend/apps/cloud/src/analytics/analytics.service.tsindex 80c4a8b6a..ba4e2dbcb 100644--- a/backend/apps/cloud/src/analytics/analytics.service.ts+++ b/backend/apps/cloud/src/analytics/analytics.service.ts@@ -278,6 +278,14 @@ const EXCLUDE_NULL_FOR = ['ctp',]+const ERROR_COLUMN_MAP: Record<string, string> = {+ name: 'error_name',+ message: 'error_message',+ filename: 'error_filename',+}++const mapErrorColumn = (type: string): string => ERROR_COLUMN_MAP[type] || type+const generateParamsQuery = (col: string,subQuery: string,@@ -285,7 +293,8 @@ const generateParamsQuery = (type: 'traffic' | 'performance' | 'captcha' | 'errors',measure?: PerfMeasure,): string => {- let columns = [`${col} as name`]+ const sqlCol = type === 'errors' ? mapErrorColumn(col) : col+ let columns = [`${sqlCol} as name`]// For regions and cities we'll return an array of objects, that will also include the country code and region code// We need the conutry code to display the flag next to the region/city name@@ -309,15 +318,15 @@ const generateParamsQuery = (const fn = MEASURES_MAP[processedMeasure]- if (col === 'pg' || col === 'host') {+ if (sqlCol === 'pg' || sqlCol === 'host') {return `SELECT ${columnsQuery}, round(divide(${fn}(pageLoad), 1000), 2) as count ${subQuery} GROUP BY ${columnsQuery}`}- return `SELECT ${columnsQuery}, round(divide(${fn}(pageLoad), 1000), 2) as count ${subQuery} ${EXCLUDE_NULL_FOR.includes(col) ? `AND ${col} IS NOT NULL` : ''} GROUP BY ${columnsQuery}`+ return `SELECT ${columnsQuery}, round(divide(${fn}(pageLoad), 1000), 2) as count ${subQuery} ${EXCLUDE_NULL_FOR.includes(sqlCol) ? `AND ${sqlCol} IS NOT NULL` : ''} GROUP BY ${columnsQuery}`}if (type === 'errors') {- return `SELECT ${columnsQuery}, count(*) as count ${subQuery} ${EXCLUDE_NULL_FOR.includes(col) ? `AND ${col} IS NOT NULL` : ''} GROUP BY ${columnsQuery}`+ return `SELECT ${columnsQuery}, count(*) as count ${subQuery} ${EXCLUDE_NULL_FOR.includes(sqlCol) ? `AND ${sqlCol} IS NOT NULL` : ''} GROUP BY ${columnsQuery}`}if (type === 'captcha') {@@ -338,6 +347,13 @@ export enum DataType {ERRORS = 'errors',}+type AnalyticsEventType = 'pageview' | 'custom_event'+type EventsAllTimeType =+ | AnalyticsEventType+ | 'performance'+ | 'error'+ | 'captcha'+const isValidOrigin = (origins: string[], origin: string) => {const escapeRegex = (str: string) =>str.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')@@ -704,6 +720,22 @@ export class AnalyticsService {return CAPTCHA_COLUMNS}+ getAnalyticsEventType(customEVFilterApplied: boolean): AnalyticsEventType {+ return customEVFilterApplied ? 'custom_event' : 'pageview'+ }++ buildAnalyticsEventsSubQuery(+ filtersQuery: string,+ customEVFilterApplied: boolean,+ isCaptcha = false,+ ): string {+ const eventType = isCaptcha+ ? 'captcha'+ : this.getAnalyticsEventType(customEVFilterApplied)++ return `FROM events WHERE pid = {pid:FixedString(12)} AND type = '${eventType}' ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String}`+ }+getGroupFromTo(from: string,to: string,@@ -857,9 +889,10 @@ export class AnalyticsService {pg,created,lagInFrame(pg) OVER (PARTITION BY psid ORDER BY created) AS prev_page- FROM analytics+ FROM eventsWHEREpid = {pid:FixedString(12)}+ AND type = 'pageview'AND created BETWEEN {groupFrom:String} AND {groupTo:String}AND pg IS NOT NULL${filtersQuery}@@ -970,15 +1003,17 @@ export class AnalyticsService {async calculateTimeBucketForAllTime(pid: string,- table: 'analytics' | 'customEV' | 'performance' | 'errors',+ eventTypes: EventsAllTimeType | readonly EventsAllTimeType[],): Promise<{timeBucket: TimeBucketType[]diff: number}> {+ const types = Array.isArray(eventTypes) ? [...eventTypes] : [eventTypes]+const { data: fromData } = await clickhouse.query({- query: `SELECT min(created) AS firstCreated FROM ${table} WHERE pid = {pid:FixedString(12)}`,- query_params: { pid },+ query: `SELECT min(created) AS firstCreated FROM events WHERE pid = {pid:FixedString(12)} AND type IN ({types:Array(String)})`,+ query_params: { pid, types },}).then((res) => res.json<{ firstCreated?: string }>())@@ -1212,8 +1247,8 @@ export class AnalyticsService {? 'argMin(pg, created)': 'argMax(pg, created)'const subQueryForPages = isContains- ? `SELECT psid FROM (SELECT psid, ${pageSelector} as page FROM analytics WHERE pid = {pid:FixedString(12)} GROUP BY psid) WHERE page ILIKE concat('%', {${param}:String}, '%')`- : `SELECT psid FROM (SELECT psid, ${pageSelector} as page FROM analytics WHERE pid = {pid:FixedString(12)} GROUP BY psid) WHERE page = {${param}:String}`+ ? `SELECT psid FROM (SELECT psid, ${pageSelector} as page FROM events WHERE pid = {pid:FixedString(12)} AND type = 'pageview' AND created BETWEEN {groupFrom:String} AND {groupTo:String} GROUP BY psid) WHERE page ILIKE concat('%', {${param}:String}, '%')`+ : `SELECT psid FROM (SELECT psid, ${pageSelector} as page FROM events WHERE pid = {pid:FixedString(12)} AND type = 'pageview' AND created BETWEEN {groupFrom:String} AND {groupTo:String} GROUP BY psid) WHERE page = {${param}:String}`// For exclusive filter (isNot) we exclude sessions with matching entry/exit pagequery += `psid ${isExclusive ? 'NOT IN' : 'IN'} (${subQueryForPages})`@@ -1254,6 +1289,11 @@ export class AnalyticsService {) {sqlColumn = 'meta.value'isArrayDataset = true+ } else if (column === 'ev') {+ // Public filter contract `ev` maps to the renamed `event_name` column+ sqlColumn = 'event_name'+ } else if (dataType === DataType.ERRORS) {+ sqlColumn = mapErrorColumn(column)}const isNullFilter =@@ -1631,21 +1671,11 @@ export class AnalyticsService {FROM (SELECTpsid,- pg AS value,- created- FROM analytics- WHERE pid = {pid:FixedString(12)}- AND psid != 0- AND created BETWEEN {groupFrom:String} AND {groupTo:String}-- UNION ALL-- SELECT- psid,- ev AS value,+ if(type = 'pageview', pg, event_name) AS value,created- FROM customEV+ FROM eventsWHERE pid = {pid:FixedString(12)}+ AND type IN ('pageview', 'custom_event')AND psid != 0AND created BETWEEN {groupFrom:String} AND {groupTo:String})@@ -1693,14 +1723,14 @@ export class AnalyticsService {psid,CAST(windowFunnel(86400)(created, ${pagesStr}) AS UInt64) AS levelFROM (- SELECT psid, pg AS value, created- FROM analytics- WHERE pid = {pid:FixedString(12)} AND psid != 0- AND created BETWEEN {groupFrom:String} AND {groupTo:String}- UNION ALL- SELECT psid, ev AS value, created- FROM customEV- WHERE pid = {pid:FixedString(12)} AND psid != 0+ SELECT+ psid,+ if(type = 'pageview', pg, event_name) AS value,+ created+ FROM events+ WHERE pid = {pid:FixedString(12)}+ AND type IN ('pageview', 'custom_event')+ AND psid != 0AND created BETWEEN {groupFrom:String} AND {groupTo:String})GROUP BY psid@@ -1715,17 +1745,11 @@ export class AnalyticsService {psid,argMin(cc, created) AS cc,argMin(if(domain(ref) != '', domain(ref), 'Direct / None'), created) AS source- FROM (- SELECT psid, cc, ref, created- FROM analytics- WHERE pid = {pid:FixedString(12)} AND psid != 0- AND created BETWEEN {groupFrom:String} AND {groupTo:String}- UNION ALL- SELECT psid, cc, ref, created- FROM customEV- WHERE pid = {pid:FixedString(12)} AND psid != 0- AND created BETWEEN {groupFrom:String} AND {groupTo:String}- )+ FROM events+ WHERE pid = {pid:FixedString(12)}+ AND type IN ('pageview', 'custom_event')+ AND psid != 0+ AND created BETWEEN {groupFrom:String} AND {groupTo:String}GROUP BY psid)SELECT step, type, val, cnt FROM (@@ -1803,14 +1827,14 @@ export class AnalyticsService {psid,windowFunnel(86400)(created, ${pagesStr}) AS levelFROM (- SELECT psid, pg AS value, created- FROM analytics- WHERE pid = {pid:FixedString(12)} AND psid != 0- AND created BETWEEN {groupFrom:String} AND {groupTo:String}- UNION ALL- SELECT psid, ev AS value, created- FROM customEV- WHERE pid = {pid:FixedString(12)} AND psid != 0+ SELECT+ psid,+ if(type = 'pageview', pg, event_name) AS value,+ created+ FROM events+ WHERE pid = {pid:FixedString(12)}+ AND type IN ('pageview', 'custom_event')+ AND psid != 0AND created BETWEEN {groupFrom:String} AND {groupTo:String})GROUP BY psid@@ -1826,19 +1850,12 @@ export class AnalyticsService {any(br) AS br,min(toTimeZone(created, {timezone:String})) AS sessionStart,max(toTimeZone(created, {timezone:String})) AS lastActivity- FROM (- SELECT psid, pid, cc, os, br, created- FROM analytics- WHERE pid = {pid:FixedString(12)} AND psid IS NOT NULL- AND created BETWEEN {groupFrom:String} AND {groupTo:String}- AND psid IN (SELECT psid FROM funnel_qualified)- UNION ALL- SELECT psid, pid, cc, os, br, created- FROM customEV- WHERE pid = {pid:FixedString(12)} AND psid IS NOT NULL+ FROM events+ WHERE pid = {pid:FixedString(12)}+ AND type IN ('pageview', 'custom_event')+ AND psid IS NOT NULLAND created BETWEEN {groupFrom:String} AND {groupTo:String}AND psid IN (SELECT psid FROM funnel_qualified)- )GROUP BY psidCasted, pid),pageview_counts AS (@@ -1846,8 +1863,8 @@ export class AnalyticsService {CAST(psid, 'String') AS psidCasted,pid,count() as count- FROM analytics- WHERE pid = {pid:FixedString(12)} AND psid IS NOT NULL+ FROM events+ WHERE pid = {pid:FixedString(12)} AND type = 'pageview' AND psid IS NOT NULLAND created BETWEEN {groupFrom:String} AND {groupTo:String}AND psid IN (SELECT psid FROM funnel_qualified)GROUP BY psidCasted, pid@@ -1857,8 +1874,8 @@ export class AnalyticsService {CAST(psid, 'String') AS psidCasted,pid,count() as count- FROM customEV- WHERE pid = {pid:FixedString(12)} AND psid IS NOT NULL+ FROM events+ WHERE pid = {pid:FixedString(12)} AND type = 'custom_event' AND psid IS NOT NULLAND created BETWEEN {groupFrom:String} AND {groupTo:String}AND psid IN (SELECT psid FROM funnel_qualified)GROUP BY psidCasted, pid@@ -1868,8 +1885,8 @@ export class AnalyticsService {CAST(psid, 'String') AS psidCasted,pid,count() as count- FROM errors- WHERE pid = {pid:FixedString(12)} AND psid IS NOT NULL+ FROM events+ WHERE pid = {pid:FixedString(12)} AND type = 'error' AND psid IS NOT NULLAND created BETWEEN {groupFrom:String} AND {groupTo:String}AND psid IN (SELECT psid FROM funnel_qualified)GROUP BY psidCasted, pid@@ -1947,8 +1964,9 @@ export class AnalyticsService {const query = `SELECTcount() as c- FROM analytics+ FROM eventsWHERE pid = {pid:FixedString(12)}+ AND type = 'pageview'AND created BETWEEN {groupFrom:String} AND {groupTo:String}`@@ -2044,12 +2062,13 @@ export class AnalyticsService {groupTo: string,): Promise<{ cc: string; count: number } | null> {const query = `- SELECT+ SELECTcc,count() as count- FROM analytics- WHERE+ FROM events+ WHEREpid = {pid:FixedString(12)}+ AND type = 'pageview'AND created BETWEEN {groupFrom:String} AND {groupTo:String}AND cc IS NOT NULLAND cc != ''@@ -2079,13 +2098,14 @@ export class AnalyticsService {const query = `WITH counts AS (- SELECT+ SELECTpid,cc,count() as cnt- FROM analytics- WHERE+ FROM events+ WHEREpid IN {pids:Array(FixedString(12))}+ AND type = 'pageview'AND created BETWEEN {groupFrom:String} AND {groupTo:String}AND cc IS NOT NULLAND cc != ''@@ -2128,12 +2148,13 @@ export class AnalyticsService {groupTo: string,): Promise<{ count: number; uniqueErrors: number }> {const query = `- SELECT+ SELECTcount() as count,count(DISTINCT eid) as uniqueErrors- FROM errors- WHERE+ FROM events+ WHEREpid = {pid:FixedString(12)}+ AND type = 'error'AND created BETWEEN {groupFrom:String} AND {groupTo:String}`@@ -2159,13 +2180,14 @@ export class AnalyticsService {}const query = `- SELECT+ SELECTpid,count() as count,count(DISTINCT eid) as uniqueErrors- FROM errors- WHERE+ FROM events+ WHEREpid IN {pids:Array(FixedString(12))}+ AND type = 'error'AND created BETWEEN {groupFrom:String} AND {groupTo:String}GROUP BY pid`@@ -2207,12 +2229,15 @@ export class AnalyticsService {}const query = `- SELECT+ SELECTpid,uniqExact(psid) as totalSessions- FROM analytics- WHERE+ FROM events+ WHEREpid IN {pids:Array(FixedString(12))}+ AND type IN ('pageview', 'custom_event', 'error')+ AND psid IS NOT NULL+ AND psid != ''AND created BETWEEN {groupFrom:String} AND {groupTo:String}GROUP BY pid`@@ -2277,15 +2302,18 @@ export class AnalyticsService {const promises = pids.map(async (pid) => {try {if (period === 'all') {- let queryAll = `+ const allTimeType = this.getAnalyticsEventType(customEVFilterApplied)++ const queryAll = `WITH analytics_counts AS (SELECTcount(*) AS all,count(DISTINCT psid) AS unique,count(DISTINCT profileId) AS users- FROM analytics+ FROM eventsWHEREpid = {pid:FixedString(12)}+ AND type = '${allTimeType}'${filtersQuery}),duration_avg AS (@@ -2296,6 +2324,14 @@ export class AnalyticsService {dateDiff('second', min(firstSeen), max(lastSeen)) as durationFROM sessionsWHERE pid = {pid:FixedString(12)}+ AND psid IN (+ SELECT DISTINCT psid+ FROM events+ WHERE pid = {pid:FixedString(12)}+ AND type = '${allTimeType}'+ AND psid IS NOT NULL+ ${filtersQuery}+ )GROUP BY psid))@@ -2305,36 +2341,6 @@ export class AnalyticsService {FROM analytics_counts, duration_avg`- if (customEVFilterApplied) {- queryAll = `- WITH analytics_counts AS (- SELECT- count(*) AS all,- count(DISTINCT psid) AS unique,- count(DISTINCT profileId) AS users- FROM customEV- WHERE- pid = {pid:FixedString(12)}- ${filtersQuery}- ),- duration_avg AS (- SELECT avgOrNull(duration) as sdur- FROM (- SELECT- psid,- dateDiff('second', min(firstSeen), max(lastSeen)) as duration- FROM sessions- WHERE pid = {pid:FixedString(12)}- GROUP BY psid- )- )- SELECT- analytics_counts.*,- duration_avg.sdur- FROM analytics_counts, duration_avg- `- }-const { data } = await clickhouse.query({query: queryAll,@@ -2378,7 +2384,7 @@ export class AnalyticsService {const { diff: allDiff, timeBucket: allowedBuckets } =await this.calculateTimeBucketForAllTime(pid,- customEVFilterApplied ? 'customEV' : 'analytics',+ this.getAnalyticsEventType(customEVFilterApplied),)const allTimeChartBucket = _includes(allowedBuckets,@@ -2419,16 +2425,19 @@ export class AnalyticsService {).format('YYYY-MM-DD HH:mm:ss')- let queryCurrent = `+ const periodType = this.getAnalyticsEventType(customEVFilterApplied)++ const queryCurrent = `WITH analytics_counts AS (SELECT1 AS sortOrder,count(*) AS all,count(DISTINCT psid) AS unique,count(DISTINCT profileId) AS users- FROM analytics+ FROM eventsWHEREpid = {pid:FixedString(12)}+ AND type = '${periodType}'AND created BETWEEN {groupFromUTC:String} AND {groupToUTC:String}${filtersQuery}),@@ -2442,8 +2451,9 @@ export class AnalyticsService {WHERE pid = {pid:FixedString(12)}AND psid IN (SELECT DISTINCT psid- FROM analytics+ FROM eventsWHERE pid = {pid:FixedString(12)}+ AND type = '${periodType}'AND psid IS NOT NULLAND created BETWEEN {groupFromUTC:String} AND {groupToUTC:String}${filtersQuery}@@ -2457,16 +2467,17 @@ export class AnalyticsService {FROM analytics_counts, duration_avg`- let queryPrevious = `+ const queryPrevious = `WITH analytics_counts AS (SELECT2 AS sortOrder,count(*) AS all,count(DISTINCT psid) AS unique,count(DISTINCT profileId) AS users- FROM analytics+ FROM eventsWHEREpid = {pid:FixedString(12)}+ AND type = '${periodType}'AND created BETWEEN {periodSubtracted:String} AND {groupFromUTC:String}${filtersQuery}),@@ -2480,8 +2491,9 @@ export class AnalyticsService {WHERE pid = {pid:FixedString(12)}AND psid IN (SELECT DISTINCT psid- FROM analytics+ FROM eventsWHERE pid = {pid:FixedString(12)}+ AND type = '${periodType}'AND psid IS NOT NULLAND created BETWEEN {periodSubtracted:String} AND {groupFromUTC:String}${filtersQuery}@@ -2495,83 +2507,6 @@ export class AnalyticsService {FROM analytics_counts, duration_avg`- if (customEVFilterApplied) {- queryCurrent = `- WITH analytics_counts AS (- SELECT- 1 AS sortOrder,- count(*) AS all,- count(DISTINCT psid) AS unique,- count(DISTINCT profileId) AS users- FROM customEV- WHERE- pid = {pid:FixedString(12)}- AND created BETWEEN {groupFromUTC:String} AND {groupToUTC:String}- ${filtersQuery}- ),- duration_avg AS (- SELECT avgOrNull(duration) as sdur- FROM (- SELECT- psid,- dateDiff('second', min(firstSeen), max(lastSeen)) as duration- FROM sessions- WHERE pid = {pid:FixedString(12)}- AND psid IN (- SELECT DISTINCT psid- FROM customEV- WHERE pid = {pid:FixedString(12)}- AND psid IS NOT NULL- AND created BETWEEN {groupFromUTC:String} AND {groupToUTC:String}- ${filtersQuery}- )- GROUP BY psid- )- )- SELECT- analytics_counts.*,- duration_avg.sdur- FROM analytics_counts, duration_avg- `- queryPrevious = `- WITH analytics_counts AS (- SELECT- 2 AS sortOrder,- count(*) AS all,- count(DISTINCT psid) AS unique,- count(DISTINCT profileId) AS users- FROM customEV- WHERE- pid = {pid:FixedString(12)}- AND created BETWEEN {periodSubtracted:String} AND {groupFromUTC:String}- ${filtersQuery}- ),- duration_avg AS (- SELECT avgOrNull(duration) as sdur- FROM (- SELECT- psid,- dateDiff('second', min(firstSeen), max(lastSeen)) as duration- FROM sessions- WHERE pid = {pid:FixedString(12)}- AND psid IN (- SELECT DISTINCT psid- FROM customEV- WHERE pid = {pid:FixedString(12)}- AND psid IS NOT NULL- AND created BETWEEN {periodSubtracted:String} AND {groupFromUTC:String}- ${filtersQuery}- )- GROUP BY psid- )- )- SELECT- analytics_counts.*,- duration_avg.sdur- FROM analytics_counts, duration_avg- `- }-const query = `${queryCurrent} UNION ALL ${queryPrevious}`let { data } = await clickhouse@@ -2666,6 +2601,55 @@ export class AnalyticsService {return result}+ private generateAnalyticsAggregationQueryForScope(+ timeBucket: TimeBucketType,+ filtersQuery: string,+ mode: ChartRenderMode,+ customEVFilterApplied: boolean,+ ): string {+ return customEVFilterApplied+ ? this.generateCustomEventsAggregationQuery(+ timeBucket,+ filtersQuery,+ mode,+ )+ : this.generateAnalyticsAggregationQuery(timeBucket, filtersQuery, mode)+ }++ private extractAnalyticsChartDataForScope(+ result: Array<TrafficCHResponse | TrafficCEFilterCHResponse>,+ xShifted: string[],+ customEVFilterApplied: boolean,+ mode: ChartRenderMode,+ ): IExtractChartData {+ const chartData = this.extractChartData(result, xShifted)++ if (customEVFilterApplied) {+ const visits =+ this.extractCustomEventsChartData(result, xShifted)?._unknown_event ||+ []++ chartData.visits = Array.from(+ { length: _size(xShifted) },+ (_, index) => visits[index] || 0,+ )+ }++ // Propagate the previous cumulative value forward for empty buckets+ if (mode === ChartRenderMode.CUMULATIVE) {+ for (let i = 1; i < chartData.visits.length; ++i) {+ if (chartData.visits[i] === 0) {+ chartData.visits[i] = chartData.visits[i - 1]+ }+ if (chartData.uniques[i] === 0) {+ chartData.uniques[i] = chartData.uniques[i - 1]+ }+ }+ }++ return chartData+ }+/*** Get simplified chart data for dashboard cards* Returns only x (dates) and visits (pageviews) for a lightweight chart@@ -2696,30 +2680,11 @@ export class AnalyticsService {},}- if (customEVFilterApplied) {- const query = this.generateCustomEventsAggregationQuery(- timeBucket,- filtersQuery,- ChartRenderMode.PERIODICAL,- )-- const { data } = await clickhouse- .query({- query,- query_params: { ...paramsData.params, timezone: safeTimezone },- })- .then((resultSet) => resultSet.json<TrafficCEFilterCHResponse>())-- const visits =- this.extractCustomEventsChartData(data, xShifted)?._unknown_event || []-- return { x: xShifted, visits }- }-- const query = this.generateAnalyticsAggregationQuery(+ const query = this.generateAnalyticsAggregationQueryForScope(timeBucket,filtersQuery,ChartRenderMode.PERIODICAL,+ customEVFilterApplied,)const { data } = await clickhouse@@ -2727,9 +2692,16 @@ export class AnalyticsService {query,query_params: { ...paramsData.params, timezone: safeTimezone },})- .then((resultSet) => resultSet.json<TrafficCHResponse>())+ .then((resultSet) =>+ resultSet.json<TrafficCHResponse | TrafficCEFilterCHResponse>(),+ )- const { visits } = this.extractChartData(data, xShifted)+ const { visits } = this.extractAnalyticsChartDataForScope(+ data,+ xShifted,+ customEVFilterApplied,+ ChartRenderMode.PERIODICAL,+ )return { x: xShifted, visits }}@@ -2750,7 +2722,7 @@ export class AnalyticsService {if (_isEmpty(period) || ['today', 'yesterday', 'custom'].includes(period)) {const safeTimezone = this.getSafeTimezone(timezone)- const { groupFrom, groupTo } = this.getGroupFromTo(+ const { groupFromUTC, groupToUTC } = this.getGroupFromTo(from,to,['today', 'yesterday'].includes(period) ? TimeBucketType.HOUR : null,@@ -2758,8 +2730,8 @@ export class AnalyticsService {safeTimezone,)- _from = groupFrom- _to = groupTo+ _from = groupFromUTC+ _to = groupToUTC}const result = {}@@ -2779,7 +2751,7 @@ export class AnalyticsService {const promises = pids.map(async (pid) => {try {if (period === 'all') {- const queryAll = `SELECT ${columnSelectors} FROM performance WHERE pid = {pid:FixedString(12)} ${filtersQuery}`+ const queryAll = `SELECT ${columnSelectors} FROM events WHERE pid = {pid:FixedString(12)} AND type = 'performance' ${filtersQuery}`const { data } = await clickhouse.query({@@ -2823,11 +2795,12 @@ export class AnalyticsService {if (_from && _to) {// diff may be 0 (when selecting data for 1 day), so let's make it 1 to grab some data for the prev day as well- const diff = dayjs(_to).diff(dayjs(_from), 'days') || 1+ const diff = dayjs.utc(_to).diff(dayjs.utc(_from), 'days') || 1now = _toperiodFormatted = _from- periodSubtracted = dayjs(_from)+ periodSubtracted = dayjs+ .utc(_from).subtract(diff, 'days').format('YYYY-MM-DD HH:mm:ss')} else {@@ -2842,8 +2815,8 @@ export class AnalyticsService {.format('YYYY-MM-DD HH:mm:ss')}- const queryCurrent = `SELECT 1 AS sortOrder, ${columnSelectors} FROM performance WHERE pid = {pid:FixedString(12)} AND created BETWEEN {periodFormatted:String} AND {now:String} ${filtersQuery}`- const queryPrevious = `SELECT 2 AS sortOrder, ${columnSelectors} FROM performance WHERE pid = {pid:FixedString(12)} AND created BETWEEN {periodSubtracted:String} AND {periodFormatted:String} ${filtersQuery}`+ const queryCurrent = `SELECT 1 AS sortOrder, ${columnSelectors} FROM events WHERE pid = {pid:FixedString(12)} AND type = 'performance' AND created BETWEEN {periodFormatted:String} AND {now:String} ${filtersQuery}`+ const queryPrevious = `SELECT 2 AS sortOrder, ${columnSelectors} FROM events WHERE pid = {pid:FixedString(12)} AND type = 'performance' AND created BETWEEN {periodSubtracted:String} AND {periodFormatted:String} ${filtersQuery}`const query = `${queryCurrent} UNION ALL ${queryPrevious}`@@ -2940,8 +2913,8 @@ export class AnalyticsService {const query = `WITH session_pages AS (SELECT psid, ${selector} as page- FROM analytics- WHERE pid = {pid:FixedString(12)}+ FROM events+ WHERE pid = {pid:FixedString(12)} AND type = 'pageview'GROUP BY psid)SELECT page FROM session_pages WHERE page IS NOT NULL GROUP BY page@@ -2957,11 +2930,11 @@ export class AnalyticsService {return _map(data, 'page')}- let query = `SELECT ${type} FROM analytics WHERE pid={pid:FixedString(12)} AND ${type} IS NOT NULL GROUP BY ${type}`-- if (type === 'ev') {- query = `SELECT ${type} FROM customEV WHERE pid={pid:FixedString(12)} AND ${type} IS NOT NULL GROUP BY ${type}`- }+ // Public filter contract `ev` reads from the renamed event_name column.+ let query =+ type === 'ev'+ ? `SELECT event_name AS ev FROM events WHERE pid={pid:FixedString(12)} AND type = 'custom_event' AND event_name IS NOT NULL GROUP BY event_name`+ : `SELECT ${type} FROM events WHERE pid={pid:FixedString(12)} AND type IN ('pageview', 'custom_event') AND ${type} IS NOT NULL GROUP BY ${type}`const { data } = await clickhouse.query({@@ -2982,7 +2955,9 @@ export class AnalyticsService {)}- const query = `SELECT ${type} FROM errors WHERE pid={pid:FixedString(12)} AND ${type} IS NOT NULL GROUP BY ${type}`+ const sqlCol = mapErrorColumn(type)++ const query = `SELECT ${sqlCol} AS ${type} FROM events WHERE pid={pid:FixedString(12)} AND type = 'error' AND ${sqlCol} IS NOT NULL GROUP BY ${sqlCol}`const { data } = await clickhouse.query({@@ -3001,13 +2976,16 @@ export class AnalyticsService {type: 'traffic' | 'errors',column: 'br' | 'os',): Promise<Array<{ name: string; version: string }>> {- const safeTable = type === 'errors' ? 'errors' : 'analytics'+ const safeType =+ type === 'errors'+ ? "type = 'error'"+ : "type IN ('pageview', 'custom_event')"const safeVersionCol = column === 'br' ? 'brv' : 'osv'const query = `- SELECT ${column}, ${safeVersionCol}- FROM ${safeTable}- WHERE pid={pid:FixedString(12)} AND ${column} IS NOT NULL AND ${safeVersionCol} IS NOT NULL+ SELECT ${column}, ${safeVersionCol}+ FROM events+ WHERE pid={pid:FixedString(12)} AND ${safeType} AND ${column} IS NOT NULL AND ${safeVersionCol} IS NOT NULLGROUP BY ${column}, ${safeVersionCol}`@@ -3067,7 +3045,7 @@ export class AnalyticsService {const query = `WITH ${withClauses.join(',\n')}- SELECT+ SELECTcolumn_name,name,count,@@ -3088,7 +3066,7 @@ export class AnalyticsService {}return `- SELECT+ SELECT'${col}' as column_name,name,count,@@ -3385,8 +3363,8 @@ export class AnalyticsService {pid,psid,${timeBucketFunc}(toTimeZone(created, {timezone:String})) as tz_created- FROM analytics- PREWHERE pid = {pid:FixedString(12)}+ FROM events+ PREWHERE pid = {pid:FixedString(12)} AND type = 'pageview'WHERE created BETWEEN ${tzFromDate} AND ${tzToDate}${filtersQuery}) as subquery@@ -3434,8 +3412,8 @@ export class AnalyticsService {FROM (SELECT${timeBucketFunc}(toTimeZone(created, {timezone:String})) as tz_created- FROM analytics- PREWHERE pid = {pid:FixedString(12)}+ FROM events+ PREWHERE pid = {pid:FixedString(12)} AND type = 'pageview'WHERE created BETWEEN ${tzFromDate} AND ${tzToDate}${filtersQuery}) as subquery@@ -3462,8 +3440,8 @@ export class AnalyticsService {FROM (SELECT${timeBucketFunc}(toTimeZone(created, {timezone:String})) as tz_created- FROM customEV- PREWHERE pid = {pid:FixedString(12)}+ FROM events+ PREWHERE pid = {pid:FixedString(12)} AND type = 'custom_event'WHERE created BETWEEN ${tzFromDate} AND ${tzToDate}${filtersQuery}) as subquery@@ -3488,8 +3466,8 @@ export class AnalyticsService {FROM (SELECT${timeBucketFunc}(toTimeZone(created, {timezone:String})) as tz_created- FROM errors- PREWHERE pid = {pid:FixedString(12)}+ FROM events+ PREWHERE pid = {pid:FixedString(12)} AND type = 'error'WHERE created BETWEEN ${tzFromDate} AND ${tzToDate}${filtersQuery}) as subquery@@ -3511,15 +3489,31 @@ export class AnalyticsService {const baseQuery = `SELECT${selector},+ avgOrNull(sessions_data.duration) as sdur,+ count(DISTINCT psid) as uniques,count() as countFROM (- SELECT *,- ${timeBucketFunc}(toTimeZone(created, {timezone:String})) as tz_created- FROM customEV+ SELECT+ pid,+ psid,+ ${timeBucketFunc}(toTimeZone(created, {timezone:String})) as tz_created+ FROM eventsWHERE pid = {pid:FixedString(12)}+ AND type = 'custom_event'AND created BETWEEN ${tzFromDate} AND ${tzToDate}${filtersQuery}) as subquery+ LEFT JOIN (+ SELECT+ pid,+ psid,+ dateDiff('second', min(firstSeen), max(lastSeen)) as duration+ FROM sessions+ WHERE pid = {pid:FixedString(12)}+ GROUP BY pid, psid+ ) as sessions_data+ ON subquery.pid = sessions_data.pid+ AND subquery.psid = sessions_data.psidGROUP BY ${groupBy}ORDER BY ${groupBy}`@@ -3528,7 +3522,8 @@ export class AnalyticsService {return `SELECT*,- sum(count) OVER (ORDER BY ${groupBy}) as count+ sum(count) OVER (ORDER BY ${groupBy}) as count,+ sum(uniques) OVER (ORDER BY ${groupBy}) as uniquesFROM (${baseQuery})`}@@ -3559,9 +3554,10 @@ export class AnalyticsService {FROM (SELECT *,${timeBucketFunc}(toTimeZone(created, {timezone:String})) as tz_created- FROM performance+ FROM eventsWHEREpid = {pid:FixedString(12)}+ AND type = 'performance'AND created BETWEEN ${tzFromDate} AND ${tzToDate}${filtersQuery}) as subquery@@ -3594,9 +3590,10 @@ export class AnalyticsService {FROM (SELECT *,${timeBucketFunc}(toTimeZone(created, {timezone:String})) as tz_created- FROM performance+ FROM eventsWHEREpid = {pid:FixedString(12)}+ AND type = 'performance'AND created BETWEEN ${tzFromDate} AND ${tzToDate}${filtersQuery}) as subquery@@ -3622,9 +3619,10 @@ export class AnalyticsService {FROM (SELECT *,${timeBucketFunc}(toTimeZone(created, {timezone:String})) as tz_created- FROM captcha+ FROM eventsWHEREpid = {pid:FixedString(12)}+ AND type = 'captcha'AND created BETWEEN ${tzFromDate} AND ${tzToDate}${filtersQuery}) as subquery@@ -3661,9 +3659,10 @@ export class AnalyticsService {FROM (SELECT pg, dv, br, os, lc, cc, rg, ct, profileId,${timeBucketFunc}(created) as tz_created- FROM errors+ FROM eventsWHEREpid = {pid:FixedString(12)}+ AND type = 'error'AND created BETWEEN {groupFrom:String} AND {groupTo:String}${filtersQuery}) as subquery@@ -3674,10 +3673,11 @@ export class AnalyticsService {if (mode === ChartRenderMode.CUMULATIVE) {return `SELECT- *,- sum(pageviews) OVER (ORDER BY ${groupBy}) as pageviews,- sum(uniques) OVER (ORDER BY ${groupBy}) as uniques+ ${groupBy},+ sum(count) OVER (ORDER BY ${groupBy}) as count,+ sum(affectedUsers) OVER (ORDER BY ${groupBy}) as affectedUsersFROM (${baseQuery})+ ORDER BY ${groupBy}`}@@ -3742,7 +3742,7 @@ export class AnalyticsService {): Promise<{ name: string; count: number }[]> {const query = `WITH session_first_pages AS (- SELECT+ SELECTpsid,argMin(pg, created) as entry_page${subQuery}@@ -3837,39 +3837,11 @@ export class AnalyticsService {): Promise<object | void> {const { xShifted } = this.generateXAxis(timeBucket, from, to, safeTimezone)- if (customEVFilterApplied) {- const query = this.generateCustomEventsAggregationQuery(- timeBucket,- filtersQuery,- mode,- )-- const { data } = await clickhouse- .query({- query,- query_params: { ...paramsData.params, timezone: safeTimezone },- })- .then((resultSet) => resultSet.json<TrafficCEFilterCHResponse>())-- const uniques =- this.extractCustomEventsChartData(data, xShifted)?._unknown_event || []-- const sdur = Array(_size(xShifted)).fill(0)-- return Promise.resolve({- chart: {- x: xShifted,- visits: uniques,- uniques,- sdur,- },- })- }-- const query = this.generateAnalyticsAggregationQuery(+ const query = this.generateAnalyticsAggregationQueryForScope(timeBucket,filtersQuery,mode,+ customEVFilterApplied,)const { data } = await clickhouse@@ -3877,21 +3849,16 @@ export class AnalyticsService {query,query_params: { ...paramsData.params, timezone: safeTimezone },})- .then((resultSet) => resultSet.json<TrafficCHResponse>())-- const { visits, uniques, sdur } = this.extractChartData(data, xShifted)+ .then((resultSet) =>+ resultSet.json<TrafficCHResponse | TrafficCEFilterCHResponse>(),+ )- // Propagate the previous cumulative value forward for empty buckets- if (mode === ChartRenderMode.CUMULATIVE) {- for (let i = 1; i < visits.length; ++i) {- if (visits[i] === 0) {- visits[i] = visits[i - 1]- }- if (uniques[i] === 0) {- uniques[i] = uniques[i - 1]- }- }- }+ const { visits, uniques, sdur } = this.extractAnalyticsChartDataForScope(+ data,+ xShifted,+ customEVFilterApplied,+ mode,+ )return Promise.resolve({chart: {@@ -4431,7 +4398,7 @@ export class AnalyticsService {filtersQuery: string,params: any,): Promise<ICustomEvent> {- const query = `SELECT ev, count() FROM customEV WHERE pid = {pid:FixedString(12)} ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String} GROUP BY ev`+ const query = `SELECT event_name AS ev, count() FROM events WHERE pid = {pid:FixedString(12)} AND type = 'custom_event' ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String} GROUP BY event_name`const result = {}const { data } = await clickhouse@@ -4461,10 +4428,10 @@ export class AnalyticsService {FROM (SELECTmeta.key AS key- FROM- analytics+ FROM eventsWHEREpid = {pid:FixedString(12)}+ AND type = 'pageview'${filtersQuery}AND created BETWEEN {groupFrom:String} AND {groupTo:String})@@ -4518,7 +4485,7 @@ export class AnalyticsService {)if (period === 'all') {- const res = await this.calculateTimeBucketForAllTime(pid, 'customEV')+ const res = await this.calculateTimeBucketForAllTime(pid, 'custom_event')diff = res.diff@@ -4546,12 +4513,12 @@ export class AnalyticsService {SELECTmeta.key,meta.value- FROM- customEV+ FROM eventsWHEREpid = {pid:FixedString(12)}+ AND type = 'custom_event'AND created BETWEEN {groupFrom:String} AND {groupTo:String}- AND ev = {event:String}+ AND event_name = {event:String}${filtersQuery})ARRAY JOIN meta.key, meta.value@@ -4610,7 +4577,7 @@ export class AnalyticsService {const [filtersQuery, filtersParams, appliedFilters, customEVFilterApplied] =this.getFiltersQuery(filters, DataType.ANALYTICS)- // We cannot make a query to customEV table using analytics table properties+ // Page properties are only present on pageview-scoped analytics rows.if (customEVFilterApplied) {return {result: [],@@ -4619,7 +4586,7 @@ export class AnalyticsService {}if (period === 'all') {- const res = await this.calculateTimeBucketForAllTime(pid, 'analytics')+ const res = await this.calculateTimeBucketForAllTime(pid, 'pageview')diff = res.diff@@ -4647,10 +4614,10 @@ export class AnalyticsService {SELECTmeta.key,meta.value- FROM- analytics+ FROM eventsWHEREpid = {pid:FixedString(12)}+ AND type = 'pageview'AND created BETWEEN {groupFrom:String} AND {groupTo:String}AND indexOf(meta.key, {property:String}) > 0${filtersQuery}@@ -4700,17 +4667,11 @@ export class AnalyticsService {const query = `SELECT uniqExact(psid) as count- FROM (- SELECT psid FROM analytics- WHERE pid = {pid:FixedString(12)}- AND created >= {since:DateTime}- AND psid IS NOT NULL- UNION ALL- SELECT psid FROM customEV- WHERE pid = {pid:FixedString(12)}- AND created >= {since:DateTime}- AND psid IS NOT NULL- )+ FROM events+ WHERE pid = {pid:FixedString(12)}+ AND type IN ('pageview', 'custom_event')+ AND created >= {since:DateTime}+ AND psid IS NOT NULL`try {@@ -4748,25 +4709,26 @@ export class AnalyticsService {const tzToDate = `toTimeZone(parseDateTimeBestEffort({groupTo:String}), {timezone:String})`const customEventsFilter =customEvents && customEvents.length > 0- ? 'AND ev IN {customEvents:Array(String)}'+ ? 'AND event_name IN {customEvents:Array(String)}': ''const query = `SELECT${selector},- ev,+ event_name AS ev,count() as countFROM (SELECT *,${timeBucketFunc}(toTimeZone(created, {timezone:String})) as tz_created- FROM customEV+ FROM eventsWHEREpid = {pid:FixedString(12)}+ AND type = 'custom_event'AND created BETWEEN ${tzFromDate} AND ${tzToDate}${filtersQuery}${customEventsFilter}) as subquery- GROUP BY ${groupBy}, ev+ GROUP BY ${groupBy}, event_nameORDER BY ${groupBy}`@@ -4864,52 +4826,47 @@ export class AnalyticsService {const queryPages = `WITH events_with_meta AS (SELECT- 'pageview' AS type,- toString(pg) AS value,- toTimeZone(analytics.created, {timezone:String}) AS created,- pid,- toString(analytics.psid) AS psid,- arrayFilter(x -> x.1 != '' AND x.2 != '', arrayZip(meta.key, meta.value)) AS metadata- FROM analytics- WHERE- pid = {pid:FixedString(12)}- AND analytics.psid IS NOT NULL- AND toString(analytics.psid) = {psid:String}-- UNION ALL-- SELECT- 'event' AS type,- toString(ev) AS value,- toTimeZone(customEV.created, {timezone:String}) AS created,- pid,- toString(customEV.psid) AS psid,- arrayFilter(x -> x.1 != '' AND x.2 != '', arrayZip(meta.key, meta.value)) AS metadata- FROM customEV- WHERE- pid = {pid:FixedString(12)}- AND customEV.psid IS NOT NULL- AND toString(customEV.psid) = {psid:String}-- UNION ALL-- SELECT- 'error' AS type,- toString(errors.name) AS value,- toTimeZone(errors.created, {timezone:String}) AS created,+ multiIf(event_type = 'pageview', 'pageview', event_type = 'custom_event', 'event', 'error') AS type,+ multiIf(+ event_type = 'pageview', toString(pg),+ event_type = 'custom_event', toString(event_name),+ toString(error_name)+ ) AS value,+ toTimeZone(created, {timezone:String}) AS created,pid,- toString(errors.psid) AS psid,- [- tuple('message', COALESCE(errors.message, '')),- tuple('lineno', toString(COALESCE(errors.lineno, 0))),- tuple('colno', toString(COALESCE(errors.colno, 0))),- tuple('filename', COALESCE(errors.filename, ''))- ] AS metadata- FROM errors- WHERE- pid = {pid:FixedString(12)}- AND errors.psid IS NOT NULL- AND toString(errors.psid) = {psid:String}+ toString(psid) AS psid,+ if(+ event_type = 'error',+ arrayFilter(x -> x.2 != '' AND x.2 != '0', [+ tuple('message', COALESCE(error_message, '')),+ tuple('lineno', ifNull(toString(lineno), '')),+ tuple('colno', ifNull(toString(colno), '')),+ tuple('filename', COALESCE(error_filename, ''))+ ]),+ arrayFilter(x -> x.1 != '' AND x.2 != '', arrayZip(meta_key, meta_value))+ ) AS metadata+ FROM (+ SELECT+ type AS event_type,+ pg,+ event_name,+ error_name,+ error_message,+ lineno,+ colno,+ error_filename,+ created,+ pid,+ psid,+ meta.key AS meta_key,+ meta.value AS meta_value+ FROM events+ WHERE+ pid = {pid:FixedString(12)}+ AND type IN ('pageview', 'custom_event', 'error')+ AND psid IS NOT NULL+ AND toString(psid) = {psid:String}+ )UNION ALL@@ -4961,12 +4918,19 @@ export class AnalyticsService {const querySessionDetails = `SELECTdv, br, brv, os, osv, lc, ref, so, me, ca, te, co, cc, rg, ct, profileId- FROM analytics+ FROM eventsWHEREpid = {pid:FixedString(12)}+ AND type IN ('pageview', 'custom_event', 'error')AND psid IS NOT NULLAND toString(psid) = {psid:String}- ORDER BY created ASC+ ORDER BY+ CASE+ WHEN type = 'pageview' THEN 0+ WHEN type = 'custom_event' THEN 1+ ELSE 2+ END,+ created ASCLIMIT 1;`@@ -5057,9 +5021,10 @@ export class AnalyticsService {const querySessionDetailsFromCustomEV = `SELECTdv, br, brv, os, osv, lc, ref, so, me, ca, te, co, cc, rg, ct, profileId- FROM customEV+ FROM eventsWHEREpid = {pid:FixedString(12)}+ AND type IN ('custom_event', 'error')AND psid IS NOT NULLAND toString(psid) = {psid:String}ORDER BY created ASC@@ -5149,94 +5114,16 @@ export class AnalyticsService {skip = 0,customEVFilterApplied = false,): Promise<object | void> {- let primaryEventsSubquery: string-- if (customEVFilterApplied) {- // When filtering by custom events, we need to:- // 1. Identify sessions (psids) that have matching custom events- // 2. Calculate session boundaries from ALL analytics events for those sessions- primaryEventsSubquery = `- SELECT- all_events.psidCasted,- all_events.pid,- all_events.cc,- all_events.os,- all_events.br,- all_events.created_for_grouping- FROM (- SELECT- CAST(analytics.psid, 'String') AS psidCasted,- analytics.pid,- analytics.cc,- analytics.os,- analytics.br,- toTimeZone(analytics.created, {timezone:String}) AS created_for_grouping- FROM analytics- WHERE- analytics.pid = {pid:FixedString(12)}- AND analytics.psid IS NOT NULL- AND analytics.created BETWEEN {groupFrom:String} AND {groupTo:String}- UNION ALL- SELECT- CAST(customEV.psid, 'String') AS psidCasted,- customEV.pid,- customEV.cc,- customEV.os,- customEV.br,- toTimeZone(customEV.created, {timezone:String}) AS created_for_grouping- FROM customEV- WHERE- customEV.pid = {pid:FixedString(12)}- AND customEV.psid IS NOT NULL- AND customEV.created BETWEEN {groupFrom:String} AND {groupTo:String}- ) AS all_events- WHERE all_events.psidCasted IN (- SELECT DISTINCT CAST(customEV.psid, 'String')- FROM customEV- WHERE- customEV.pid = {pid:FixedString(12)}- AND customEV.psid IS NOT NULL- AND customEV.created BETWEEN {groupFrom:String} AND {groupTo:String}- ${filtersQuery}- )- `- } else {- primaryEventsSubquery = `- SELECT- CAST(analytics.psid, 'String') AS psidCasted,- analytics.pid,- analytics.cc,- analytics.os,- analytics.br,- toTimeZone(analytics.created, {timezone:String}) AS created_for_grouping- FROM analytics- WHERE- analytics.pid = {pid:FixedString(12)}- AND analytics.psid IS NOT NULL- AND analytics.created BETWEEN {groupFrom:String} AND {groupTo:String}- ${filtersQuery}- UNION ALL- SELECT- CAST(customEV.psid, 'String') AS psidCasted,- customEV.pid,- customEV.cc,- customEV.os,- customEV.br,- toTimeZone(customEV.created, {timezone:String}) AS created_for_grouping- FROM customEV- WHERE- customEV.pid = {pid:FixedString(12)}- AND customEV.psid IS NOT NULL- AND customEV.created BETWEEN {groupFrom:String} AND {groupTo:String}- ${filtersQuery}- `- }+ const primaryEventsSubquery = this.buildSessionsListPrimaryEventsSubquery(+ filtersQuery,+ customEVFilterApplied,+ )const query = `WITH distinct_sessions_filtered AS (SELECTpsidCasted,- pid,+ pid,any(cc) AS cc,any(os) AS os,any(br) AS br,@@ -5246,32 +5133,32 @@ export class AnalyticsService {GROUP BY psidCasted, pid),pageview_counts AS (- SELECT+ SELECTCAST(psid, 'String') AS psidCasted,- pid,- count() as count- FROM analytics- WHERE pid = {pid:FixedString(12)} AND psid IS NOT NULL+ pid,+ count() as count+ FROM events+ WHERE pid = {pid:FixedString(12)} AND type = 'pageview' AND psid IS NOT NULLAND created BETWEEN {groupFrom:String} AND {groupTo:String}GROUP BY psidCasted, pid),event_counts AS (- SELECT+ SELECTCAST(psid, 'String') AS psidCasted,- pid,- count() as count- FROM customEV- WHERE pid = {pid:FixedString(12)} AND psid IS NOT NULL+ pid,+ count() as count+ FROM events+ WHERE pid = {pid:FixedString(12)} AND type = 'custom_event' AND psid IS NOT NULLAND created BETWEEN {groupFrom:String} AND {groupTo:String}GROUP BY psidCasted, pid),error_counts AS (- SELECT+ SELECTCAST(psid, 'String') AS psidCasted,- pid,- count() as count- FROM errors- WHERE pid = {pid:FixedString(12)} AND psid IS NOT NULL+ pid,+ count() as count+ FROM events+ WHERE pid = {pid:FixedString(12)} AND type = 'error' AND psid IS NOT NULLAND created BETWEEN {groupFrom:String} AND {groupTo:String}GROUP BY psidCasted, pid),@@ -5296,9 +5183,9 @@ export class AnalyticsService {GROUP BY psidCasted, pid),session_duration_agg AS (- SELECT- CAST(psid, 'String') AS psidCasted,- pid,+ SELECT+ CAST(psid, 'String') AS psidCasted,+ pid,dateDiff('second', min(firstSeen), max(lastSeen)) as avg_duration,argMax(profileId, lastSeen) as profileIdFROM sessions@@ -5360,75 +5247,65 @@ export class AnalyticsService {return data}- async getProfilesList(- pid: string,+ private buildSessionsListPrimaryEventsSubquery(filtersQuery: string,- paramsData: any,- safeTimezone: string,- take = 30,- skip = 0,- profileType: 'all' | 'anonymous' | 'identified' = 'all',- customEVFilterApplied = false,- ): Promise<object[]> {- let profileTypeFilter = ''- if (profileType === 'anonymous') {- profileTypeFilter = `AND profileId LIKE '${AnalyticsService.PROFILE_PREFIX_ANON}%'`- } else if (profileType === 'identified') {- profileTypeFilter = `AND profileId LIKE '${AnalyticsService.PROFILE_PREFIX_USER}%'`- }+ customEVFilterApplied: boolean,+ ): string {+ const scopedSessionFilter = customEVFilterApplied+ ? `+ AND CAST(psid, 'String') IN (+ SELECT DISTINCT CAST(psid, 'String')+ FROM events+ WHERE+ pid = {pid:FixedString(12)}+ AND type = 'custom_event'+ AND psid IS NOT NULL+ AND created BETWEEN {groupFrom:String} AND {groupTo:String}+ ${filtersQuery}+ )+ `+ : filtersQuery- let allProfileDataCTE: string+ return `+ SELECT+ CAST(psid, 'String') AS psidCasted,+ pid,+ cc,+ os,+ br,+ toTimeZone(created, {timezone:String}) AS created_for_grouping+ FROM events+ WHERE+ pid = {pid:FixedString(12)}+ AND type IN ('pageview', 'custom_event', 'error')+ AND psid IS NOT NULL+ AND created BETWEEN {groupFrom:String} AND {groupTo:String}+ ${scopedSessionFilter}+ `+ }- if (customEVFilterApplied) {- allProfileDataCTE = `- all_profile_data AS (- SELECT- profileId,- psid,- cc,- os,- br,- dv,- created,- 1 AS isPageview,- 0 AS isEvent- FROM analytics- WHERE pid = {pid:FixedString(12)}- AND created BETWEEN {groupFrom:String} AND {groupTo:String}- AND profileId IS NOT NULL- AND profileId != ''- ${profileTypeFilter}+ private buildProfilesListDataCTE(+ filtersQuery: string,+ profileTypeFilter: string,+ customEVFilterApplied: boolean,+ ): string {+ const scopedProfileFilter = customEVFilterApplied+ ? `AND profileId IN (SELECT DISTINCT profileId- FROM customEV+ FROM eventsWHERE pid = {pid:FixedString(12)}+ AND type = 'custom_event'AND created BETWEEN {groupFrom:String} AND {groupTo:String}AND profileId IS NOT NULLAND profileId != ''${profileTypeFilter}${filtersQuery})- UNION ALL- SELECT- profileId,- psid,- cc,- os,- br,- dv,- created,- 0 AS isPageview,- 1 AS isEvent- FROM customEV- WHERE pid = {pid:FixedString(12)}- AND created BETWEEN {groupFrom:String} AND {groupTo:String}- AND profileId IS NOT NULL- AND profileId != ''- ${profileTypeFilter}- ${filtersQuery}- )`- } else {- allProfileDataCTE = `+ `+ : filtersQuery++ return `all_profile_data AS (SELECTprofileId,@@ -5438,36 +5315,43 @@ export class AnalyticsService {br,dv,created,- 1 AS isPageview,- 0 AS isEvent- FROM analytics+ if(type = 'pageview', 1, 0) AS isPageview,+ if(type = 'custom_event', 1, 0) AS isEvent,+ if(type = 'error', 1, 0) AS isError+ FROM eventsWHERE pid = {pid:FixedString(12)}+ AND type IN ('pageview', 'custom_event', 'error')AND created BETWEEN {groupFrom:String} AND {groupTo:String}AND profileId IS NOT NULLAND profileId != ''${profileTypeFilter}- ${filtersQuery}- UNION ALL- SELECT- profileId,- psid,- cc,- os,- br,- dv,- created,- 0 AS isPageview,- 1 AS isEvent- FROM customEV- WHERE pid = {pid:FixedString(12)}- AND created BETWEEN {groupFrom:String} AND {groupTo:String}- AND profileId IS NOT NULL- AND profileId != ''- ${profileTypeFilter}- ${filtersQuery}+ ${scopedProfileFilter})`+ }++ async getProfilesList(+ pid: string,+ filtersQuery: string,+ paramsData: any,+ safeTimezone: string,+ take = 30,+ skip = 0,+ profileType: 'all' | 'anonymous' | 'identified' = 'all',+ customEVFilterApplied = false,+ ): Promise<object[]> {+ let profileTypeFilter = ''+ if (profileType === 'anonymous') {+ profileTypeFilter = `AND profileId LIKE '${AnalyticsService.PROFILE_PREFIX_ANON}%'`+ } else if (profileType === 'identified') {+ profileTypeFilter = `AND profileId LIKE '${AnalyticsService.PROFILE_PREFIX_USER}%'`}+ const allProfileDataCTE = this.buildProfilesListDataCTE(+ filtersQuery,+ profileTypeFilter,+ customEVFilterApplied,+ )+const query = `WITH ${allProfileDataCTE},profile_aggregated AS (@@ -5476,6 +5360,7 @@ export class AnalyticsService {sum(isPageview) AS pageviewsCount,countDistinct(psid) AS sessionsCount,sum(isEvent) AS eventsCount,+ sum(isError) AS errorsCount,min(created) AS firstSeen,max(created) AS lastSeen,any(cc) AS cc_agg,@@ -5484,25 +5369,13 @@ export class AnalyticsService {any(dv) AS dv_aggFROM all_profile_dataGROUP BY profileId- ),- profile_errors AS (- SELECT- profileId,- count() AS errorsCount- FROM errors- WHERE pid = {pid:FixedString(12)}- AND errors.created BETWEEN {groupFrom:String} AND {groupTo:String}- AND profileId IS NOT NULL- AND profileId != ''- ${profileTypeFilter}- GROUP BY profileId)SELECTpa.profileId AS profileId,pa.sessionsCount AS sessionsCount,pa.pageviewsCount AS pageviewsCount,pa.eventsCount AS eventsCount,- COALESCE(perr.errorsCount, 0) AS errorsCount,+ pa.errorsCount AS errorsCount,pa.firstSeen AS firstSeen,pa.lastSeen AS lastSeen,pa.cc_agg AS cc,@@ -5510,7 +5383,6 @@ export class AnalyticsService {pa.br_agg AS br,pa.dv_agg AS dvFROM profile_aggregated AS pa- LEFT JOIN profile_errors AS perr ON pa.profileId = perr.profileIdORDER BY pa.lastSeen DESCLIMIT {take:UInt32}OFFSET {skip:UInt32}@@ -5547,40 +5419,59 @@ export class AnalyticsService {AND profileId = {profileId:String}`- // Query avg duration from analytics table (more accurate than sessions table)+ // Prefer pageview timings, but fall back to sessions for event/error-only profiles.const queryAvgDuration = `- SELECT- avg(session_duration) AS avgDuration- FROM (+ WITH pageview_avg AS (SELECT- psid,- dateDiff('second', min(created), max(created)) AS session_duration- FROM analytics- WHERE pid = {pid:FixedString(12)}- AND profileId = {profileId:String}- AND psid IS NOT NULL- GROUP BY psid- HAVING session_duration > 0+ avgOrNull(session_duration) AS avgDuration+ FROM (+ SELECT+ psid,+ dateDiff('second', min(created), max(created)) AS session_duration+ FROM events+ WHERE pid = {pid:FixedString(12)}+ AND type = 'pageview'+ AND profileId = {profileId:String}+ AND psid IS NOT NULL+ GROUP BY psid+ HAVING session_duration > 0+ )+ ),+ sessions_avg AS (+ SELECT+ avgOrNull(session_duration) AS avgDuration+ FROM (+ SELECT+ psid,+ dateDiff('second', min(firstSeen), max(lastSeen)) AS session_duration+ FROM sessions FINAL+ WHERE pid = {pid:FixedString(12)}+ AND profileId = {profileId:String}+ GROUP BY psid+ HAVING session_duration > 0+ ))+ SELECT+ coalesce(nullIf(pageview_avg.avgDuration, 0), sessions_avg.avgDuration, 0) AS avgDuration+ FROM pageview_avg, sessions_avg`- // Query analytics for accurate pageview countconst queryPageviews = `SELECT count() AS pageviewsCount- FROM analytics+ FROM eventsWHERE pid = {pid:FixedString(12)}+ AND type = 'pageview'AND profileId = {profileId:String}`- // Query customEV for accurate events countconst queryEvents = `SELECT count() AS eventsCount- FROM customEV+ FROM eventsWHERE pid = {pid:FixedString(12)}+ AND type = 'custom_event'AND profileId = {profileId:String}`- // Query for device/location detailsconst queryDetails = `SELECTany(cc) AS cc,@@ -5592,9 +5483,10 @@ export class AnalyticsService {any(brv) AS brv,any(dv) AS dv,any(lc) AS lc- FROM analytics+ FROM eventsWHERE pid = {pid:FixedString(12)}AND profileId = {profileId:String}+ AND type IN ('pageview', 'custom_event', 'error')`// Query for total revenue from profile (only sales, not refunds)@@ -5679,8 +5571,9 @@ export class AnalyticsService {SELECTpg AS page,count() AS count- FROM analytics+ FROM eventsWHERE pid = {pid:FixedString(12)}+ AND type = 'pageview'AND profileId = {profileId:String}GROUP BY pgORDER BY count DESC@@ -5709,30 +5602,15 @@ export class AnalyticsService {.format('YYYY-MM-DD')const query = `- WITH all_activity AS (- SELECT- toDate(created) AS date,- 1 AS isPageview,- 0 AS isEvent- FROM analytics- WHERE pid = {pid:FixedString(12)}- AND profileId = {profileId:String}- AND created >= {startDate:Date}- UNION ALL- SELECT- toDate(created) AS date,- 0 AS isPageview,- 1 AS isEvent- FROM customEV- WHERE pid = {pid:FixedString(12)}- AND profileId = {profileId:String}- AND created >= {startDate:Date}- )SELECT- date,- sum(isPageview) AS pageviews,- sum(isEvent) AS events- FROM all_activity+ toDate(created) AS date,+ countIf(type = 'pageview') AS pageviews,+ countIf(type = 'custom_event') AS events+ FROM events+ WHERE pid = {pid:FixedString(12)}+ AND type IN ('pageview', 'custom_event')+ AND profileId = {profileId:String}+ AND created >= {startDate:Date}GROUP BY dateORDER BY date ASC`@@ -5767,8 +5645,9 @@ export class AnalyticsService {SELECT${timeBucketFunc}(toTimeZone(created, {timezone:String})) AS time,count() AS count- FROM analytics+ FROM eventsWHERE pid = {pid:FixedString(12)}+ AND type = 'pageview'AND profileId = {profileId:String}AND created BETWEEN {from:String} AND {to:String}GROUP BY time@@ -5779,8 +5658,9 @@ export class AnalyticsService {SELECT${timeBucketFunc}(toTimeZone(created, {timezone:String})) AS time,count() AS count- FROM customEV+ FROM eventsWHERE pid = {pid:FixedString(12)}+ AND type = 'custom_event'AND profileId = {profileId:String}AND created BETWEEN {from:String} AND {to:String}GROUP BY time@@ -5791,8 +5671,9 @@ export class AnalyticsService {SELECT${timeBucketFunc}(toTimeZone(created, {timezone:String})) AS time,count() AS count- FROM errors+ FROM eventsWHERE pid = {pid:FixedString(12)}+ AND type = 'error'AND profileId = {profileId:String}AND created BETWEEN {from:String} AND {to:String}GROUP BY time@@ -5853,70 +5734,26 @@ export class AnalyticsService {}}- async getProfileSessionsList(- pid: string,- profileId: string,+ private buildProfileSessionsEventsCTE(filtersQuery: string,- paramsData: any,- safeTimezone: string,- take = 30,- skip = 0,- customEVFilterApplied = false,- ): Promise<object[]> {- let allProfileEventsCTE: string+ customEVFilterApplied: boolean,+ ): string {+ const scopedSessionFilter = customEVFilterApplied+ ? `+ AND CAST(psid, 'String') IN (+ SELECT DISTINCT CAST(psid, 'String')+ FROM events+ WHERE pid = {pid:FixedString(12)}+ AND type = 'custom_event'+ AND profileId = {profileId:String}+ AND psid IS NOT NULL+ AND created BETWEEN {groupFrom:String} AND {groupTo:String}+ ${filtersQuery}+ )+ `+ : filtersQuery- if (customEVFilterApplied) {- allProfileEventsCTE = `- all_profile_events AS (- SELECT- all_events.psidCasted,- all_events.pid,- all_events.profileId,- all_events.cc,- all_events.os,- all_events.br,- all_events.created_tz- FROM (- SELECT- CAST(psid, 'String') AS psidCasted,- pid,- profileId,- cc,- os,- br,- toTimeZone(created, {timezone:String}) AS created_tz- FROM analytics- WHERE pid = {pid:FixedString(12)}- AND profileId = {profileId:String}- AND psid IS NOT NULL- AND created BETWEEN {groupFrom:String} AND {groupTo:String}- UNION ALL- SELECT- CAST(psid, 'String') AS psidCasted,- pid,- profileId,- cc,- os,- br,- toTimeZone(created, {timezone:String}) AS created_tz- FROM customEV- WHERE pid = {pid:FixedString(12)}- AND profileId = {profileId:String}- AND psid IS NOT NULL- AND created BETWEEN {groupFrom:String} AND {groupTo:String}- ) AS all_events- WHERE all_events.psidCasted IN (- SELECT DISTINCT CAST(psid, 'String')- FROM customEV- WHERE pid = {pid:FixedString(12)}- AND profileId = {profileId:String}- AND psid IS NOT NULL- AND created BETWEEN {groupFrom:String} AND {groupTo:String}- ${filtersQuery}- )- )`- } else {- allProfileEventsCTE = `+ return `all_profile_events AS (SELECTCAST(psid, 'String') AS psidCasted,@@ -5926,29 +5763,30 @@ export class AnalyticsService {os,br,toTimeZone(created, {timezone:String}) AS created_tz- FROM analytics- WHERE pid = {pid:FixedString(12)}- AND profileId = {profileId:String}- AND psid IS NOT NULL- AND created BETWEEN {groupFrom:String} AND {groupTo:String}- ${filtersQuery}- UNION ALL- SELECT- CAST(psid, 'String') AS psidCasted,- pid,- profileId,- cc,- os,- br,- toTimeZone(created, {timezone:String}) AS created_tz- FROM customEV+ FROM eventsWHERE pid = {pid:FixedString(12)}+ AND type IN ('pageview', 'custom_event', 'error')AND profileId = {profileId:String}AND psid IS NOT NULLAND created BETWEEN {groupFrom:String} AND {groupTo:String}- ${filtersQuery}+ ${scopedSessionFilter})`- }+ }++ async getProfileSessionsList(+ pid: string,+ profileId: string,+ filtersQuery: string,+ paramsData: any,+ safeTimezone: string,+ take = 30,+ skip = 0,+ customEVFilterApplied = false,+ ): Promise<object[]> {+ const allProfileEventsCTE = this.buildProfileSessionsEventsCTE(+ filtersQuery,+ customEVFilterApplied,+ )const query = `WITH ${allProfileEventsCTE},@@ -5970,8 +5808,9 @@ export class AnalyticsService {CAST(psid, 'String') AS psidCasted,pid,count() as count- FROM analytics+ FROM eventsWHERE pid = {pid:FixedString(12)}+ AND type = 'pageview'AND profileId = {profileId:String}AND psid IS NOT NULLAND created BETWEEN {groupFrom:String} AND {groupTo:String}@@ -5982,8 +5821,9 @@ export class AnalyticsService {CAST(psid, 'String') AS psidCasted,pid,count() as count- FROM customEV+ FROM eventsWHERE pid = {pid:FixedString(12)}+ AND type = 'custom_event'AND profileId = {profileId:String}AND psid IS NOT NULLAND created BETWEEN {groupFrom:String} AND {groupTo:String}@@ -5994,8 +5834,9 @@ export class AnalyticsService {CAST(psid, 'String') AS psidCasted,pid,count() as count- FROM errors+ FROM eventsWHERE pid = {pid:FixedString(12)}+ AND type = 'error'AND profileId = {profileId:String}AND psid IS NOT NULLAND created BETWEEN {groupFrom:String} AND {groupTo:String}@@ -6081,10 +5922,18 @@ export class AnalyticsService {count(DISTINCT psid) as sessions,status.statusFROM (- SELECT eid, name, message, filename, psid, profileId, toTimeZone(errors.created, {timezone:String}) AS created- FROM errors+ SELECT+ eid,+ error_name AS name,+ error_message AS message,+ error_filename AS filename,+ psid,+ profileId,+ toTimeZone(created, {timezone:String}) AS created+ FROM eventsWHERE pid = {pid:FixedString(12)}- AND errors.created BETWEEN {groupFrom:String} AND {groupTo:String}+ AND type = 'error'+ AND created BETWEEN {groupFrom:String} AND {groupTo:String}${filtersQuery}) AS errorsLEFT JOIN (@@ -6098,7 +5947,7 @@ export class AnalyticsService {${parsedOptions?.showResolved? ''- : "WHERE status.status = 'active' OR status.status = 'regressed'"+ : "WHERE status.status = 'active' OR status.status = 'regressed' OR status.status IS NULL"}GROUP BY errors.eid, status.statusORDER BY last_seen DESC@@ -6143,14 +5992,15 @@ export class AnalyticsService {FROM (SELECTeid,- name,- message,- filename,+ error_name AS name,+ error_message AS message,+ error_filename AS filename,colno,lineno,stackTrace- FROM errors+ FROM eventsWHERE pid = {pid:FixedString(12)}+ AND type = 'error'AND eid = {eid:FixedString(32)}AND created BETWEEN {groupFrom:String} AND {groupTo:String}ORDER BY created DESC@@ -6173,10 +6023,12 @@ export class AnalyticsService {max(created) AS last_seen,min(created) AS first_seen,count() AS count- FROM errors+ FROM eventsWHEREpid = {pid:FixedString(12)}- AND eid = {eid:FixedString(32)};+ AND type = 'error'+ AND eid = {eid:FixedString(32)}+ AND created BETWEEN {groupFrom:String} AND {groupTo:String};`const queryMetadata = `@@ -6188,9 +6040,10 @@ export class AnalyticsService {SELECTmeta.key,meta.value- FROM errors+ FROM eventsWHEREpid = {pid:FixedString(12)}+ AND type = 'error'AND eid = {eid:FixedString(32)}AND created BETWEEN {groupFrom:String} AND {groupTo:String})@@ -6237,8 +6090,8 @@ export class AnalyticsService {timeBucket,groupFrom,groupTo,- `FROM errors WHERE eid = {eid:FixedString(32)} AND created BETWEEN {groupFrom:String} AND {groupTo:String}`,- 'AND eid = {eid:FixedString(32)}',+ `FROM events WHERE type = 'error' AND pid = {pid:FixedString(12)} AND eid = {eid:FixedString(32)} AND created BETWEEN {groupFrom:String} AND {groupTo:String}`,+ 'AND pid = {pid:FixedString(12)} AND eid = {eid:FixedString(32)}',paramsData,safeTimezone,ChartRenderMode.PERIODICAL,@@ -6264,17 +6117,23 @@ export class AnalyticsService {groupTo: string,timeBucket: string,showResolved: boolean,+ sessionFiltersQuery = '',+ sessionFiltersParams: Record<string, unknown> = {},): Promise<any> {const resolvedFilter = showResolved? '': "AND (status.status = 'active' OR status.status = 'regressed' OR status.status IS NULL)"- // Get total sessions from analytics table for the time range+ // Get total sessions from all matching events for the time rangeconst queryTotalSessions = `SELECT count(DISTINCT psid) as totalSessions- FROM analytics+ FROM eventsWHERE pid = {pid:FixedString(12)}+ AND type IN ('pageview', 'custom_event', 'error')+ AND psid IS NOT NULL+ AND psid != ''AND created BETWEEN {groupFrom:String} AND {groupTo:String}+ ${sessionFiltersQuery}`// Get error stats: total errors, unique errors, affected sessions, affected users@@ -6284,7 +6143,7 @@ export class AnalyticsService {count(DISTINCT eid) as uniqueErrors,count(DISTINCT psid) as affectedSessions,count(DISTINCT profileId) as affectedUsers- FROM errors+ FROM events AS errorsLEFT JOIN (SELECT eid, argMax(status, updated) AS statusFROM error_statuses@@ -6292,6 +6151,7 @@ export class AnalyticsService {GROUP BY eid) AS status ON errors.eid = status.eidWHERE pid = {pid:FixedString(12)}+ AND type = 'error'AND created BETWEEN {groupFrom:String} AND {groupTo:String}${filtersQuery}${resolvedFilter}@@ -6301,12 +6161,12 @@ export class AnalyticsService {const queryMostFrequentError = `SELECTerrors.eid as eid,- any(errors.name) as name,- any(errors.message) as message,+ any(errors.error_name) as name,+ any(errors.error_message) as message,count(*) as count,count(DISTINCT errors.profileId) as usersAffected,max(errors.created) as lastSeen- FROM errors+ FROM events AS errorsLEFT JOIN (SELECT eid, argMax(status, updated) AS statusFROM error_statuses@@ -6314,6 +6174,7 @@ export class AnalyticsService {GROUP BY eid) AS status ON errors.eid = status.eidWHERE errors.pid = {pid:FixedString(12)}+ AND errors.type = 'error'AND errors.created BETWEEN {groupFrom:String} AND {groupTo:String}${filtersQuery}${resolvedFilter}@@ -6338,7 +6199,7 @@ export class AnalyticsService {SELECTprofileId,${timeBucketFunc}(toTimeZone(created, {timezone:String})) as tz_created- FROM errors+ FROM events AS errorsLEFT JOIN (SELECT eid, argMax(status, updated) AS statusFROM error_statuses@@ -6346,6 +6207,7 @@ export class AnalyticsService {GROUP BY eid) AS status ON errors.eid = status.eidWHERE pid = {pid:FixedString(12)}+ AND type = 'error'AND created BETWEEN {groupFrom:String} AND {groupTo:String}${filtersQuery}${resolvedFilter}@@ -6368,7 +6230,10 @@ export class AnalyticsService {clickhouse.query({query: queryTotalSessions,- query_params: paramsData.params,+ query_params: {+ ...paramsData.params,+ ...sessionFiltersParams,+ },}).then((resultSet) => resultSet.json<any>()).then(({ data }) => data[0]),@@ -6457,29 +6322,62 @@ export class AnalyticsService {): Promise<{ sessions: any[]; total: number }> {const queryCount = `SELECT count(DISTINCT psid) as total- FROM errors+ FROM eventsWHERE pid = {pid:FixedString(12)}+ AND type = 'error'AND eid = {eid:FixedString(32)}AND created BETWEEN {groupFrom:String} AND {groupTo:String}`const querySessions = `- SELECT DISTINCT+ SELECTCAST(errors.psid, 'String') as psid,- any(errors.profileId) as profileId,+ errors.profileId as profileId,any(analytics.cc) as cc,any(analytics.br) as br,any(analytics.os) as os,- min(errors.created) as firstErrorAt,- max(errors.created) as lastErrorAt,- count(*) as errorCount- FROM errors- LEFT JOIN analytics ON errors.psid = analytics.psid AND errors.pid = analytics.pid- WHERE errors.pid = {pid:FixedString(12)}- AND errors.eid = {eid:FixedString(32)}- AND errors.created BETWEEN {groupFrom:String} AND {groupTo:String}- GROUP BY errors.psid- ORDER BY lastErrorAt DESC+ errors.firstErrorAt as firstErrorAt,+ errors.lastErrorAt as lastErrorAt,+ errors.errorCount as errorCount+ FROM (+ SELECT+ pid,+ psid,+ any(profileId) AS profileId,+ min(created) AS firstErrorAt,+ max(created) AS lastErrorAt,+ count(*) AS errorCount+ FROM events+ WHERE pid = {pid:FixedString(12)}+ AND type = 'error'+ AND eid = {eid:FixedString(32)}+ AND created BETWEEN {groupFrom:String} AND {groupTo:String}+ GROUP BY pid, psid+ ) AS errors+ LEFT JOIN (+ SELECT+ pid,+ psid,+ any(cc) AS cc,+ any(br) AS br,+ any(os) AS os,+ countIf(type = 'pageview') AS pageviews+ FROM events+ WHERE pid = {pid:FixedString(12)}+ AND type IN ('pageview', 'error', 'custom_event')+ AND psid IS NOT NULL+ AND created BETWEEN {groupFrom:String} AND {groupTo:String}+ GROUP BY pid, psid+ ) AS analytics+ ON errors.psid = analytics.psid+ AND errors.pid = analytics.pid+ GROUP BY+ errors.psid,+ errors.profileId,+ errors.firstErrorAt,+ errors.lastErrorAt,+ errors.errorCount+ ORDER BY errors.lastErrorAt DESCLIMIT {take:UInt32}OFFSET {skip:UInt32}`@@ -6513,6 +6411,10 @@ export class AnalyticsService {}async validateEIDs(eids: string[], pid: string) {+ if (_isEmpty(eids)) {+ throw new BadRequestException('At least one error ID is required')+ }+const params = _reduce(eids,(acc, curr, index) => ({@@ -6522,7 +6424,7 @@ export class AnalyticsService {{},)- const query = `SELECT count(DISTINCT eid) as count FROM errors WHERE pid = {pid:FixedString(12)} AND eid IN (${_map(+ const query = `SELECT count(DISTINCT eid) as count FROM events WHERE pid = {pid:FixedString(12)} AND type = 'error' AND eid IN (${_map(params,(val, key) => `{${key}:FixedString(32)}`,).join(', ')})`@@ -6604,15 +6506,7 @@ export class AnalyticsService {events: number}> {const query = `- SELECT 'traffic' AS type, count(*) AS count FROM analytics- UNION ALL- SELECT 'customEV' AS type, count(*) AS count FROM customEV- UNION ALL- SELECT 'performance' AS type, count(*) AS count FROM performance- UNION ALL- SELECT 'captcha' AS type, count(*) AS count FROM captcha- UNION ALL- SELECT 'errors' AS type, count(*) AS count FROM errors+ SELECT count(*) AS count FROM events`const users = await this.userService.count({@@ -6732,12 +6626,13 @@ export class AnalyticsService {key,round(sum(${aggFunction}), 2) AS sum,round(avg(${aggFunction}), 2) AS avg- FROM customEV+ FROM eventsARRAY JOIN meta.key AS key, meta.value AS valueWHEREpid = {pid:FixedString(12)}+ AND type = 'custom_event'AND key = {metricKey:String}- AND ev = {customEventName:String}+ AND event_name = {customEventName:String}AND created BETWEEN {periodFormatted:String} AND {now:String}${kvQuery}${filtersQuery}@@ -6750,12 +6645,13 @@ export class AnalyticsService {key,round(sum(${aggFunction}), 2) AS sum,round(avg(${aggFunction}), 2) AS avg- FROM customEV+ FROM eventsARRAY JOIN meta.key AS key, meta.value AS valueWHEREpid = {pid:FixedString(12)}+ AND type = 'custom_event'AND key = {metricKey:String}- AND ev = {customEventName:String}+ AND event_name = {customEventName:String}AND created BETWEEN {periodSubtracted:String} AND {periodFormatted:String}${kvQuery}${filtersQuery}diff --git a/backend/apps/cloud/src/analytics/utils/transformers.ts b/backend/apps/cloud/src/analytics/utils/transformers.tsindex b420cdac8..7ec1ed491 100644--- a/backend/apps/cloud/src/analytics/utils/transformers.ts+++ b/backend/apps/cloud/src/analytics/utils/transformers.ts@@ -8,7 +8,7 @@ import utc from 'dayjs/plugin/utc'dayjs.extend(utc)const processMetaKV = (- rawMeta: Record<string, string>,+ rawMeta: Record<string, string> | null | undefined,): { 'meta.key': string[] | null; 'meta.value': string[] | null } => {if (!rawMeta || _isEmpty(rawMeta)) {return {@@ -23,10 +23,10 @@ const processMetaKV = (}}-interface TrafficTransformerOptions {- psid: string- profileId: string+interface CommonOptions {pid: string+ psid?: string | null+ profileId?: string | nullhost?: string | nullpg?: string | nulldv?: string | null@@ -52,262 +52,42 @@ interface TrafficTransformerOptions {meta?: Record<string, string> | null}-export const trafficTransformer = ({- psid,- profileId,- pid,- host,- pg,- dv,- br,- brv,- os,- osv,- lc,- ref,- so,- me,- ca,- te,- co,- cc,- rg,- rgc,- ct,- isp,- og,- ut,- ctp,- meta,-}: TrafficTransformerOptions) => {- return {- psid,- profileId,- pid,- host: host || null,- pg: pg || null,- dv: dv || null,- br: br || null,- brv: brv || null,- os: os || null,- osv: osv || null,- lc: lc || null,- ref: ref || null,- so: so || null,- me: me || null,- ca: ca || null,- te: te || null,- co: co || null,- cc: cc || null,- rg: rg || null,- rgc: rgc || null,- ct: ct || null,- isp: isp || null,- og: og || null,- ut: ut || null,- ctp: ctp || null,- ...processMetaKV(meta),- created: dayjs.utc().format('YYYY-MM-DD HH:mm:ss'),- }+interface PageviewOptions extends CommonOptions {+ type: 'pageview'}-interface CustomEventTransformerOptions {- psid: string- profileId: string- pid: string- host?: string | null+interface CustomEventOptions extends CommonOptions {+ type: 'custom_event'ev: string- pg?: string | null- dv?: string | null- br?: string | null- brv?: string | null- os?: string | null- osv?: string | null- lc?: string | null- ref?: string | null- so?: string | null- me?: string | null- ca?: string | null- te?: string | null- co?: string | null- cc?: string | null- rg?: string | null- rgc?: string | null- ct?: string | null- isp?: string | null- og?: string | null- ut?: string | null- ctp?: string | null- meta?: Record<string, string> | null-}--export const customEventTransformer = ({- psid,- profileId,- pid,- host,- ev,- pg,- dv,- br,- brv,- os,- osv,- lc,- ref,- so,- me,- ca,- te,- co,- cc,- rg,- rgc,- ct,- isp,- og,- ut,- ctp,- meta,-}: CustomEventTransformerOptions) => {- return {- psid,- profileId,- pid,- host: host || null,- ev,- pg: pg || null,- dv: dv || null,- br: br || null,- brv: brv || null,- os: os || null,- osv: osv || null,- lc: lc || null,- ref: ref || null,- so: so || null,- me: me || null,- ca: ca || null,- te: te || null,- co: co || null,- cc: cc || null,- rg: rg || null,- rgc: rgc || null,- ct: ct || null,- isp: isp || null,- og: og || null,- ut: ut || null,- ctp: ctp || null,- ...processMetaKV(meta),- created: dayjs.utc().format('YYYY-MM-DD HH:mm:ss'),- }}-interface ErrorEventTransformerOptions {- psid: string- profileId: string+interface ErrorEventOptions extends CommonOptions {+ type: 'error'eid: string- pid: string- host?: string | null- pg?: string | null- dv?: string | null- br?: string | null- brv?: string | null- os?: string | null- osv?: string | null- lc?: string | null- cc?: string | null- rg?: string | null- rgc?: string | null- ct?: string | null- isp?: string | null- og?: string | null- ut?: string | null- ctp?: string | nullname?: string | nullmessage?: string | nulllineno?: number | nullcolno?: number | nullfilename?: string | nullstackTrace?: string | null- meta?: Record<string, string> | null-}--export const errorEventTransformer = ({- psid,- profileId,- eid,- pid,- host,- pg,- dv,- br,- brv,- os,- osv,- lc,- cc,- rg,- rgc,- ct,- isp,- og,- ut,- ctp,- name,- message,- lineno,- colno,- filename,- stackTrace,- meta,-}: ErrorEventTransformerOptions) => {- return {- psid,- profileId,- eid,- pid,- host: host || null,- pg: pg || null,- dv: dv || null,- br: br || null,- brv: brv || null,- os: os || null,- osv: osv || null,- lc: lc || null,- cc: cc || null,- rg: rg || null,- rgc: rgc || null,- ct: ct || null,- isp: isp || null,- og: og || null,- ut: ut || null,- ctp: ctp || null,- name: name || null,- message: message || null,- lineno: lineno || null,- colno: colno || null,- filename: filename || null,- stackTrace: stackTrace || null,- ...processMetaKV(meta),- created: dayjs.utc().format('YYYY-MM-DD HH:mm:ss'),- }}-interface PerformanceTransformerOptions {- pid: string- host?: string | null- pg?: string | null- dv?: string | null- br?: string | null- brv?: string | null- cc?: string | null- rg?: string | null- rgc?: string | null- ct?: string | null- isp?: string | null- og?: string | null- ut?: string | null- ctp?: string | null+interface PerformanceOptions extends Omit<+ CommonOptions,+ | 'psid'+ | 'profileId'+ | 'os'+ | 'osv'+ | 'lc'+ | 'ref'+ | 'so'+ | 'me'+ | 'ca'+ | 'te'+ | 'co'+ | 'meta'+> {+ type: 'performance'dns: numbertls: numberconn: number@@ -318,53 +98,131 @@ interface PerformanceTransformerOptions {ttfb: number}-export const performanceTransformer = ({- pid,- host,- pg,- dv,- br,- brv,- cc,- rg,- rgc,- ct,- isp,- og,- ut,- ctp,- dns,- tls,- conn,- response,- render,- domLoad,- pageLoad,- ttfb,-}: PerformanceTransformerOptions) => {+interface CaptchaOptions {+ type: 'captcha'+ pid: string+ dv?: string | null+ br?: string | null+ os?: string | null+ cc?: string | null+ timestamp?: number+}++type EventTransformerOptions =+ | PageviewOptions+ | CustomEventOptions+ | ErrorEventOptions+ | PerformanceOptions+ | CaptchaOptions++const buildCommon = (opts: CommonOptions) => ({+ pid: opts.pid,+ psid: opts.psid ?? null,+ profileId: opts.profileId ?? null,+ host: opts.host || null,+ pg: opts.pg || null,+ dv: opts.dv || null,+ br: opts.br || null,+ brv: opts.brv || null,+ os: opts.os || null,+ osv: opts.osv || null,+ lc: opts.lc || null,+ ref: opts.ref || null,+ so: opts.so || null,+ me: opts.me || null,+ ca: opts.ca || null,+ te: opts.te || null,+ co: opts.co || null,+ cc: opts.cc || null,+ rg: opts.rg || null,+ rgc: opts.rgc || null,+ ct: opts.ct || null,+ isp: opts.isp || null,+ og: opts.og || null,+ ut: opts.ut || null,+ ctp: opts.ctp || null,+ ...processMetaKV(opts.meta),+})++// Single transformer for all event kinds; the discriminator is `type`.+// Renames at the storage boundary: ev -> event_name, name -> error_name,+// message -> error_message, filename -> error_filename. DTO and API field+// names stay as the legacy short forms.+export const eventTransformer = (opts: EventTransformerOptions) => {+ const created = dayjs.utc().format('YYYY-MM-DD HH:mm:ss')++ if (opts.type === 'pageview') {+ return {+ type: 'pageview' as const,+ ...buildCommon(opts),+ created,+ }+ }++ if (opts.type === 'custom_event') {+ return {+ type: 'custom_event' as const,+ ...buildCommon(opts),+ event_name: opts.ev,+ created,+ }+ }++ if (opts.type === 'error') {+ return {+ type: 'error' as const,+ ...buildCommon(opts),+ eid: opts.eid,+ error_name: opts.name || null,+ error_message: opts.message || null,+ stackTrace: opts.stackTrace || null,+ lineno: opts.lineno ?? null,+ colno: opts.colno ?? null,+ error_filename: opts.filename || null,+ created,+ }+ }++ if (opts.type === 'performance') {+ return {+ type: 'performance' as const,+ pid: opts.pid,+ host: opts.host || null,+ pg: opts.pg || null,+ dv: opts.dv || null,+ br: opts.br || null,+ brv: opts.brv || null,+ cc: opts.cc || null,+ rg: opts.rg || null,+ rgc: opts.rgc || null,+ ct: opts.ct || null,+ isp: opts.isp || null,+ og: opts.og || null,+ ut: opts.ut || null,+ ctp: opts.ctp || null,+ dns: _round(opts.dns),+ tls: _round(opts.tls),+ conn: _round(opts.conn),+ response: _round(opts.response),+ render: _round(opts.render),+ domLoad: _round(opts.domLoad),+ pageLoad: _round(opts.pageLoad),+ ttfb: _round(opts.ttfb),+ created,+ }+ }++ // captchareturn {- pid,- host: host || null,- pg: pg || null,- dv: dv || null,- br: br || null,- brv: brv || null,- cc: cc || null,- rg: rg || null,- rgc: rgc || null,- ct: ct || null,- isp: isp || null,- og: og || null,- ut: ut || null,- ctp: ctp || null,- dns: _round(dns),- tls: _round(tls),- conn: _round(conn),- response: _round(response),- render: _round(render),- domLoad: _round(domLoad),- pageLoad: _round(pageLoad),- ttfb: _round(ttfb),- created: dayjs.utc().format('YYYY-MM-DD HH:mm:ss'),+ type: 'captcha' as const,+ pid: opts.pid,+ dv: opts.dv || null,+ br: opts.br || null,+ os: opts.os || null,+ cc: opts.cc || null,+ created:+ typeof opts.timestamp === 'number'+ ? dayjs.utc(opts.timestamp).format('YYYY-MM-DD HH:mm:ss')+ : created,}}diff --git a/backend/apps/cloud/src/captcha/captcha.service.ts b/backend/apps/cloud/src/captcha/captcha.service.tsindex 921e6d7b7..99ac3516c 100644--- a/backend/apps/cloud/src/captcha/captcha.service.ts+++ b/backend/apps/cloud/src/captcha/captcha.service.ts@@ -17,7 +17,7 @@ import {} from '../common/constants'import { getIPDetails } from '../common/utils'import { GeneratedChallenge } from './interfaces/generated-captcha'-import { captchaTransformer } from './utils/transformers'+import { eventTransformer } from '../analytics/utils/transformers'import { clickhouse } from '../common/integrations/clickhouse'dayjs.extend(utc)@@ -152,18 +152,19 @@ export class CaptchaService {const osName = ua.os.nameconst { country } = getIPDetails(ip)- const transformed = captchaTransformer(+ const transformed = eventTransformer({+ type: 'captcha',pid,- deviceType,- browserName,- osName,- country,+ dv: deviceType,+ br: browserName,+ os: osName,+ cc: country,timestamp,- )+ })try {await clickhouse.insert({- table: 'captcha',+ table: 'events',format: 'JSONEachRow',values: [transformed],clickhouse_settings: {diff --git a/backend/apps/cloud/src/captcha/utils/transformers.ts b/backend/apps/cloud/src/captcha/utils/transformers.tsdeleted file mode 100644index 3a0d21d43..000000000--- a/backend/apps/cloud/src/captcha/utils/transformers.ts+++ /dev/null@@ -1,22 +0,0 @@-import dayjs from 'dayjs'-import utc from 'dayjs/plugin/utc'--dayjs.extend(utc)--export const captchaTransformer = (- pid: string,- dv: string | null,- br: string | null,- os: string | null,- cc: string | null,- timestamp: number,-) => {- return {- pid,- dv: dv || null,- br: br || null,- os: os || null,- cc: cc || null,- created: dayjs.utc(timestamp).format('YYYY-MM-DD HH:mm:ss'),- }-}diff --git a/backend/apps/cloud/src/data-import/data-import.processor.ts b/backend/apps/cloud/src/data-import/data-import.processor.tsindex 18799910d..019945a24 100644--- a/backend/apps/cloud/src/data-import/data-import.processor.ts+++ b/backend/apps/cloud/src/data-import/data-import.processor.ts@@ -107,8 +107,7 @@ export class DataImportProcessor extends WorkerHost {let minDate: string | null = nulllet maxDate: string | null = null- const analyticsBatch: Record<string, unknown>[] = []- const customEVBatch: Record<string, unknown>[] = []+ const eventsBatch: Record<string, unknown>[] = []const context = isApiBased? {@@ -142,28 +141,17 @@ export class DataImportProcessor extends WorkerHost {if (!maxDate || created > maxDate) maxDate = created}- if (row.table === 'analytics') {- analyticsBatch.push(row.data)- } else {- customEVBatch.push(row.data)- }-- if (analyticsBatch.length >= BATCH_SIZE) {- await this.flushBatch('analytics', analyticsBatch)- importedRows += analyticsBatch.length- analyticsBatch.length = 0- }+ eventsBatch.push({ type: row.type, ...row.data })- if (customEVBatch.length >= BATCH_SIZE) {- await this.flushBatch('customEV', customEVBatch)- importedRows += customEVBatch.length- customEVBatch.length = 0+ if (eventsBatch.length >= BATCH_SIZE) {+ await this.flushBatch(eventsBatch)+ importedRows += eventsBatch.length+ eventsBatch.length = 0}if (totalRows % 10000 === 0) {const progress = {- importedRows:- importedRows + analyticsBatch.length + customEVBatch.length,+ importedRows: importedRows + eventsBatch.length,totalRows,}@@ -181,14 +169,9 @@ export class DataImportProcessor extends WorkerHost {}}- if (analyticsBatch.length > 0) {- await this.flushBatch('analytics', analyticsBatch)- importedRows += analyticsBatch.length- }-- if (customEVBatch.length > 0) {- await this.flushBatch('customEV', customEVBatch)- importedRows += customEVBatch.length+ if (eventsBatch.length > 0) {+ await this.flushBatch(eventsBatch)+ importedRows += eventsBatch.length}await this.dataImportService.markCompleted(id, {@@ -235,12 +218,9 @@ export class DataImportProcessor extends WorkerHost {}}- private async flushBatch(- table: 'analytics' | 'customEV',- batch: Record<string, unknown>[],- ): Promise<void> {+ private async flushBatch(batch: Record<string, unknown>[]): Promise<void> {await clickhouse.insert({- table: `${CLICKHOUSE_DB}.${table}`,+ table: `${CLICKHOUSE_DB}.events`,values: batch,format: 'JSONEachRow',})diff --git a/backend/apps/cloud/src/data-import/data-import.service.ts b/backend/apps/cloud/src/data-import/data-import.service.tsindex 63f60b59e..834b53086 100644--- a/backend/apps/cloud/src/data-import/data-import.service.ts+++ b/backend/apps/cloud/src/data-import/data-import.service.ts@@ -161,11 +161,7 @@ export class DataImportService {importId: number,): Promise<void> {await clickhouse.command({- query: `ALTER TABLE ${CLICKHOUSE_DB}.analytics DELETE WHERE pid = {pid:FixedString(12)} AND importID = {importID:UInt8}`,- query_params: { pid: projectId, importID: importId },- })- await clickhouse.command({- query: `ALTER TABLE ${CLICKHOUSE_DB}.customEV DELETE WHERE pid = {pid:FixedString(12)} AND importID = {importID:UInt8}`,+ query: `ALTER TABLE ${CLICKHOUSE_DB}.events DELETE WHERE pid = {pid:FixedString(12)} AND importID = {importID:UInt8} AND type IN ('pageview', 'custom_event')`,query_params: { pid: projectId, importID: importId },})}@@ -207,7 +203,7 @@ export class DataImportService {): Promise<boolean> {const result = await clickhouse.query({- query: `SELECT count() as cnt FROM ${CLICKHOUSE_DB}.analytics WHERE pid = {pid:FixedString(12)} AND importID IS NOT NULL AND created >= {from:DateTime} AND created <= {to:DateTime} LIMIT 1`,+ query: `SELECT count() as cnt FROM ${CLICKHOUSE_DB}.events WHERE pid = {pid:FixedString(12)} AND type = 'pageview' AND importID IS NOT NULL AND created >= {from:DateTime} AND created <= {to:DateTime} LIMIT 1`,query_params: { pid: projectId, from, to },}).then((resultSet) => resultSet.json<{ cnt: string }>())diff --git a/backend/apps/cloud/src/data-import/mappers/fathom.mapper.ts b/backend/apps/cloud/src/data-import/mappers/fathom.mapper.tsindex f74c72122..b9dce040d 100644--- a/backend/apps/cloud/src/data-import/mappers/fathom.mapper.ts+++ b/backend/apps/cloud/src/data-import/mappers/fathom.mapper.ts@@ -206,13 +206,13 @@ export class FathomMapper implements ImportMapper {if (fileType === 'pageviews') {for (let i = 0; i < count; i++) {- yield { table: 'analytics', data: baseData }+ yield { type: 'pageview', data: baseData }}} else {const eventName = truncate(normalizeNull(row.event_name), 256) || ''- const evData = { ...baseData, ev: eventName }+ const evData = { ...baseData, event_name: eventName }for (let i = 0; i < count; i++) {- yield { table: 'customEV', data: evData }+ yield { type: 'custom_event', data: evData }}}}diff --git a/backend/apps/cloud/src/data-import/mappers/ga4.mapper.ts b/backend/apps/cloud/src/data-import/mappers/ga4.mapper.tsindex 5cd625c58..2ee657aa9 100644--- a/backend/apps/cloud/src/data-import/mappers/ga4.mapper.ts+++ b/backend/apps/cloud/src/data-import/mappers/ga4.mapper.ts@@ -357,7 +357,7 @@ export class Ga4Mapper implements ImportMapper {}for (let i = 0; i < count; i++) {- yield { table: 'analytics', data }+ yield { type: 'pageview', data }}}@@ -441,7 +441,7 @@ export class Ga4Mapper implements ImportMapper {pid,host: truncate(host, 253),pg: truncate(pagePath, 2048),- ev: truncate(eventName, 256),+ event_name: truncate(eventName, 256),dv: device,br: truncate(browser, 30),brv: null,@@ -465,7 +465,7 @@ export class Ga4Mapper implements ImportMapper {}for (let i = 0; i < count; i++) {- yield { table: 'customEV', data }+ yield { type: 'custom_event', data }}}diff --git a/backend/apps/cloud/src/data-import/mappers/mapper.interface.ts b/backend/apps/cloud/src/data-import/mappers/mapper.interface.tsindex 0351b00b0..cf19c69c8 100644--- a/backend/apps/cloud/src/data-import/mappers/mapper.interface.ts+++ b/backend/apps/cloud/src/data-import/mappers/mapper.interface.ts@@ -6,7 +6,7 @@ export class ImportError extends Error {}export interface AnalyticsImportRow {- table: 'analytics' | 'customEV'+ type: 'pageview' | 'custom_event'data: Record<string, unknown>}diff --git a/backend/apps/cloud/src/data-import/mappers/plausible.mapper.ts b/backend/apps/cloud/src/data-import/mappers/plausible.mapper.tsindex 99141bb58..f0c5926d3 100644--- a/backend/apps/cloud/src/data-import/mappers/plausible.mapper.ts+++ b/backend/apps/cloud/src/data-import/mappers/plausible.mapper.ts@@ -859,7 +859,7 @@ export class PlausibleMapper implements ImportMapper {created,})- yield { table: 'analytics', data }+ yield { type: 'pageview', data }}}@@ -889,8 +889,8 @@ export class PlausibleMapper implements ImportMapper {page: null,created,})- data.ev = truncate(ev.name, 256) || ''- yield { table: 'customEV', data }+ data.event_name = truncate(ev.name, 256) || ''+ yield { type: 'custom_event', data }}}}diff --git a/backend/apps/cloud/src/data-import/mappers/simple-analytics.mapper.ts b/backend/apps/cloud/src/data-import/mappers/simple-analytics.mapper.tsindex 880b863b9..b95e37380 100644--- a/backend/apps/cloud/src/data-import/mappers/simple-analytics.mapper.ts+++ b/backend/apps/cloud/src/data-import/mappers/simple-analytics.mapper.ts@@ -176,11 +176,11 @@ export class SimpleAnalyticsMapper implements ImportMapper {const datapoint = normalizeNull(row.datapoint)if (!datapoint || datapoint === 'pageview') {- yield { table: 'analytics', data: baseData }+ yield { type: 'pageview', data: baseData }} else {yield {- table: 'customEV',- data: { ...baseData, ev: truncate(datapoint, 256) || '' },+ type: 'custom_event',+ data: { ...baseData, event_name: truncate(datapoint, 256) || '' },}}}diff --git a/backend/apps/cloud/src/data-import/mappers/umami.mapper.ts b/backend/apps/cloud/src/data-import/mappers/umami.mapper.tsindex 192a9638b..3d8c9d9b8 100644--- a/backend/apps/cloud/src/data-import/mappers/umami.mapper.ts+++ b/backend/apps/cloud/src/data-import/mappers/umami.mapper.ts@@ -372,12 +372,12 @@ export class UmamiMapper implements ImportMapper {}if (eventType === '1') {- yield { table: 'analytics', data: baseData }+ yield { type: 'pageview', data: baseData }} else {const eventName = truncate(normalizeNull(row.event_name), 256) || ''yield {- table: 'customEV',- data: { ...baseData, ev: eventName },+ type: 'custom_event',+ data: { ...baseData, event_name: eventName },}}}diff --git a/backend/apps/cloud/src/experiment/experiment.controller.ts b/backend/apps/cloud/src/experiment/experiment.controller.tsindex 52f230267..869176359 100644--- a/backend/apps/cloud/src/experiment/experiment.controller.ts+++ b/backend/apps/cloud/src/experiment/experiment.controller.ts@@ -60,6 +60,7 @@ import {} from './dto/experiment.dto'import { ExperimentService } from './experiment.service'import { GoalService } from '../goal/goal.service'+import { Goal, GoalMatchType, GoalType } from '../goal/entity/goal.entity'import { FeatureFlagService } from '../feature-flag/feature-flag.service'import {FeatureFlag,@@ -73,6 +74,15 @@ import { Pagination } from '../common/pagination'const EXPERIMENTS_MAXIMUM = 20 // Maximum experiments per projectconst FEATURE_FLAG_KEY_REGEX = /^[a-zA-Z0-9_-]+$/+type GoalEventConditions = {+ eventType: 'pageview' | 'custom_event'+ matchColumn: 'pg' | 'event_name'+ matchCondition: string+ metaCondition: string+ metaParams: Record<string, string>+ goalValue: string+}+@ApiTags('Experiment')@Controller(['experiment', 'v1/experiment'])export class ExperimentController {@@ -881,11 +891,14 @@ export class ExperimentController {timeBucketParam || getLowestPossibleTimeBucket(period, from, to)let allowedTimeBucketForPeriodAll: TimeBucketType[] | undefinedlet diff: number | undefined+ const goalConditions = experiment.goal+ ? this.buildGoalEventConditions(experiment.goal)+ : nullif (period === 'all') {const res = await this.analyticsService.calculateTimeBucketForAllTime(experiment.project.id,- 'analytics',+ goalConditions?.eventType || 'pageview',)diff = res.diff@@ -938,43 +951,21 @@ export class ExperimentController {}let conversionsData: { variantKey: string; conversions: number }[] = []- if (experiment.goal) {- const goalType = experiment.goal.type- const table = goalType === 'custom_event' ? 'customEV' : 'analytics'- const matchColumn = goalType === 'custom_event' ? 'ev' : 'pg'- const goalValue = experiment.goal.value || ''-- let matchCondition = ''- if (experiment.goal.matchType === 'exact') {- matchCondition = `c.${matchColumn} = {goalValue:String}`- } else {- matchCondition = `c.${matchColumn} ILIKE concat('%', {goalValue:String}, '%')`- }-- let metaCondition = ''- const metaParams: Record<string, string> = {}- const metadataFilters = experiment.goal.metadataFilters- if (metadataFilters && metadataFilters.length > 0) {- const conditions: string[] = []- metadataFilters.forEach((filter, index) => {- const keyParam = `metaKey${index}`- const valueParam = `metaValue${index}`- metaParams[keyParam] = filter.key- metaParams[valueParam] = filter.value- conditions.push(- `has(c.meta.key, {${keyParam}:String}) AND c.meta.value[indexOf(c.meta.key, {${keyParam}:String})] = {${valueParam}:String}`,- )- })-- metaCondition = `AND (${conditions.join(' AND ')})`- }+ if (goalConditions) {+ const {+ eventType,+ matchCondition,+ metaCondition,+ metaParams,+ goalValue,+ } = goalConditionsconst conversionsQuery = `SELECTe.variantKey,uniqExact(e.profileId) as conversionsFROM experiment_exposures e- INNER JOIN ${table} c ON e.pid = c.pid AND e.profileId = assumeNotNull(c.profileId)+ INNER JOIN events c ON e.pid = c.pid AND e.profileId = assumeNotNull(c.profileId) AND c.type = '${eventType}'WHEREe.pid = {pid:FixedString(12)}AND e.experimentId = {experimentId:String}@@ -1105,6 +1096,50 @@ export class ExperimentController {}}+ private buildGoalEventConditions(goal: Goal): GoalEventConditions {+ const eventType =+ goal.type === GoalType.CUSTOM_EVENT ? 'custom_event' : 'pageview'+ const matchColumn =+ goal.type === GoalType.CUSTOM_EVENT ? 'event_name' : 'pg'+ const goalValue = goal.value || ''+ const metaParams: Record<string, string> = {}++ let matchCondition = ''+ if (goal.matchType === GoalMatchType.EXACT) {+ matchCondition = `c.${matchColumn} = {goalValue:String}`+ } else if (goalValue.trim() === '') {+ matchCondition = '1=0'+ } else {+ matchCondition = `c.${matchColumn} ILIKE concat('%', {goalValue:String}, '%')`+ }++ let metaCondition = ''+ if (goal.metadataFilters && goal.metadataFilters.length > 0) {+ const conditions: string[] = []++ goal.metadataFilters.forEach((filter, index) => {+ const keyParam = `metaKey${index}`+ const valueParam = `metaValue${index}`+ metaParams[keyParam] = filter.key+ metaParams[valueParam] = filter.value+ conditions.push(+ `has(c.meta.key, {${keyParam}:String}) AND c.meta.value[indexOf(c.meta.key, {${keyParam}:String})] = {${valueParam}:String}`,+ )+ })++ metaCondition = `AND (${conditions.join(' AND ')})`+ }++ return {+ eventType,+ matchColumn,+ matchCondition,+ metaCondition,+ metaParams,+ goalValue,+ }+ }+/*** Generate time-series chart data for experiment win probabilities*/@@ -1177,39 +1212,18 @@ export class ExperimentController {let conversionsData: any[] = []if (experiment.goal) {- const goalType = experiment.goal.type- const table = goalType === 'custom_event' ? 'customEV' : 'analytics'- const matchColumn = goalType === 'custom_event' ? 'ev' : 'pg'- const goalValue = experiment.goal.value || ''+ const {+ eventType,+ matchCondition,+ metaCondition,+ metaParams,+ goalValue,+ } = this.buildGoalEventConditions(experiment.goal)const conversionsDateColumnsSelect = this.getTimeBucketDateColumnsSelect(timeBucket,'firstConversion',)- let matchCondition = ''- if (experiment.goal.matchType === 'exact') {- matchCondition = `c.${matchColumn} = {goalValue:String}`- } else {- matchCondition = `c.${matchColumn} ILIKE concat('%', {goalValue:String}, '%')`- }-- let metaCondition = ''- const metaParams: Record<string, string> = {}- const metadataFilters = experiment.goal.metadataFilters- if (metadataFilters && metadataFilters.length > 0) {- const conditions: string[] = []- metadataFilters.forEach((filter, index) => {- const keyParam = `metaKey${index}`- const valueParam = `metaValue${index}`- metaParams[keyParam] = filter.key- metaParams[valueParam] = filter.value- conditions.push(- `has(c.meta.key, {${keyParam}:String}) AND c.meta.value[indexOf(c.meta.key, {${keyParam}:String})] = {${valueParam}:String}`,- )- })- metaCondition = `AND (${conditions.join(' AND ')})`- }-const conversionsQuery = `SELECT${conversionsDateColumnsSelect},@@ -1221,7 +1235,7 @@ export class ExperimentController {e.profileId as profileId,min(c.created) as firstConversionFROM experiment_exposures e- INNER JOIN ${table} c ON e.pid = c.pid AND e.profileId = assumeNotNull(c.profileId)+ INNER JOIN events c ON e.pid = c.pid AND e.profileId = assumeNotNull(c.profileId) AND c.type = '${eventType}'WHEREe.pid = {pid:FixedString(12)}AND e.experimentId = {experimentId:String}diff --git a/backend/apps/cloud/src/goal/goal.controller.ts b/backend/apps/cloud/src/goal/goal.controller.tsindex 657554968..ed2807bee 100644--- a/backend/apps/cloud/src/goal/goal.controller.ts+++ b/backend/apps/cloud/src/goal/goal.controller.ts@@ -332,38 +332,38 @@ export class GoalController {})}- private buildGoalMatchCondition(- goal: Goal,- _table: 'analytics' | 'customEV',- ): { condition: string; params: Record<string, string> } {+ private buildGoalMatchCondition(goal: Goal): {+ condition: string+ params: Record<string, string>+ } {const params: Record<string, string> = {}if (goal.type === GoalType.CUSTOM_EVENT) {- // For custom events, match the event nameif (goal.matchType === GoalMatchType.EXACT) {params.goalValue = goal.value || ''- return { condition: `ev = {goalValue:String}`, params }- } else {- // Contains match- params.goalValue = goal.value || ''- return {- condition: `ev ILIKE concat('%', {goalValue:String}, '%')`,- params,- }+ return { condition: `event_name = {goalValue:String}`, params }}- } else {- // For pageview goals, match the page path- if (goal.matchType === GoalMatchType.EXACT) {- params.goalValue = goal.value || ''- return { condition: `pg = {goalValue:String}`, params }- } else {- // Contains match- params.goalValue = goal.value || ''- return {- condition: `pg ILIKE concat('%', {goalValue:String}, '%')`,- params,- }+ if ((goal.value || '').trim() === '') {+ return { condition: '1=0', params: {} }}+ params.goalValue = goal.value || ''+ return {+ condition: `event_name ILIKE concat('%', {goalValue:String}, '%')`,+ params,+ }+ }++ if (goal.matchType === GoalMatchType.EXACT) {+ params.goalValue = goal.value || ''+ return { condition: `pg = {goalValue:String}`, params }+ }+ if ((goal.value || '').trim() === '') {+ return { condition: '1=0', params: {} }+ }+ params.goalValue = goal.value || ''+ return {+ condition: `pg ILIKE concat('%', {goalValue:String}, '%')`,+ params,}}@@ -429,9 +429,10 @@ export class GoalController {safeTimezone,)- const table = goal.type === GoalType.CUSTOM_EVENT ? 'customEV' : 'analytics'+ const goalType =+ goal.type === GoalType.CUSTOM_EVENT ? 'custom_event' : 'pageview'const { condition: matchCondition, params: matchParams } =- this.buildGoalMatchCondition(goal, table)+ this.buildGoalMatchCondition(goal)const { condition: metaCondition, params: metaParams } =this.buildMetadataCondition(goal)@@ -440,9 +441,10 @@ export class GoalController {SELECTcount(*) as conversions,uniqExact(psid) as uniqueSessions- FROM ${table}+ FROM eventsWHEREpid = {pid:FixedString(12)}+ AND type = {goalType:String}AND ${matchCondition}${metaCondition}AND created BETWEEN {groupFrom:String} AND {groupTo:String}@@ -450,6 +452,7 @@ export class GoalController {const queryParams = {pid: goal.project.id,+ goalType,groupFrom: groupFromUTC,groupTo: groupToUTC,...matchParams,@@ -468,9 +471,10 @@ export class GoalController {// Get total unique sessions for conversion rateconst totalSessionsQuery = `SELECT uniqExact(psid) as totalSessions- FROM analytics+ FROM eventsWHEREpid = {pid:FixedString(12)}+ AND type = 'pageview'AND created BETWEEN {groupFrom:String} AND {groupTo:String}`@@ -649,7 +653,8 @@ export class GoalController {safeTimezone,)- const table = goal.type === GoalType.CUSTOM_EVENT ? 'customEV' : 'analytics'+ const goalType =+ goal.type === GoalType.CUSTOM_EVENT ? 'custom_event' : 'pageview'const timeBucketFunc = Object.prototype.hasOwnProperty.call(timeBucketConversion,resolvedTimeBucket,@@ -659,7 +664,7 @@ export class GoalController {const [selector, groupBy] = this.getGroupSubquery(resolvedTimeBucket)const { condition: matchCondition, params: matchParams } =- this.buildGoalMatchCondition(goal, table)+ this.buildGoalMatchCondition(goal)const { condition: metaCondition, params: metaParams } =this.buildMetadataCondition(goal)@@ -671,9 +676,10 @@ export class GoalController {FROM (SELECT *,${timeBucketFunc}(toTimeZone(created, {timezone:String})) as tz_created- FROM ${table}+ FROM eventsWHEREpid = {pid:FixedString(12)}+ AND type = {goalType:String}AND ${matchCondition}${metaCondition}AND created BETWEEN {groupFrom:String} AND {groupTo:String}@@ -684,6 +690,7 @@ export class GoalController {const queryParams = {pid: goal.project.id,+ goalType,groupFrom: groupFromUTC,groupTo: groupToUTC,timezone: safeTimezone,diff --git a/backend/apps/cloud/src/project/project.controller.ts b/backend/apps/cloud/src/project/project.controller.tsindex 5e54aab17..13711340a 100644--- a/backend/apps/cloud/src/project/project.controller.ts+++ b/backend/apps/cloud/src/project/project.controller.ts@@ -745,12 +745,8 @@ export class ProjectController {}const queries = [- 'ALTER TABLE analytics DELETE WHERE pid={pid:FixedString(12)}',- 'ALTER TABLE customEV DELETE WHERE pid={pid:FixedString(12)}',- 'ALTER TABLE performance DELETE WHERE pid={pid:FixedString(12)}',- 'ALTER TABLE errors DELETE WHERE pid={pid:FixedString(12)}',+ 'ALTER TABLE events DELETE WHERE pid={pid:FixedString(12)}','ALTER TABLE error_statuses DELETE WHERE pid={pid:FixedString(12)}',- 'ALTER TABLE captcha DELETE WHERE pid={pid:FixedString(12)}',]try {@@ -1607,12 +1603,8 @@ export class ProjectController {}const queries = [- 'ALTER TABLE analytics DELETE WHERE pid={pid:FixedString(12)}',- 'ALTER TABLE customEV DELETE WHERE pid={pid:FixedString(12)}',- 'ALTER TABLE performance DELETE WHERE pid={pid:FixedString(12)}',- 'ALTER TABLE errors DELETE WHERE pid={pid:FixedString(12)}',+ 'ALTER TABLE events DELETE WHERE pid={pid:FixedString(12)}','ALTER TABLE error_statuses DELETE WHERE pid={pid:FixedString(12)}',- 'ALTER TABLE captcha DELETE WHERE pid={pid:FixedString(12)}',]try {diff --git a/backend/apps/cloud/src/project/project.service.ts b/backend/apps/cloud/src/project/project.service.tsindex 5e5c29c63..f974e49c8 100644--- a/backend/apps/cloud/src/project/project.service.ts+++ b/backend/apps/cloud/src/project/project.service.ts@@ -489,12 +489,8 @@ export class ProjectService {const projectIds = _map(projects, 'id')const queries = [- 'ALTER TABLE analytics DELETE WHERE pid IN ({pids:Array(FixedString(12))})',- 'ALTER TABLE customEV DELETE WHERE pid IN ({pids:Array(FixedString(12))})',- 'ALTER TABLE performance DELETE WHERE pid IN ({pids:Array(FixedString(12))})',- 'ALTER TABLE errors DELETE WHERE pid IN ({pids:Array(FixedString(12))})',+ 'ALTER TABLE events DELETE WHERE pid IN ({pids:Array(FixedString(12))})','ALTER TABLE error_statuses DELETE WHERE pid IN ({pids:Array(FixedString(12))})',- 'ALTER TABLE captcha DELETE WHERE pid IN ({pids:Array(FixedString(12))})',]await Promise.all(@@ -654,7 +650,7 @@ export class ProjectService {)}- let query = `ALTER TABLE analytics DELETE WHERE pid={pid:FixedString(12)} AND (`+ let query = `ALTER TABLE events DELETE WHERE pid={pid:FixedString(12)} AND type = 'pageview' AND (`const params = {pid,@@ -684,10 +680,7 @@ export class ProjectService {to: string,): Promise<void> {const queries = [- 'ALTER TABLE analytics DELETE WHERE pid = {pid:FixedString(12)} AND created BETWEEN {from:String} AND {to:String}',- 'ALTER TABLE customEV DELETE WHERE pid = {pid:FixedString(12)} AND created BETWEEN {from:String} AND {to:String}',- 'ALTER TABLE performance DELETE WHERE pid = {pid:FixedString(12)} AND created BETWEEN {from:String} AND {to:String}',- 'ALTER TABLE errors DELETE WHERE pid = {pid:FixedString(12)} AND created BETWEEN {from:String} AND {to:String}',+ "ALTER TABLE events DELETE WHERE pid = {pid:FixedString(12)} AND type IN ('pageview', 'custom_event', 'performance', 'error', 'captcha') AND created BETWEEN {from:String} AND {to:String}",'ALTER TABLE error_statuses DELETE WHERE pid = {pid:FixedString(12)} AND created BETWEEN {from:String} AND {to:String}',]@@ -820,52 +813,37 @@ export class ProjectService {monthEnd,}- const selector = `+ const query = `+ SELECT+ countIf(type = 'pageview') AS pageviews,+ countIf(type = 'custom_event') AS customEvents,+ countIf(type = 'captcha') AS captcha,+ countIf(type = 'error') AS errors+ FROM eventsWHERE created BETWEEN {monthStart:String} AND {monthEnd:String}- AND pid IN ({pids:Array(FixedString(12))})+ AND pid IN ({pids:Array(FixedString(12))})+ AND type IN ('pageview', 'custom_event', 'captcha', 'error')`- const countEVQuery = `SELECT count() FROM analytics ${selector}`- const countCustomEVQuery = `SELECT count() FROM customEV ${selector}`- const countCaptchaQuery = `SELECT count() FROM captcha ${selector}`- const countErrorsQuery = `SELECT count() FROM errors ${selector}`-- const [- { data: pageviews },- { data: customEvents },- { data: captcha },- { data: errors },- ] = await Promise.all([- clickhouse- .query({- query: countEVQuery,- query_params: params,- })- .then((resultSet) => resultSet.json()),- clickhouse- .query({- query: countCustomEVQuery,- query_params: params,- })- .then((resultSet) => resultSet.json()),- clickhouse- .query({- query: countCaptchaQuery,- query_params: params,- })- .then((resultSet) => resultSet.json()),- clickhouse- .query({- query: countErrorsQuery,- query_params: params,- })- .then((resultSet) => resultSet.json()),- ])+ const { data: counts } = await clickhouse+ .query({+ query,+ query_params: params,+ })+ .then((resultSet) =>+ resultSet.json<{+ pageviews: string | number+ customEvents: string | number+ captcha: string | number+ errors: string | number+ }>(),+ )- totalPageviews += pageviews[0]['count()']- totalCustomEvents += customEvents[0]['count()']- totalCaptcha += captcha[0]['count()']- totalErrors += errors[0]['count()']+ const row = counts[0]+ totalPageviews += Number(row?.pageviews) || 0+ totalCustomEvents += Number(row?.customEvents) || 0+ totalCaptcha += Number(row?.captcha) || 0+ totalErrors += Number(row?.errors) || 0}count = totalPageviews + totalCustomEvents + totalCaptcha + totalErrors@@ -916,54 +894,43 @@ export class ProjectService {// Process PIDs in chunksfor (let i = 0; i < pids.length; i += CHUNK_SIZE) {const pidChunk = pids.slice(i, i + CHUNK_SIZE)- const params = { pids: pidChunk }-- const selector = `- WHERE pid IN ({pids:Array(FixedString(12))})- AND created BETWEEN '${monthStart}' AND '${monthEnd}'- `-- const countEVQuery = `SELECT count() FROM analytics ${selector}`- const countCustomEVQuery = `SELECT count() FROM customEV ${selector}`- const countCaptchaQuery = `SELECT count() FROM captcha ${selector}`- const countErrorsQuery = `SELECT count() FROM errors ${selector}`-- const [- { data: rawTraffic },- { data: rawCustomEvents },- { data: rawCaptcha },- { data: rawErrors },- ] = await Promise.all([- clickhouse- .query({- query: countEVQuery,- query_params: params,- })- .then((resultSet) => resultSet.json()),- clickhouse- .query({- query: countCustomEVQuery,- query_params: params,- })- .then((resultSet) => resultSet.json()),- clickhouse- .query({- query: countCaptchaQuery,- query_params: params,- })- .then((resultSet) => resultSet.json()),- clickhouse- .query({- query: countErrorsQuery,- query_params: params,- })- .then((resultSet) => resultSet.json()),- ])+ const params = {+ pids: pidChunk,+ monthStart,+ monthEnd,+ }- totalTraffic += rawTraffic[0]['count()']- totalCustomEvents += rawCustomEvents[0]['count()']- totalCaptcha += rawCaptcha[0]['count()']- totalErrors += rawErrors[0]['count()']+ const query = `+ SELECT+ countIf(type = 'pageview') AS traffic,+ countIf(type = 'custom_event') AS customEvents,+ countIf(type = 'captcha') AS captcha,+ countIf(type = 'error') AS errors+ FROM events+ WHERE pid IN ({pids:Array(FixedString(12))})+ AND created BETWEEN {monthStart:String} AND {monthEnd:String}+ AND type IN ('pageview', 'custom_event', 'captcha', 'error')+ `++ const { data: counts } = await clickhouse+ .query({+ query,+ query_params: params,+ })+ .then((resultSet) =>+ resultSet.json<{+ traffic: string | number+ customEvents: string | number+ captcha: string | number+ errors: string | number+ }>(),+ )++ const row = counts[0]+ totalTraffic += Number(row?.traffic) || 0+ totalCustomEvents += Number(row?.customEvents) || 0+ totalCaptcha += Number(row?.captcha) || 0+ totalErrors += Number(row?.errors) || 0}const total =@@ -1043,36 +1010,10 @@ export class ProjectService {)const query = `- SELECT- pid,- CASE- WHEN EXISTS (- SELECT 1- FROM analytics- WHERE pid IN (${pids})- )- OR EXISTS (- SELECT 1- FROM customEV- WHERE pid IN (${pids})- )- THEN 1- ELSE 0- END AS exists- FROM- (- SELECT DISTINCT pid- FROM- (- SELECT pid- FROM analytics- WHERE pid IN (${pids})- UNION ALL- SELECT pid- FROM customEV- WHERE pid IN (${pids})- ) AS t- );+ SELECT DISTINCT pid+ FROM events+ WHERE pid IN (${pids})+ AND type IN ('pageview', 'custom_event')`const { data } = await clickhouse@@ -1105,23 +1046,10 @@ export class ProjectService {)const query = `- SELECT- pid,- CASE- WHEN EXISTS (- SELECT 1- FROM errors- WHERE pid IN (${pids})- )- THEN 1- ELSE 0- END AS exists- FROM- (- SELECT DISTINCT pid- FROM errors- WHERE pid IN (${pids})- );+ SELECT DISTINCT pid+ FROM events+ WHERE pid IN (${pids})+ AND type = 'error'`const { data } = await clickhouse@@ -1154,23 +1082,10 @@ export class ProjectService {)const query = `- SELECT- pid,- CASE- WHEN EXISTS (- SELECT 1- FROM captcha- WHERE pid IN (${pids})- )- THEN 1- ELSE 0- END AS exists- FROM- (- SELECT DISTINCT pid- FROM captcha- WHERE pid IN (${pids})- );+ SELECT DISTINCT pid+ FROM events+ WHERE pid IN (${pids})+ AND type = 'captcha'`const { data } = await clickhousediff --git a/backend/apps/cloud/src/task-manager/task-manager.service.ts b/backend/apps/cloud/src/task-manager/task-manager.service.tsindex 83b53f2e3..e4b4cb4a7 100644--- a/backend/apps/cloud/src/task-manager/task-manager.service.ts+++ b/backend/apps/cloud/src/task-manager/task-manager.service.ts@@ -143,13 +143,15 @@ const mapLimit = async <T, R>(return results}-// TODO: Count for other tables like captcha, errors, customEV tooconst generatePlanUsageQueryForUser = (): string => {// NOTE: keep all values parameterized to avoid injection and formatting issues.+ // Counts every billable event kind (pageview, custom_event, error, captcha)+ // in a single scan over the unified events table.return `SELECT {uid:String} AS id, count(*) AS "count"- FROM analytics+ FROM eventsWHERE pid IN ({pids:Array(FixedString(12))})+ AND type IN ('pageview', 'custom_event', 'error', 'captcha')AND created BETWEEN {from:String} AND {to:String}`}@@ -385,27 +387,53 @@ export class TaskManagerService {): { condition: string; params: Record<string, string> } {const goalValue = (goal.value ?? '').toString()- // If goal value is blank, never match anything (avoid LIKE '%%')- if (goalValue.trim() === '') {- return { condition: '1=0', params: {} }- }-const params: Record<string, string> = {}- const column = goal.type === GoalType.CUSTOM_EVENT ? 'ev' : 'pg'+ const column = goal.type === GoalType.CUSTOM_EVENT ? 'event_name' : 'pg'+ const appendMetadataFilters = (condition: string) => {+ if (!goal.metadataFilters || goal.metadataFilters.length === 0) {+ return { condition, params }+ }++ const metaConditions: string[] = []++ goal.metadataFilters.forEach((filter, index) => {+ const keyParam = `${paramKey}_metaKey${index}`+ const valueParam = `${paramKey}_metaValue${index}`+ params[keyParam] = filter.key+ params[valueParam] = filter.value+ metaConditions.push(+ `has(meta.key, {${keyParam}:String}) AND meta.value[indexOf(meta.key, {${keyParam}:String})] = {${valueParam}:String}`,+ )+ })++ return {+ condition: `(${condition}) AND (${metaConditions.join(' AND ')})`,+ params,+ }+ }if (goal.matchType === GoalMatchType.EXACT) {params[paramKey] = goalValue- return { condition: `${column} = {${paramKey}:String}`, params }+ return appendMetadataFilters(`${column} = {${paramKey}:String}`)}if (goal.matchType === GoalMatchType.CONTAINS) {+ // Avoid wildcard goals matching every row via LIKE '%%'.+ if (goalValue.trim() === '') {+ return { condition: '1=0', params: {} }+ }+params[paramKey] = `%${goalValue}%`- return { condition: `${column} LIKE {${paramKey}:String}`, params }+ return appendMetadataFilters(`${column} ILIKE {${paramKey}:String}`)}// Regex goal+ if (!goalValue || goalValue === '') {+ return appendMetadataFilters('false')+ }+params[paramKey] = goalValue- return { condition: `match(${column}, {${paramKey}:String})`, params }+ return appendMetadataFilters(`match(${column}, {${paramKey}:String})`)}/**@@ -427,7 +455,7 @@ export class TaskManagerService {const total = Number(totalSessions) || 0const buildUnionQuery = (- table: 'analytics' | 'customEV',+ eventType: 'pageview' | 'custom_event',goals: Goal[],): { query: string; params: Record<string, any> } | null => {if (_isEmpty(goals)) {@@ -459,9 +487,10 @@ export class TaskManagerService {{${goalIdKey}:String} as goalId,count(*) as conversions,uniqExact(psid) as uniqueSessions- FROM ${table}+ FROM eventsWHEREpid = {pid:FixedString(12)}+ AND type = '${eventType}'AND ${condition}AND created BETWEEN {groupFrom:String} AND {groupTo:String}`)@@ -488,8 +517,8 @@ export class TaskManagerService {(g) => g.type === GoalType.CUSTOM_EVENT,)- const analyticsQuery = buildUnionQuery('analytics', analyticsGoals)- const customEventQuery = buildUnionQuery('customEV', customEventGoals)+ const analyticsQuery = buildUnionQuery('pageview', analyticsGoals)+ const customEventQuery = buildUnionQuery('custom_event', customEventGoals)const [analyticsRes, customEventRes] = await Promise.all([analyticsQuery@@ -573,24 +602,10 @@ export class TaskManagerService {for (let i = 0; i < pids.length; i += CHUNK_SIZE) {const pidChunk = pids.slice(i, i + CHUNK_SIZE)const query = `- SELECT sum(cnt) AS totalEvents- FROM (- SELECT count() AS cnt- FROM analytics- WHERE pid IN ({pids:Array(FixedString(12))})- UNION ALL- SELECT count() AS cnt- FROM customEV- WHERE pid IN ({pids:Array(FixedString(12))})- UNION ALL- SELECT count() AS cnt- FROM captcha- WHERE pid IN ({pids:Array(FixedString(12))})- UNION ALL- SELECT count() AS cnt- FROM errors- WHERE pid IN ({pids:Array(FixedString(12))})- )+ SELECT count() AS totalEvents+ FROM events+ WHERE pid IN ({pids:Array(FixedString(12))})+ AND type IN ('pageview', 'custom_event', 'error', 'captcha', 'performance')`const { data } = await clickhouse@@ -1717,8 +1732,8 @@ export class TaskManagerService {count()FROM (SELECT eid, min(created) as first_seen- FROM errors- WHERE pid = {pid:FixedString(12)}+ FROM events+ WHERE pid = {pid:FixedString(12)} AND type = 'error'GROUP BY eid)WHERE first_seen >= now() - ${subtractSecondsTimeframe}@@ -1727,9 +1742,10 @@ export class TaskManagerService {query = `SELECTcount()- FROM errors+ FROM eventsWHEREpid = {pid:FixedString(12)}+ AND type = 'error'AND created >= now() - ${subtractSecondsTimeframe}`}@@ -1737,10 +1753,11 @@ export class TaskManagerService {query = `SELECTcount()- FROM customEV+ FROM eventsWHEREpid = {pid:FixedString(12)}- AND ev = {ev:String}+ AND type = 'custom_event'+ AND event_name = {ev:String}AND created >= now() - ${subtractSecondsTimeframe}`queryParams.ev = alert.queryCustomEvent@@ -1749,9 +1766,10 @@ export class TaskManagerService {query = `SELECTcount(${isUnique ? 'DISTINCT psid' : '*'})- FROM analytics+ FROM eventsWHEREpid = {pid:FixedString(12)}+ AND type = 'pageview'AND created >= now() - ${subtractSecondsTimeframe}`}@@ -1798,14 +1816,14 @@ export class TaskManagerService {if (alert.alertOnNewErrorsOnly) {detailQuery = `- SELECT eid, name, message, lineno, colno, filename- FROM errors- WHERE pid = {pid:FixedString(12)} AND eid IN (+ SELECT eid, error_name AS name, error_message AS message, lineno, colno, error_filename AS filename+ FROM events+ WHERE pid = {pid:FixedString(12)} AND type = 'error' AND eid IN (SELECT eidFROM (SELECT eid, min(created) AS first_seen_for_eid- FROM errors- WHERE pid = {pid:FixedString(12)}+ FROM events+ WHERE pid = {pid:FixedString(12)} AND type = 'error'GROUP BY eid)WHERE first_seen_for_eid >= now() - ${subtractSecondsTimeframe}@@ -1815,10 +1833,11 @@ export class TaskManagerService {`} else {detailQuery = `- SELECT eid, name, message, lineno, colno, filename- FROM errors+ SELECT eid, error_name AS name, error_message AS message, lineno, colno, error_filename AS filename+ FROM eventsWHEREpid = {pid:FixedString(12)}+ AND type = 'error'AND created >= now() - ${subtractSecondsTimeframe}ORDER BY created DESCLIMIT 1@@ -1988,9 +2007,12 @@ export class TaskManagerService {},select: ['id'],})- const now = dayjs.utc().format('YYYY-MM-DD')+ const now = dayjs.utc().format('YYYY-MM-DD HH:mm:ss')// a bit more than 2 months ago- const nineWeeksAgo = dayjs.utc().subtract(9, 'w').format('YYYY-MM-DD')+ const nineWeeksAgo = dayjs+ .utc()+ .subtract(9, 'w')+ .format('YYYY-MM-DD HH:mm:ss')await mapLimit(users, REPORTS_USERS_CONCURRENCY, async (user) => {const { id } = user@@ -2002,61 +2024,10 @@ export class TaskManagerService {return}- // No need to check for performance activity because it's not tracked without tracking analytics- const queryAnalytics = `SELECT count() FROM analytics WHERE pid IN ({pids:Array(FixedString(12))}) AND created BETWEEN {nineWeeksAgo:String} AND {now:String}`- const queryCaptcha = `SELECT count() FROM captcha WHERE pid IN ({pids:Array(FixedString(12))}) AND created BETWEEN {nineWeeksAgo:String} AND {now:String}`- const queryCustomEvents = `SELECT count() FROM customEV WHERE pid IN ({pids:Array(FixedString(12))}) AND created BETWEEN {nineWeeksAgo:String} AND {now:String}`+ const queryEvents = `SELECT 1 FROM events WHERE pid IN ({pids:Array(FixedString(12))}) AND type IN ('pageview', 'captcha', 'custom_event', 'error', 'performance') AND created BETWEEN {nineWeeksAgo:String} AND {now:String} LIMIT 1`// Process project IDs in chunks to avoid ClickHouse field value limit- let totalAnalytics = 0- let totalCaptcha = 0- let totalCustomEvents = 0-- for (let i = 0; i < pids.length; i += CHUNK_SIZE) {- const pidChunk = pids.slice(i, i + CHUNK_SIZE)- const queryParams = {- pids: pidChunk,- nineWeeksAgo,- now,- }-- const { data: analyticsResult } = await clickhouse- .query({- query: queryAnalytics,- query_params: queryParams,- })- .then((resultSet) => resultSet.json<{ 'count()': number }>())-- totalAnalytics += analyticsResult[0]['count()']-- // Early return if we found activity- if (totalAnalytics > 0) {- return- }- }-- for (let i = 0; i < pids.length; i += CHUNK_SIZE) {- const pidChunk = pids.slice(i, i + CHUNK_SIZE)- const queryParams = {- pids: pidChunk,- nineWeeksAgo,- now,- }-- const { data: captchaResult } = await clickhouse- .query({- query: queryCaptcha,- query_params: queryParams,- })- .then((resultSet) => resultSet.json<{ 'count()': number }>())-- totalCaptcha += captchaResult[0]['count()']-- // Early return if we found activity- if (totalCaptcha > 0) {- return- }- }+ let totalEvents = 0for (let i = 0; i < pids.length; i += CHUNK_SIZE) {const pidChunk = pids.slice(i, i + CHUNK_SIZE)@@ -2066,17 +2037,17 @@ export class TaskManagerService {now,}- const { data: customEventsResult } = await clickhouse+ const { data: eventsResult } = await clickhouse.query({- query: queryCustomEvents,+ query: queryEvents,query_params: queryParams,})- .then((resultSet) => resultSet.json<{ 'count()': number }>())+ .then((resultSet) => resultSet.json<Record<string, number>>())- totalCustomEvents += customEventsResult[0]['count()']+ totalEvents += eventsResult.length// Early return if we found activity- if (totalCustomEvents > 0) {+ if (totalEvents > 0) {return}}diff --git a/backend/apps/community/src/analytics/analytics.controller.ts b/backend/apps/community/src/analytics/analytics.controller.tsindex c6313fad7..1d5b37746 100644--- a/backend/apps/community/src/analytics/analytics.controller.ts+++ b/backend/apps/community/src/analytics/analytics.controller.ts@@ -73,12 +73,7 @@ import {GetErrorOverviewOptions,} from './dto/get-error-overview.dto'import { PatchStatusDto } from './dto/patch-status.dto'-import {- customEventTransformer,- errorEventTransformer,- performanceTransformer,- trafficTransformer,-} from './utils/transformers'+import { eventTransformer } from './utils/transformers'import { enrichTrafficSource } from './utils/clickIdSources'import { MAX_METRICS_IN_VIEW } from '../project/dto/create-project-view.dto'import { GetOverallStatsDto } from './dto/get-overall-stats.dto'@@ -250,7 +245,7 @@ export class AnalyticsController {if (period === 'all') {const res = await this.analyticsService.calculateTimeBucketForAllTime(pid,- isCaptcha ? 'captcha' : 'analytics',+ isCaptcha ? 'captcha' : 'pageview',)diff = res.diff@@ -278,13 +273,11 @@ export class AnalyticsController {diff,)- let subQuery = `FROM ${- isCaptcha ? 'captcha' : 'analytics'- } WHERE pid = {pid:FixedString(12)} ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String}`-- if (customEVFilterApplied && !isCaptcha) {- subQuery = `FROM customEV WHERE pid = {pid:FixedString(12)} ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String}`- }+ const subQuery = this.analyticsService.buildAnalyticsEventsSubQuery(+ filtersQuery,+ customEVFilterApplied,+ isCaptcha,+ )const paramsData = {params: {@@ -413,12 +406,15 @@ export class AnalyticsController {let diffif (period === 'all') {- const [analyticsRes, customEVRes] = await Promise.all([- this.analyticsService.calculateTimeBucketForAllTime(pid, 'analytics'),- this.analyticsService.calculateTimeBucketForAllTime(pid, 'customEV'),+ const [pageviewRes, customEventRes] = await Promise.all([+ this.analyticsService.calculateTimeBucketForAllTime(pid, 'pageview'),+ this.analyticsService.calculateTimeBucketForAllTime(+ pid,+ 'custom_event',+ ),])- diff = Math.max(analyticsRes.diff, customEVRes.diff)+ diff = Math.max(pageviewRes.diff, customEventRes.diff)}const safeTimezone = this.analyticsService.getSafeTimezone(timezone)@@ -526,12 +522,15 @@ export class AnalyticsController {let diffif (period === 'all') {- const [analyticsRes, customEVRes] = await Promise.all([- this.analyticsService.calculateTimeBucketForAllTime(pid, 'analytics'),- this.analyticsService.calculateTimeBucketForAllTime(pid, 'customEV'),+ const [pageviewRes, customEventRes] = await Promise.all([+ this.analyticsService.calculateTimeBucketForAllTime(pid, 'pageview'),+ this.analyticsService.calculateTimeBucketForAllTime(+ pid,+ 'custom_event',+ ),])- diff = Math.max(analyticsRes.diff, customEVRes.diff)+ diff = Math.max(pageviewRes.diff, customEventRes.diff)}const safeTimezone = this.analyticsService.getSafeTimezone(timezone)@@ -732,7 +731,7 @@ export class AnalyticsController {diff,)- const subQuery = `FROM performance WHERE pid = {pid:FixedString(12)} ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String}`+ const subQuery = `FROM events WHERE pid = {pid:FixedString(12)} AND type = 'performance' ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String}`const paramsData = { params: { pid, groupFrom, groupTo, ...filtersParams } }@@ -836,7 +835,7 @@ export class AnalyticsController {if (period === 'all') {const res = await this.analyticsService.calculateTimeBucketForAllTime(pid,- 'analytics',+ 'pageview',)diff = res.diff@@ -1026,32 +1025,12 @@ export class AnalyticsController {any(os) AS os,any(cc) AS cc,toString(psid) AS psid- FROM- (- SELECT- psid,- dv,- br,- os,- cc- FROM analytics- WHERE- pid = {pid:FixedString(12)}- AND created >= {since:DateTime}- AND psid IS NOT NULL- UNION ALL- SELECT- psid,- dv,- br,- os,- cc- FROM customEV- WHERE- pid = {pid:FixedString(12)}- AND created >= {since:DateTime}- AND psid IS NOT NULL- )+ FROM events+ WHERE+ pid = {pid:FixedString(12)}+ AND type IN ('pageview', 'custom_event')+ AND created >= {since:DateTime}+ AND psid IS NOT NULLGROUP BY psid`@@ -1151,7 +1130,8 @@ export class AnalyticsController {enrichTrafficSource(eventsDTO)- const transformed = customEventTransformer({+ const transformed = eventTransformer({+ type: 'custom_event',psid,profileId,pid: eventsDTO.pid,@@ -1183,7 +1163,7 @@ export class AnalyticsController {try {await clickhouse.insert({- table: 'customEV',+ table: 'events',format: 'JSONEachRow',values: [transformed],clickhouse_settings: { async_insert: 1 },@@ -1321,7 +1301,8 @@ export class AnalyticsController {enrichTrafficSource(logDTO)- const transformed = trafficTransformer({+ const transformed = eventTransformer({+ type: 'pageview',psid,profileId,pid: logDTO.pid,@@ -1364,7 +1345,8 @@ export class AnalyticsController {ttfb,} = logDTO.perf- perfTransformed = performanceTransformer({+ perfTransformed = eventTransformer({+ type: 'performance',pid: logDTO.pid,host: this.analyticsService.getHostFromOrigin(headers.origin),pg: logDTO.pg,@@ -1392,7 +1374,7 @@ export class AnalyticsController {try {await clickhouse.insert({- table: 'analytics',+ table: 'events',format: 'JSONEachRow',values: [transformed],clickhouse_settings: { async_insert: 1 },@@ -1400,7 +1382,7 @@ export class AnalyticsController {if (!_isEmpty(perfTransformed)) {await clickhouse.insert({- table: 'performance',+ table: 'events',format: 'JSONEachRow',values: [perfTransformed],clickhouse_settings: { async_insert: 1 },@@ -1489,7 +1471,8 @@ export class AnalyticsController {const { deviceType, browserName, browserVersion, osName, osVersion } =await this.analyticsService.getRequestInformation(headers)- const transformed = trafficTransformer({+ const transformed = eventTransformer({+ type: 'pageview',psid,profileId,pid: logDTO.pid,@@ -1520,7 +1503,7 @@ export class AnalyticsController {try {await clickhouse.insert({- table: 'analytics',+ table: 'events',format: 'JSONEachRow',values: [transformed],clickhouse_settings: { async_insert: 1 },@@ -1571,7 +1554,7 @@ export class AnalyticsController {if (period === 'all') {const res = await this.analyticsService.calculateTimeBucketForAllTime(pid,- customEVFilterApplied ? 'customEV' : 'analytics',+ this.analyticsService.getAnalyticsEventType(customEVFilterApplied),)timeBucket = res.timeBucket[0]@@ -1675,7 +1658,7 @@ export class AnalyticsController {if (period === VALID_PERIODS[VALID_PERIODS.length - 1]) {const res = await this.analyticsService.calculateTimeBucketForAllTime(pid,- 'customEV',+ 'custom_event',)newTimeBucket = _includes(res.timeBucket, timeBucket)@@ -1869,7 +1852,8 @@ export class AnalyticsController {const { name, message, lineno, colno, filename, stackTrace, meta } =errorDTO- const transformed = errorEventTransformer({+ const transformed = eventTransformer({+ type: 'error',psid,profileId,eid: this.analyticsService.getErrorID(errorDTO),@@ -1901,7 +1885,7 @@ export class AnalyticsController {try {await clickhouse.insert({- table: 'errors',+ table: 'events',format: 'JSONEachRow',values: [transformed],clickhouse_settings: { async_insert: 1 },@@ -1986,7 +1970,7 @@ export class AnalyticsController {if (period === 'all') {const res = await this.analyticsService.calculateTimeBucketForAllTime(pid,- 'errors',+ 'error',)timeBucket = res.timeBucket[0]@@ -2063,7 +2047,7 @@ export class AnalyticsController {if (period === 'all') {const res = await this.analyticsService.calculateTimeBucketForAllTime(pid,- 'errors',+ 'error',)newTimeBucket = res.timeBucket[0]@@ -2139,7 +2123,7 @@ export class AnalyticsController {if (period === 'all') {const res = await this.analyticsService.calculateTimeBucketForAllTime(pid,- 'errors',+ 'error',)newTimeBucket = res.timeBucket[0]@@ -2151,6 +2135,8 @@ export class AnalyticsController {DataType.ERRORS,true,)+ const [sessionFiltersQuery, sessionFiltersParams] =+ this.analyticsService.getFiltersQuery(filters, DataType.ANALYTICS, true)const safeTimezone = this.analyticsService.getSafeTimezone(timezone)const { groupFromUTC, groupToUTC } = this.analyticsService.getGroupFromTo(@@ -2180,6 +2166,8 @@ export class AnalyticsController {groupToUTC,newTimeBucket,parsedOptions.showResolved || false,+ sessionFiltersQuery,+ sessionFiltersParams,)}@@ -2218,7 +2206,7 @@ export class AnalyticsController {if (period === 'all') {const res = await this.analyticsService.calculateTimeBucketForAllTime(pid,- 'errors',+ 'error',)newTimeBucket = res.timeBucket[0]@@ -2291,7 +2279,7 @@ export class AnalyticsController {if (period === 'all') {const res = await this.analyticsService.calculateTimeBucketForAllTime(pid,- customEVFilterApplied ? 'customEV' : 'analytics',+ this.analyticsService.getAnalyticsEventType(customEVFilterApplied),)timeBucket = res.timeBucket[0]@@ -2368,7 +2356,7 @@ export class AnalyticsController {if (period === 'all') {const res = await this.analyticsService.calculateTimeBucketForAllTime(pid,- 'analytics',+ 'pageview',)timeBucket = res.timeBucket[0]diff = res.diff@@ -2454,7 +2442,7 @@ export class AnalyticsController {if (period === 'all') {const res = await this.analyticsService.calculateTimeBucketForAllTime(pid,- customEVFilterApplied ? 'customEV' : 'analytics',+ this.analyticsService.getAnalyticsEventType(customEVFilterApplied),)timeBucket = res.timeBucket[0]diff --git a/backend/apps/community/src/analytics/analytics.service.ts b/backend/apps/community/src/analytics/analytics.service.tsindex f63a48d82..4215d6bce 100644--- a/backend/apps/community/src/analytics/analytics.service.ts+++ b/backend/apps/community/src/analytics/analytics.service.ts@@ -270,6 +270,14 @@ const EXCLUDE_NULL_FOR = ['ctp',]+const ERROR_COLUMN_MAP: Record<string, string> = {+ name: 'error_name',+ message: 'error_message',+ filename: 'error_filename',+}++const mapErrorColumn = (type: string): string => ERROR_COLUMN_MAP[type] || type+const generateParamsQuery = (col: string,subQuery: string,@@ -277,7 +285,8 @@ const generateParamsQuery = (type: 'traffic' | 'performance' | 'errors' | 'captcha',measure?: PerfMeasure,): string => {- let columns = [`${col} as name`]+ const sqlCol = type === 'errors' ? mapErrorColumn(col) : col+ let columns = [`${sqlCol} as name`]// For regions and cities we'll return an array of objects, that will also include the country code and region code// We need the conutry code to display the flag next to the region/city name@@ -301,15 +310,15 @@ const generateParamsQuery = (const fn = MEASURES_MAP[processedMeasure]- if (col === 'pg' || col === 'host') {+ if (sqlCol === 'pg' || sqlCol === 'host') {return `SELECT ${columnsQuery}, round(divide(${fn}(pageLoad), 1000), 2) as count ${subQuery} GROUP BY ${columnsQuery}`}- return `SELECT ${columnsQuery}, round(divide(${fn}(pageLoad), 1000), 2) as count ${subQuery} ${EXCLUDE_NULL_FOR.includes(col) ? `AND ${col} IS NOT NULL` : ''} GROUP BY ${columnsQuery}`+ return `SELECT ${columnsQuery}, round(divide(${fn}(pageLoad), 1000), 2) as count ${subQuery} ${EXCLUDE_NULL_FOR.includes(sqlCol) ? `AND ${sqlCol} IS NOT NULL` : ''} GROUP BY ${columnsQuery}`}if (type === 'errors') {- return `SELECT ${columnsQuery}, count(*) as count ${subQuery} ${EXCLUDE_NULL_FOR.includes(col) ? `AND ${col} IS NOT NULL` : ''} GROUP BY ${columnsQuery}`+ return `SELECT ${columnsQuery}, count(*) as count ${subQuery} ${EXCLUDE_NULL_FOR.includes(sqlCol) ? `AND ${sqlCol} IS NOT NULL` : ''} GROUP BY ${columnsQuery}`}if (type === 'captcha') {@@ -330,6 +339,13 @@ export enum DataType {CAPTCHA = 'captcha',}+type AnalyticsEventType = 'pageview' | 'custom_event'+type EventsAllTimeType =+ | AnalyticsEventType+ | 'performance'+ | 'error'+ | 'captcha'+const isValidOrigin = (origins: string[], origin: string) => {const escapeRegex = (str: string) =>str.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')@@ -665,7 +681,9 @@ export class AnalyticsService {)}- const query = `SELECT ${type} FROM errors WHERE pid={pid:FixedString(12)} AND ${type} IS NOT NULL GROUP BY ${type}`+ const sqlCol = mapErrorColumn(type)++ const query = `SELECT ${sqlCol} AS ${type} FROM events WHERE pid={pid:FixedString(12)} AND type = 'error' AND ${sqlCol} IS NOT NULL GROUP BY ${sqlCol}`const { data } = await clickhouse.query({@@ -684,13 +702,13 @@ export class AnalyticsService {type: 'traffic' | 'errors',column: 'br' | 'os',): Promise<Array<{ name: string; version: string }>> {- const safeTable = type === 'errors' ? 'errors' : 'analytics'+ const safeType = type === 'errors' ? 'error' : 'pageview'const safeVersionCol = column === 'br' ? 'brv' : 'osv'const query = `- SELECT ${column}, ${safeVersionCol}- FROM ${safeTable}- WHERE pid={pid:FixedString(12)} AND ${column} IS NOT NULL AND ${safeVersionCol} IS NOT NULL+ SELECT ${column}, ${safeVersionCol}+ FROM events+ WHERE pid={pid:FixedString(12)} AND type = '${safeType}' AND ${column} IS NOT NULL AND ${safeVersionCol} IS NOT NULLGROUP BY ${column}, ${safeVersionCol}`@@ -723,6 +741,22 @@ export class AnalyticsService {return CAPTCHA_COLUMNS}+ getAnalyticsEventType(customEVFilterApplied: boolean): AnalyticsEventType {+ return customEVFilterApplied ? 'custom_event' : 'pageview'+ }++ buildAnalyticsEventsSubQuery(+ filtersQuery: string,+ customEVFilterApplied: boolean,+ isCaptcha = false,+ ): string {+ const eventType = isCaptcha+ ? 'captcha'+ : this.getAnalyticsEventType(customEVFilterApplied)++ return `FROM events WHERE pid = {pid:FixedString(12)} AND type = '${eventType}' ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String}`+ }+getGroupFromTo(from: string,to: string,@@ -871,9 +905,10 @@ export class AnalyticsService {pg,created,lagInFrame(pg) OVER (PARTITION BY psid ORDER BY created) AS prev_page- FROM analytics+ FROM eventsWHEREpid = {pid:FixedString(12)}+ AND type = 'pageview'AND created BETWEEN {groupFrom:String} AND {groupTo:String}AND pg IS NOT NULL${filtersQuery}@@ -986,15 +1021,17 @@ export class AnalyticsService {async calculateTimeBucketForAllTime(pid: string,- table: 'analytics' | 'customEV' | 'performance' | 'errors' | 'captcha',+ eventTypes: EventsAllTimeType | readonly EventsAllTimeType[],): Promise<{timeBucket: TimeBucketType[]diff: number}> {+ const types = Array.isArray(eventTypes) ? [...eventTypes] : [eventTypes]+const { data: fromData } = await clickhouse.query({- query: `SELECT min(created) AS firstCreated FROM ${table} WHERE pid = {pid:FixedString(12)}`,- query_params: { pid },+ query: `SELECT min(created) AS firstCreated FROM events WHERE pid = {pid:FixedString(12)} AND type IN ({types:Array(String)})`,+ query_params: { pid, types },}).then((res) => res.json<{ firstCreated?: string }>())@@ -1187,7 +1224,8 @@ export class AnalyticsService {}const { filter, isExclusive, isContains } = converted[column][f]- let sqlColumn = column+ let sqlColumn =+ dataType === DataType.ERRORS ? mapErrorColumn(column) : columnlet isArrayDataset = falseconst param = `qf_${col}_${f}`@@ -1228,8 +1266,8 @@ export class AnalyticsService {? 'argMin(pg, created)': 'argMax(pg, created)'const subQueryForPages = isContains- ? `SELECT psid FROM (SELECT psid, ${pageSelector} as page FROM analytics WHERE pid = {pid:FixedString(12)} GROUP BY psid) WHERE page ILIKE concat('%', {${param}:String}, '%')`- : `SELECT psid FROM (SELECT psid, ${pageSelector} as page FROM analytics WHERE pid = {pid:FixedString(12)} GROUP BY psid) WHERE page = {${param}:String}`+ ? `SELECT psid FROM (SELECT psid, ${pageSelector} as page FROM events WHERE pid = {pid:FixedString(12)} AND type = 'pageview' GROUP BY psid) WHERE page ILIKE concat('%', {${param}:String}, '%')`+ : `SELECT psid FROM (SELECT psid, ${pageSelector} as page FROM events WHERE pid = {pid:FixedString(12)} AND type = 'pageview' GROUP BY psid) WHERE page = {${param}:String}`// For exclusive filter (isNot) we exclude sessions with matching entry/exit pagequery += `psid ${isExclusive ? 'NOT IN' : 'IN'} (${subQueryForPages})`@@ -1270,6 +1308,9 @@ export class AnalyticsService {) {sqlColumn = 'meta.value'isArrayDataset = true+ } else if (column === 'ev') {+ // Public filter contract `ev` maps to the renamed `event_name` column+ sqlColumn = 'event_name'}const isNullFilter =@@ -1586,21 +1627,11 @@ export class AnalyticsService {FROM (SELECTpsid,- pg AS value,+ if(type = 'pageview', pg, event_name) AS value,created- FROM analytics- WHERE pid = {pid:FixedString(12)}- AND psid != 0- AND created BETWEEN {groupFrom:String} AND {groupTo:String}-- UNION ALL-- SELECT- psid,- ev AS value,- created- FROM customEV+ FROM eventsWHERE pid = {pid:FixedString(12)}+ AND type IN ('pageview', 'custom_event')AND psid != 0AND created BETWEEN {groupFrom:String} AND {groupTo:String})@@ -1648,14 +1679,14 @@ export class AnalyticsService {psid,CAST(windowFunnel(86400)(created, ${pagesStr}) AS UInt64) AS levelFROM (- SELECT psid, pg AS value, created- FROM analytics- WHERE pid = {pid:FixedString(12)} AND psid != 0- AND created BETWEEN {groupFrom:String} AND {groupTo:String}- UNION ALL- SELECT psid, ev AS value, created- FROM customEV- WHERE pid = {pid:FixedString(12)} AND psid != 0+ SELECT+ psid,+ if(type = 'pageview', pg, event_name) AS value,+ created+ FROM events+ WHERE pid = {pid:FixedString(12)}+ AND type IN ('pageview', 'custom_event')+ AND psid != 0AND created BETWEEN {groupFrom:String} AND {groupTo:String})GROUP BY psid@@ -1670,17 +1701,11 @@ export class AnalyticsService {psid,argMin(cc, created) AS cc,argMin(if(domain(ref) != '', domain(ref), 'Direct / None'), created) AS source- FROM (- SELECT psid, cc, ref, created- FROM analytics- WHERE pid = {pid:FixedString(12)} AND psid != 0- AND created BETWEEN {groupFrom:String} AND {groupTo:String}- UNION ALL- SELECT psid, cc, ref, created- FROM customEV- WHERE pid = {pid:FixedString(12)} AND psid != 0- AND created BETWEEN {groupFrom:String} AND {groupTo:String}- )+ FROM events+ WHERE pid = {pid:FixedString(12)}+ AND type IN ('pageview', 'custom_event')+ AND psid != 0+ AND created BETWEEN {groupFrom:String} AND {groupTo:String}GROUP BY psid)SELECT step, type, val, cnt FROM (@@ -1758,14 +1783,14 @@ export class AnalyticsService {psid,windowFunnel(86400)(created, ${pagesStr}) AS levelFROM (- SELECT psid, pg AS value, created- FROM analytics- WHERE pid = {pid:FixedString(12)} AND psid != 0- AND created BETWEEN {groupFrom:String} AND {groupTo:String}- UNION ALL- SELECT psid, ev AS value, created- FROM customEV- WHERE pid = {pid:FixedString(12)} AND psid != 0+ SELECT+ psid,+ if(type = 'pageview', pg, event_name) AS value,+ created+ FROM events+ WHERE pid = {pid:FixedString(12)}+ AND type IN ('pageview', 'custom_event')+ AND psid != 0AND created BETWEEN {groupFrom:String} AND {groupTo:String})GROUP BY psid@@ -1781,19 +1806,12 @@ export class AnalyticsService {any(br) AS br,min(toTimeZone(created, {timezone:String})) AS sessionStart,max(toTimeZone(created, {timezone:String})) AS lastActivity- FROM (- SELECT psid, pid, cc, os, br, created- FROM analytics- WHERE pid = {pid:FixedString(12)} AND psid IS NOT NULL- AND created BETWEEN {groupFrom:String} AND {groupTo:String}- AND psid IN (SELECT psid FROM funnel_qualified)- UNION ALL- SELECT psid, pid, cc, os, br, created- FROM customEV- WHERE pid = {pid:FixedString(12)} AND psid IS NOT NULL+ FROM events+ WHERE pid = {pid:FixedString(12)}+ AND type IN ('pageview', 'custom_event')+ AND psid IS NOT NULLAND created BETWEEN {groupFrom:String} AND {groupTo:String}AND psid IN (SELECT psid FROM funnel_qualified)- )GROUP BY psidCasted, pid),pageview_counts AS (@@ -1801,8 +1819,8 @@ export class AnalyticsService {CAST(psid, 'String') AS psidCasted,pid,count() as count- FROM analytics- WHERE pid = {pid:FixedString(12)} AND psid IS NOT NULL+ FROM events+ WHERE pid = {pid:FixedString(12)} AND type = 'pageview' AND psid IS NOT NULLAND created BETWEEN {groupFrom:String} AND {groupTo:String}AND psid IN (SELECT psid FROM funnel_qualified)GROUP BY psidCasted, pid@@ -1812,8 +1830,8 @@ export class AnalyticsService {CAST(psid, 'String') AS psidCasted,pid,count() as count- FROM customEV- WHERE pid = {pid:FixedString(12)} AND psid IS NOT NULL+ FROM events+ WHERE pid = {pid:FixedString(12)} AND type = 'custom_event' AND psid IS NOT NULLAND created BETWEEN {groupFrom:String} AND {groupTo:String}AND psid IN (SELECT psid FROM funnel_qualified)GROUP BY psidCasted, pid@@ -1823,8 +1841,8 @@ export class AnalyticsService {CAST(psid, 'String') AS psidCasted,pid,count() as count- FROM errors- WHERE pid = {pid:FixedString(12)} AND psid IS NOT NULL+ FROM events+ WHERE pid = {pid:FixedString(12)} AND type = 'error' AND psid IS NOT NULLAND created BETWEEN {groupFrom:String} AND {groupTo:String}AND psid IN (SELECT psid FROM funnel_qualified)GROUP BY psidCasted, pid@@ -1860,7 +1878,7 @@ export class AnalyticsService {COALESCE(errc.count, 0) AS errors,dsf.sessionStart,dsf.lastActivity,- if(dateDiff('second', dsf.lastActivity, now()) < ${LIVE_SESSION_THRESHOLD_SECONDS}, 1, 0) AS isLive,+ if(dateDiff('second', toTimeZone(dsf.lastActivity, 'UTC'), toTimeZone(now(), 'UTC')) < ${LIVE_SESSION_THRESHOLD_SECONDS}, 1, 0) AS isLive,sda.avg_duration AS sdur,sda.profileId AS profileId,if(startsWith(sda.profileId, '${AnalyticsService.PROFILE_PREFIX_USER}'), 1, 0) AS isIdentified,@@ -1902,8 +1920,9 @@ export class AnalyticsService {const query = `SELECTcount() as c- FROM analytics+ FROM eventsWHERE pid = {pid:FixedString(12)}+ AND type = 'pageview'AND created BETWEEN {groupFrom:String} AND {groupTo:String}`const { data } = await clickhouse@@ -1954,15 +1973,18 @@ export class AnalyticsService {const promises = pids.map(async (pid) => {try {if (period === 'all') {- let queryAll = `+ const allTimeType = this.getAnalyticsEventType(customEVFilterApplied)++ const queryAll = `WITH analytics_counts AS (SELECTcount(*) AS all,count(DISTINCT psid) AS unique,count(DISTINCT profileId) AS users- FROM analytics+ FROM eventsWHEREpid = {pid:FixedString(12)}+ AND type = '${allTimeType}'${filtersQuery}),duration_avg AS (@@ -1973,6 +1995,14 @@ export class AnalyticsService {dateDiff('second', min(firstSeen), max(lastSeen)) as durationFROM sessionsWHERE pid = {pid:FixedString(12)}+ AND psid IN (+ SELECT DISTINCT psid+ FROM events+ WHERE pid = {pid:FixedString(12)}+ AND type = '${allTimeType}'+ AND psid IS NOT NULL+ ${filtersQuery}+ )GROUP BY psid))@@ -1982,36 +2012,6 @@ export class AnalyticsService {FROM analytics_counts, duration_avg`- if (customEVFilterApplied) {- queryAll = `- WITH analytics_counts AS (- SELECT- count(*) AS all,- count(DISTINCT psid) AS unique,- count(DISTINCT profileId) AS users- FROM customEV- WHERE- pid = {pid:FixedString(12)}- ${filtersQuery}- ),- duration_avg AS (- SELECT avgOrNull(duration) as sdur- FROM (- SELECT- psid,- dateDiff('second', min(firstSeen), max(lastSeen)) as duration- FROM sessions- WHERE pid = {pid:FixedString(12)}- GROUP BY psid- )- )- SELECT- analytics_counts.*,- duration_avg.sdur- FROM analytics_counts, duration_avg- `- }-const { data } = await clickhouse.query({query: queryAll,@@ -2055,7 +2055,7 @@ export class AnalyticsService {const { diff: allDiff, timeBucket: allowedBuckets } =await this.calculateTimeBucketForAllTime(pid,- customEVFilterApplied ? 'customEV' : 'analytics',+ this.getAnalyticsEventType(customEVFilterApplied),)const allTimeChartBucket = _includes(allowedBuckets,@@ -2096,16 +2096,19 @@ export class AnalyticsService {).format('YYYY-MM-DD HH:mm:ss')- let queryCurrent = `+ const periodType = this.getAnalyticsEventType(customEVFilterApplied)++ const queryCurrent = `WITH analytics_counts AS (SELECT1 AS sortOrder,count(*) AS all,count(DISTINCT psid) AS unique,count(DISTINCT profileId) AS users- FROM analytics+ FROM eventsWHEREpid = {pid:FixedString(12)}+ AND type = '${periodType}'AND created BETWEEN {groupFromUTC:String} AND {groupToUTC:String}${filtersQuery}),@@ -2119,8 +2122,9 @@ export class AnalyticsService {WHERE pid = {pid:FixedString(12)}AND psid IN (SELECT DISTINCT psid- FROM analytics+ FROM eventsWHERE pid = {pid:FixedString(12)}+ AND type = '${periodType}'AND psid IS NOT NULLAND created BETWEEN {groupFromUTC:String} AND {groupToUTC:String}${filtersQuery}@@ -2134,16 +2138,17 @@ export class AnalyticsService {FROM analytics_counts, duration_avg`- let queryPrevious = `+ const queryPrevious = `WITH analytics_counts AS (SELECT2 AS sortOrder,count(*) AS all,count(DISTINCT psid) AS unique,count(DISTINCT profileId) AS users- FROM analytics+ FROM eventsWHEREpid = {pid:FixedString(12)}+ AND type = '${periodType}'AND created BETWEEN {periodSubtracted:String} AND {groupFromUTC:String}${filtersQuery}),@@ -2157,8 +2162,9 @@ export class AnalyticsService {WHERE pid = {pid:FixedString(12)}AND psid IN (SELECT DISTINCT psid- FROM analytics+ FROM eventsWHERE pid = {pid:FixedString(12)}+ AND type = '${periodType}'AND psid IS NOT NULLAND created BETWEEN {periodSubtracted:String} AND {groupFromUTC:String}${filtersQuery}@@ -2172,83 +2178,6 @@ export class AnalyticsService {FROM analytics_counts, duration_avg`- if (customEVFilterApplied) {- queryCurrent = `- WITH analytics_counts AS (- SELECT- 1 AS sortOrder,- count(*) AS all,- count(DISTINCT psid) AS unique,- count(DISTINCT profileId) AS users- FROM customEV- WHERE- pid = {pid:FixedString(12)}- AND created BETWEEN {groupFromUTC:String} AND {groupToUTC:String}- ${filtersQuery}- ),- duration_avg AS (- SELECT avgOrNull(duration) as sdur- FROM (- SELECT- psid,- dateDiff('second', min(firstSeen), max(lastSeen)) as duration- FROM sessions- WHERE pid = {pid:FixedString(12)}- AND psid IN (- SELECT DISTINCT psid- FROM customEV- WHERE pid = {pid:FixedString(12)}- AND psid IS NOT NULL- AND created BETWEEN {groupFromUTC:String} AND {groupToUTC:String}- ${filtersQuery}- )- GROUP BY psid- )- )- SELECT- analytics_counts.*,- duration_avg.sdur- FROM analytics_counts, duration_avg- `- queryPrevious = `- WITH analytics_counts AS (- SELECT- 2 AS sortOrder,- count(*) AS all,- count(DISTINCT psid) AS unique,- count(DISTINCT profileId) AS users- FROM customEV- WHERE- pid = {pid:FixedString(12)}- AND created BETWEEN {periodSubtracted:String} AND {groupFromUTC:String}- ${filtersQuery}- ),- duration_avg AS (- SELECT avgOrNull(duration) as sdur- FROM (- SELECT- psid,- dateDiff('second', min(firstSeen), max(lastSeen)) as duration- FROM sessions- WHERE pid = {pid:FixedString(12)}- AND psid IN (- SELECT DISTINCT psid- FROM customEV- WHERE pid = {pid:FixedString(12)}- AND psid IS NOT NULL- AND created BETWEEN {periodSubtracted:String} AND {groupFromUTC:String}- ${filtersQuery}- )- GROUP BY psid- )- )- SELECT- analytics_counts.*,- duration_avg.sdur- FROM analytics_counts, duration_avg- `- }-const query = `${queryCurrent} UNION ALL ${queryPrevious}`let { data } = await clickhouse@@ -2343,6 +2272,63 @@ export class AnalyticsService {return result}+ private generateAnalyticsAggregationQueryForScope(+ timeBucket: TimeBucketType,+ filtersQuery: string,+ mode: ChartRenderMode,+ customEVFilterApplied: boolean,+ ): string {+ return customEVFilterApplied+ ? this.generateCustomEventsAggregationQuery(+ timeBucket,+ filtersQuery,+ mode,+ )+ : this.generateAnalyticsAggregationQuery(timeBucket, filtersQuery, mode)+ }++ private extractAnalyticsChartDataForScope(+ result: Array<TrafficCHResponse | TrafficCEFilterCHResponse>,+ xShifted: string[],+ customEVFilterApplied: boolean,+ mode: ChartRenderMode,+ ): IExtractChartData {+ if (customEVFilterApplied) {+ const customEventsData = this.extractCustomEventsChartData(+ result,+ xShifted,+ )+ const uniques =+ customEventsData._uniques ||+ customEventsData._unknown_event ||+ Array(_size(xShifted)).fill(0)+ const visits = customEventsData._count || Array(_size(xShifted)).fill(0)+ const sdur = customEventsData._sdur || Array(_size(xShifted)).fill(0)++ return {+ visits,+ uniques,+ sdur,+ }+ }++ const chartData = this.extractChartData(result, xShifted)++ // Propagate the previous cumulative value forward for empty buckets+ if (mode === ChartRenderMode.CUMULATIVE) {+ for (let i = 1; i < chartData.visits.length; ++i) {+ if (chartData.visits[i] === 0) {+ chartData.visits[i] = chartData.visits[i - 1]+ }+ if (chartData.uniques[i] === 0) {+ chartData.uniques[i] = chartData.uniques[i - 1]+ }+ }+ }++ return chartData+ }+/*** Get simplified chart data for dashboard cards* Returns only x (dates) and visits (pageviews) for a lightweight chart@@ -2373,30 +2359,11 @@ export class AnalyticsService {},}- if (customEVFilterApplied) {- const query = this.generateCustomEventsAggregationQuery(- timeBucket,- filtersQuery,- ChartRenderMode.PERIODICAL,- )-- const { data } = await clickhouse- .query({- query,- query_params: { ...paramsData.params, timezone: safeTimezone },- })- .then((resultSet) => resultSet.json<TrafficCEFilterCHResponse>())-- const visits =- this.extractCustomEventsChartData(data, xShifted)?._unknown_event || []-- return { x: xShifted, visits }- }-- const query = this.generateAnalyticsAggregationQuery(+ const query = this.generateAnalyticsAggregationQueryForScope(timeBucket,filtersQuery,ChartRenderMode.PERIODICAL,+ customEVFilterApplied,)const { data } = await clickhouse@@ -2404,9 +2371,16 @@ export class AnalyticsService {query,query_params: { ...paramsData.params, timezone: safeTimezone },})- .then((resultSet) => resultSet.json<TrafficCHResponse>())+ .then((resultSet) =>+ resultSet.json<TrafficCHResponse | TrafficCEFilterCHResponse>(),+ )- const { visits } = this.extractChartData(data, xShifted)+ const { visits } = this.extractAnalyticsChartDataForScope(+ data,+ xShifted,+ customEVFilterApplied,+ ChartRenderMode.PERIODICAL,+ )return { x: xShifted, visits }}@@ -2456,7 +2430,7 @@ export class AnalyticsService {const promises = pids.map(async (pid) => {try {if (period === 'all') {- const queryAll = `SELECT ${columnSelectors} FROM performance WHERE pid = {pid:FixedString(12)} ${filtersQuery}`+ const queryAll = `SELECT ${columnSelectors} FROM events WHERE pid = {pid:FixedString(12)} AND type = 'performance' ${filtersQuery}`const { data } = await clickhouse.query({query: queryAll,@@ -2518,8 +2492,8 @@ export class AnalyticsService {.format('YYYY-MM-DD HH:mm:ss')}- const queryCurrent = `SELECT 1 AS sortOrder, ${columnSelectors} FROM performance WHERE pid = {pid:FixedString(12)} AND created BETWEEN {periodFormatted:String} AND {now:String} ${filtersQuery}`- const queryPrevious = `SELECT 2 AS sortOrder, ${columnSelectors} FROM performance WHERE pid = {pid:FixedString(12)} AND created BETWEEN {periodSubtracted:String} AND {periodFormatted:String} ${filtersQuery}`+ const queryCurrent = `SELECT 1 AS sortOrder, ${columnSelectors} FROM events WHERE pid = {pid:FixedString(12)} AND type = 'performance' AND created BETWEEN {periodFormatted:String} AND {now:String} ${filtersQuery}`+ const queryPrevious = `SELECT 2 AS sortOrder, ${columnSelectors} FROM events WHERE pid = {pid:FixedString(12)} AND type = 'performance' AND created BETWEEN {periodSubtracted:String} AND {periodFormatted:String} ${filtersQuery}`const query = `${queryCurrent} UNION ALL ${queryPrevious}`@@ -2616,8 +2590,8 @@ export class AnalyticsService {const query = `WITH session_pages AS (SELECT psid, ${selector} as page- FROM analytics- WHERE pid = {pid:FixedString(12)}+ FROM events+ WHERE pid = {pid:FixedString(12)} AND type = 'pageview'GROUP BY psid)SELECT page FROM session_pages WHERE page IS NOT NULL GROUP BY page@@ -2633,11 +2607,10 @@ export class AnalyticsService {return _map(data, 'page')}- let query = `SELECT ${type} FROM analytics WHERE pid={pid:FixedString(12)} AND ${type} IS NOT NULL GROUP BY ${type}`+ let query = `SELECT ${type} FROM events WHERE pid={pid:FixedString(12)} AND type = 'pageview' AND ${type} IS NOT NULL GROUP BY ${type}`if (type === 'ev') {- query =- 'SELECT ev FROM customEV WHERE pid={pid:FixedString(12)} AND ev IS NOT NULL GROUP BY ev'+ query = `SELECT event_name AS ev FROM events WHERE pid={pid:FixedString(12)} AND type = 'custom_event' AND event_name IS NOT NULL GROUP BY event_name`}const { data } = await clickhouse@@ -2695,7 +2668,7 @@ export class AnalyticsService {const query = `WITH ${withClauses.join(',\n')}- SELECT+ SELECTcolumn_name,name,count,@@ -2716,7 +2689,7 @@ export class AnalyticsService {}return `- SELECT+ SELECT'${col}' as column_name,name,count,@@ -2914,9 +2887,15 @@ export class AnalyticsService {extractCustomEventsChartData(queryResult, x: string[]): any {const result = {}+ const uniques = Array(x.length).fill(0)+ const sdur = Array(x.length).fill(0)+ let hasUniques = false+ let hasSdur = falsefor (let row = 0; row < _size(queryResult); ++row) {- const { ev = '_unknown_event' } = queryResult[row]+ const rowEventName = queryResult[row].ev+ const ev =+ typeof rowEventName === 'undefined' ? '_unknown_event' : rowEventNameconst dateString = this.generateDateString(queryResult[row])@@ -2928,9 +2907,35 @@ export class AnalyticsService {}result[ev][index] = queryResult[row].count++ if (typeof rowEventName === 'undefined') {+ if (!result['_count']) {+ result['_count'] = Array(x.length).fill(0)+ }++ result['_count'][index] = queryResult[row].count || 0+ }++ if (typeof queryResult[row].uniques !== 'undefined') {+ uniques[index] = queryResult[row].uniques || 0+ hasUniques = true+ }++ if (typeof queryResult[row].sdur !== 'undefined') {+ sdur[index] = queryResult[row].sdur || 0+ hasSdur = true+ }}}+ if (hasUniques) {+ result['_uniques'] = uniques+ }++ if (hasSdur) {+ result['_sdur'] = sdur+ }+return result}@@ -3005,8 +3010,8 @@ export class AnalyticsService {pid,psid,${timeBucketFunc}(toTimeZone(created, {timezone:String})) as tz_created- FROM analytics- PREWHERE pid = {pid:FixedString(12)}+ FROM events+ PREWHERE pid = {pid:FixedString(12)} AND type = 'pageview'WHERE created BETWEEN ${tzFromDate} AND ${tzToDate}${filtersQuery}) as subquery@@ -3054,8 +3059,8 @@ export class AnalyticsService {FROM (SELECT${timeBucketFunc}(toTimeZone(created, {timezone:String})) as tz_created- FROM analytics- PREWHERE pid = {pid:FixedString(12)}+ FROM events+ PREWHERE pid = {pid:FixedString(12)} AND type = 'pageview'WHERE created BETWEEN ${tzFromDate} AND ${tzToDate}${filtersQuery}) as subquery@@ -3082,8 +3087,8 @@ export class AnalyticsService {FROM (SELECT${timeBucketFunc}(toTimeZone(created, {timezone:String})) as tz_created- FROM customEV- PREWHERE pid = {pid:FixedString(12)}+ FROM events+ PREWHERE pid = {pid:FixedString(12)} AND type = 'custom_event'WHERE created BETWEEN ${tzFromDate} AND ${tzToDate}${filtersQuery}) as subquery@@ -3108,8 +3113,8 @@ export class AnalyticsService {FROM (SELECT${timeBucketFunc}(toTimeZone(created, {timezone:String})) as tz_created- FROM errors- PREWHERE pid = {pid:FixedString(12)}+ FROM events+ PREWHERE pid = {pid:FixedString(12)} AND type = 'error'WHERE created BETWEEN ${tzFromDate} AND ${tzToDate}${filtersQuery}) as subquery@@ -3131,15 +3136,31 @@ export class AnalyticsService {const baseQuery = `SELECT${selector},- count() as count+ count() as count,+ countDistinct(subquery.psid) as uniques,+ avgOrNull(sessions_data.duration) as sdurFROM (- SELECT *,- ${timeBucketFunc}(toTimeZone(created, {timezone:String})) as tz_created- FROM customEV+ SELECT+ pid,+ psid,+ ${timeBucketFunc}(toTimeZone(created, {timezone:String})) as tz_created+ FROM eventsWHERE pid = {pid:FixedString(12)}+ AND type = 'custom_event'AND created BETWEEN ${tzFromDate} AND ${tzToDate}${filtersQuery}) as subquery+ LEFT JOIN (+ SELECT+ pid,+ psid,+ dateDiff('second', min(firstSeen), max(lastSeen)) as duration+ FROM sessions+ WHERE pid = {pid:FixedString(12)}+ GROUP BY pid, psid+ ) as sessions_data+ ON subquery.pid = sessions_data.pid+ AND subquery.psid = sessions_data.psidGROUP BY ${groupBy}ORDER BY ${groupBy}`@@ -3147,8 +3168,10 @@ export class AnalyticsService {if (mode === ChartRenderMode.CUMULATIVE) {return `SELECT- *,- sum(count) OVER (ORDER BY ${groupBy}) as count+ ${groupBy},+ sum(count) OVER (ORDER BY ${groupBy}) as count,+ sum(uniques) OVER (ORDER BY ${groupBy}) as uniques,+ sdurFROM (${baseQuery})`}@@ -3179,9 +3202,10 @@ export class AnalyticsService {FROM (SELECT *,${timeBucketFunc}(toTimeZone(created, {timezone:String})) as tz_created- FROM performance+ FROM eventsWHEREpid = {pid:FixedString(12)}+ AND type = 'performance'AND created BETWEEN ${tzFromDate} AND ${tzToDate}${filtersQuery}) as subquery@@ -3214,9 +3238,10 @@ export class AnalyticsService {FROM (SELECT *,${timeBucketFunc}(toTimeZone(created, {timezone:String})) as tz_created- FROM performance+ FROM eventsWHEREpid = {pid:FixedString(12)}+ AND type = 'performance'AND created BETWEEN ${tzFromDate} AND ${tzToDate}${filtersQuery}) as subquery@@ -3274,9 +3299,10 @@ export class AnalyticsService {FROM (SELECT pg, dv, br, os, lc, cc, rg, ct, profileId,${timeBucketFunc}(created) as tz_created- FROM errors+ FROM eventsWHEREpid = {pid:FixedString(12)}+ AND type = 'error'AND created BETWEEN {groupFrom:String} AND {groupTo:String}${filtersQuery}) as subquery@@ -3287,10 +3313,11 @@ export class AnalyticsService {if (mode === ChartRenderMode.CUMULATIVE) {return `SELECT- *,- sum(pageviews) OVER (ORDER BY ${groupBy}) as pageviews,- sum(uniques) OVER (ORDER BY ${groupBy}) as uniques+ ${groupBy},+ sum(count) OVER (ORDER BY ${groupBy}) as count,+ sum(affectedUsers) OVER (ORDER BY ${groupBy}) as affectedUsersFROM (${baseQuery})+ ORDER BY ${groupBy}`}@@ -3355,7 +3382,7 @@ export class AnalyticsService {): Promise<{ name: string; count: number }[]> {const query = `WITH session_first_pages AS (- SELECT+ SELECTpsid,argMin(pg, created) as entry_page${subQuery}@@ -3450,39 +3477,11 @@ export class AnalyticsService {): Promise<object | void> {const { xShifted } = this.generateXAxis(timeBucket, from, to, safeTimezone)- if (customEVFilterApplied) {- const query = this.generateCustomEventsAggregationQuery(- timeBucket,- filtersQuery,- mode,- )-- const { data } = await clickhouse- .query({- query,- query_params: { ...paramsData.params, timezone: safeTimezone },- })- .then((resultSet) => resultSet.json<TrafficCEFilterCHResponse>())-- const uniques =- this.extractCustomEventsChartData(data, xShifted)?._unknown_event || []-- const sdur = Array(_size(xShifted)).fill(0)-- return Promise.resolve({- chart: {- x: xShifted,- visits: uniques,- uniques,- sdur,- },- })- }-- const query = this.generateAnalyticsAggregationQuery(+ const query = this.generateAnalyticsAggregationQueryForScope(timeBucket,filtersQuery,mode,+ customEVFilterApplied,)const { data } = await clickhouse@@ -3490,21 +3489,16 @@ export class AnalyticsService {query,query_params: { ...paramsData.params, timezone: safeTimezone },})- .then((resultSet) => resultSet.json<TrafficCHResponse>())-- const { visits, uniques, sdur } = this.extractChartData(data, xShifted)+ .then((resultSet) =>+ resultSet.json<TrafficCHResponse | TrafficCEFilterCHResponse>(),+ )- // Propagate the previous cumulative value forward for empty buckets- if (mode === ChartRenderMode.CUMULATIVE) {- for (let i = 1; i < visits.length; ++i) {- if (visits[i] === 0) {- visits[i] = visits[i - 1]- }- if (uniques[i] === 0) {- uniques[i] = uniques[i - 1]- }- }- }+ const { visits, uniques, sdur } = this.extractAnalyticsChartDataForScope(+ data,+ xShifted,+ customEVFilterApplied,+ mode,+ )return Promise.resolve({chart: {@@ -3798,9 +3792,10 @@ export class AnalyticsService {FROM (SELECT *,${timeBucketFunc}(toTimeZone(created, {timezone:String})) as tz_created- FROM captcha+ FROM eventsWHEREpid = {pid:FixedString(12)}+ AND type = 'captcha'AND created BETWEEN ${tzFromDate} AND ${tzToDate}${filtersQuery}) as subquery@@ -4050,7 +4045,7 @@ export class AnalyticsService {filtersQuery: string,params: any,): Promise<ICustomEvent> {- const query = `SELECT ev, count() FROM customEV WHERE pid = {pid:FixedString(12)} ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String} GROUP BY ev`+ const query = `SELECT event_name AS ev, count() FROM events WHERE pid = {pid:FixedString(12)} AND type = 'custom_event' ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String} GROUP BY event_name`const result = {}const { data } = await clickhouse@@ -4081,9 +4076,10 @@ export class AnalyticsService {SELECTmeta.key AS keyFROM- analytics+ eventsWHEREpid = {pid:FixedString(12)}+ AND type = 'pageview'${filtersQuery}AND created BETWEEN {groupFrom:String} AND {groupTo:String})@@ -4137,7 +4133,7 @@ export class AnalyticsService {)if (period === 'all') {- const res = await this.calculateTimeBucketForAllTime(pid, 'customEV')+ const res = await this.calculateTimeBucketForAllTime(pid, 'custom_event')diff = res.diff@@ -4165,12 +4161,12 @@ export class AnalyticsService {SELECTmeta.key,meta.value- FROM- customEV+ FROM eventsWHEREpid = {pid:FixedString(12)}+ AND type = 'custom_event'AND created BETWEEN {groupFrom:String} AND {groupTo:String}- AND ev = {event:String}+ AND event_name = {event:String}${filtersQuery})ARRAY JOIN meta.key, meta.value@@ -4229,7 +4225,7 @@ export class AnalyticsService {const [filtersQuery, filtersParams, appliedFilters, customEVFilterApplied] =this.getFiltersQuery(filters, DataType.ANALYTICS)- // We cannot make a query to customEV table using analytics table properties+ // Page properties are only present on pageview-scoped analytics rows.if (customEVFilterApplied) {return {result: [],@@ -4238,7 +4234,7 @@ export class AnalyticsService {}if (period === 'all') {- const res = await this.calculateTimeBucketForAllTime(pid, 'analytics')+ const res = await this.calculateTimeBucketForAllTime(pid, 'pageview')diff = res.diff@@ -4266,10 +4262,10 @@ export class AnalyticsService {SELECTmeta.key,meta.value- FROM- analytics+ FROM eventsWHEREpid = {pid:FixedString(12)}+ AND type = 'pageview'AND created BETWEEN {groupFrom:String} AND {groupTo:String}AND indexOf(meta.key, {property:String}) > 0${filtersQuery}@@ -4319,17 +4315,11 @@ export class AnalyticsService {const query = `SELECT uniqExact(psid) as count- FROM (- SELECT psid FROM analytics- WHERE pid = {pid:FixedString(12)}- AND created >= {since:DateTime}- AND psid IS NOT NULL- UNION ALL- SELECT psid FROM customEV- WHERE pid = {pid:FixedString(12)}- AND created >= {since:DateTime}- AND psid IS NOT NULL- )+ FROM events+ WHERE pid = {pid:FixedString(12)}+ AND type IN ('pageview', 'custom_event', 'error')+ AND created >= {since:DateTime}+ AND psid IS NOT NULL`try {@@ -4367,25 +4357,26 @@ export class AnalyticsService {const tzToDate = `toTimeZone(parseDateTimeBestEffort({groupTo:String}), {timezone:String})`const customEventsFilter =customEvents && customEvents.length > 0- ? 'AND ev IN {customEvents:Array(String)}'+ ? 'AND event_name IN {customEvents:Array(String)}': ''const query = `SELECT${selector},- ev,+ event_name AS ev,count() as countFROM (SELECT *,${timeBucketFunc}(toTimeZone(created, {timezone:String})) as tz_created- FROM customEV+ FROM eventsWHEREpid = {pid:FixedString(12)}+ AND type = 'custom_event'AND created BETWEEN ${tzFromDate} AND ${tzToDate}${filtersQuery}${customEventsFilter}) as subquery- GROUP BY ${groupBy}, ev+ GROUP BY ${groupBy}, event_nameORDER BY ${groupBy}`@@ -4430,16 +4421,21 @@ export class AnalyticsService {}return _map(pages, (page: IPageflow) => {- if (!page.metadata) {- return page+ const { created_utc: _createdUtc, ...displayPage } = page++ if (!displayPage.metadata) {+ return displayPage}return {- ...page,- metadata: _map(page.metadata, ([key, value]: [string, string]) => ({- key,- value,- })),+ ...displayPage,+ metadata: _map(+ displayPage.metadata,+ ([key, value]: [string, string]) => ({+ key,+ value,+ }),+ ),}})}@@ -4452,70 +4448,69 @@ export class AnalyticsService {const queryPages = `WITH events_with_meta AS (SELECT- 'pageview' AS type,- pg AS value,- toTimeZone(analytics.created, {timezone:String}) AS created,- pid,- psid,- arrayFilter(x -> x.1 != '' AND x.2 != '', arrayZip(meta.key, meta.value)) AS metadata- FROM analytics- WHERE- pid = {pid:FixedString(12)}- AND analytics.psid IS NOT NULL- AND CAST(analytics.psid AS String) = {psid:String}-- UNION ALL-- SELECT- 'event' AS type,- ev AS value,- toTimeZone(customEV.created, {timezone:String}) AS created,+ multiIf(event_type = 'pageview', 'pageview', event_type = 'custom_event', 'event', 'error') AS type,+ multiIf(+ event_type = 'pageview', toString(pg),+ event_type = 'custom_event', toString(event_name),+ toString(error_name)+ ) AS value,+ created AS created_utc,+ toTimeZone(created, {timezone:String}) AS created_local,pid,- psid,- arrayFilter(x -> x.1 != '' AND x.2 != '', arrayZip(meta.key, meta.value)) AS metadata- FROM customEV- WHERE- pid = {pid:FixedString(12)}- AND customEV.psid IS NOT NULL- AND CAST(customEV.psid AS String) = {psid:String}-- UNION ALL-- SELECT- 'error' AS type,- errors.name AS value,- toTimeZone(errors.created, {timezone:String}) AS created,- pid,- psid,- [- tuple('message', COALESCE(errors.message, '')),- tuple('lineno', CAST(COALESCE(errors.lineno, 0), 'String')),- tuple('colno', CAST(COALESCE(errors.colno, 0), 'String')),- tuple('filename', COALESCE(errors.filename, ''))- ] AS metadata- FROM errors- WHERE- pid = {pid:FixedString(12)}- AND errors.psid IS NOT NULL- AND CAST(errors.psid AS String) = {psid:String}+ toString(psid) AS psid,+ if(+ event_type = 'error',+ [+ tuple('message', COALESCE(error_message, '')),+ tuple('lineno', toString(COALESCE(lineno, 0))),+ tuple('colno', toString(COALESCE(colno, 0))),+ tuple('filename', COALESCE(error_filename, ''))+ ],+ arrayFilter(x -> x.1 != '' AND x.2 != '', arrayZip(meta_key, meta_value))+ ) AS metadata+ FROM (+ SELECT+ type AS event_type,+ pg,+ event_name,+ error_name,+ error_message,+ lineno,+ colno,+ error_filename,+ created,+ pid,+ psid,+ meta.key AS meta_key,+ meta.value AS meta_value+ FROM events+ WHERE+ pid = {pid:FixedString(12)}+ AND type IN ('pageview', 'custom_event', 'error')+ AND psid IS NOT NULL+ AND toString(psid) = {psid:String}+ ))SELECTtype,value,- created,+ created_local AS created,+ created_utc,metadataFROM events_with_meta- ORDER BY created ASC+ ORDER BY created_utc ASC`const querySessionDetails = `SELECTdv, br, brv, os, osv, lc, ref, so, me, ca, te, co, cc, rg, ct, profileId- FROM analytics+ FROM eventsWHEREpid = {pid:FixedString(12)}- AND CAST(psid, 'String') = {psid:String}+ AND type IN ('pageview', 'custom_event', 'error')+ AND psid IS NOT NULL+ AND toString(psid) = {psid:String}ORDER BY created ASCLIMIT 1;`@@ -4581,7 +4576,7 @@ export class AnalyticsService {}if (!duration && lastSeen && Array.isArray(pages) && pages.length >= 1) {- const first = dayjs(pages[0].created)+ const first = dayjs(pages[0].created_utc)const diffSeconds = dayjs(lastSeen).diff(first, 'second')if (diffSeconds > 0) {duration = diffSeconds@@ -4589,8 +4584,8 @@ export class AnalyticsService {}if (!duration && Array.isArray(pages) && pages.length >= 2) {- const first = dayjs(pages[0].created)- const last = dayjs(pages[pages.length - 1].created)+ const first = dayjs(pages[0].created_utc)+ const last = dayjs(pages[pages.length - 1].created_utc)const diffSeconds = last.diff(first, 'second')if (diffSeconds > 0) {duration = diffSeconds@@ -4601,11 +4596,12 @@ export class AnalyticsService {const querySessionDetailsFromCustomEV = `SELECTdv, br, brv, os, osv, lc, ref, so, me, ca, te, co, cc, rg, ct, profileId- FROM customEV+ FROM eventsWHEREpid = {pid:FixedString(12)}+ AND type IN ('custom_event', 'error')AND psid IS NOT NULL- AND CAST(psid, 'String') = {psid:String}+ AND toString(psid) = {psid:String}ORDER BY created ASCLIMIT 1;`@@ -4626,12 +4622,12 @@ export class AnalyticsService {if (!_isEmpty(pages)) {const from = dayjs- .utc(pages[0].created)+ .utc(pages[0].created_utc).tz(safeTimezone).startOf('minute').format('YYYY-MM-DD HH:mm:ss')const to = dayjs- .utc(pages[_size(pages) - 1].created)+ .utc(pages[_size(pages) - 1].created_utc).tz(safeTimezone).endOf('minute').format('YYYY-MM-DD HH:mm:ss')@@ -4649,8 +4645,8 @@ export class AnalyticsService {{params: {...paramsData.params,- groupFrom: pages[0].created,- groupTo: pages[_size(pages) - 1].created,+ groupFrom: from,+ groupTo: to,},},safeTimezone,@@ -4663,7 +4659,7 @@ export class AnalyticsService {let isLive = falseif (!_isEmpty(pages)) {- const lastActivityTime = dayjs(pages[pages.length - 1].created)+ const lastActivityTime = dayjs(pages[pages.length - 1].created_utc)const liveThresholdTime = dayjs().subtract(LIVE_SESSION_THRESHOLD_SECONDS,@@ -4693,61 +4689,16 @@ export class AnalyticsService {skip = 0,customEVFilterApplied = false,): Promise<object | void> {- let primaryEventsSubquery: string-- if (customEVFilterApplied) {- primaryEventsSubquery = `- SELECT- CAST(customEV.psid, 'String') AS psidCasted,- customEV.pid,- customEV.cc,- customEV.os,- customEV.br,- toTimeZone(customEV.created, {timezone:String}) AS created_for_grouping- FROM customEV- WHERE- customEV.pid = {pid:FixedString(12)}- AND customEV.psid IS NOT NULL- AND customEV.created BETWEEN {groupFrom:String} AND {groupTo:String}- ${filtersQuery}- `- } else {- primaryEventsSubquery = `- SELECT- CAST(analytics.psid, 'String') AS psidCasted,- analytics.pid,- analytics.cc,- analytics.os,- analytics.br,- toTimeZone(analytics.created, {timezone:String}) AS created_for_grouping- FROM analytics- WHERE- analytics.pid = {pid:FixedString(12)}- AND analytics.psid IS NOT NULL- AND analytics.created BETWEEN {groupFrom:String} AND {groupTo:String}- ${filtersQuery}- UNION ALL- SELECT- CAST(customEV.psid, 'String') AS psidCasted,- customEV.pid,- customEV.cc,- customEV.os,- customEV.br,- toTimeZone(customEV.created, {timezone:String}) AS created_for_grouping- FROM customEV- WHERE- customEV.pid = {pid:FixedString(12)}- AND customEV.psid IS NOT NULL- AND customEV.created BETWEEN {groupFrom:String} AND {groupTo:String}- ${filtersQuery}- `- }+ const primaryEventsSubquery = this.buildSessionsListPrimaryEventsSubquery(+ filtersQuery,+ customEVFilterApplied,+ )const query = `WITH distinct_sessions_filtered AS (SELECTpsidCasted,- pid,+ pid,any(cc) AS cc,any(os) AS os,any(br) AS br,@@ -4757,39 +4708,39 @@ export class AnalyticsService {GROUP BY psidCasted, pid),pageview_counts AS (- SELECT+ SELECTCAST(psid, 'String') AS psidCasted,- pid,- count() as count- FROM analytics- WHERE pid = {pid:FixedString(12)} AND psid IS NOT NULL+ pid,+ count() as count+ FROM events+ WHERE pid = {pid:FixedString(12)} AND type = 'pageview' AND psid IS NOT NULLAND created BETWEEN {groupFrom:String} AND {groupTo:String}GROUP BY psidCasted, pid),event_counts AS (- SELECT+ SELECTCAST(psid, 'String') AS psidCasted,- pid,- count() as count- FROM customEV- WHERE pid = {pid:FixedString(12)} AND psid IS NOT NULL+ pid,+ count() as count+ FROM events+ WHERE pid = {pid:FixedString(12)} AND type = 'custom_event' AND psid IS NOT NULLAND created BETWEEN {groupFrom:String} AND {groupTo:String}GROUP BY psidCasted, pid),error_counts AS (- SELECT+ SELECTCAST(psid, 'String') AS psidCasted,- pid,- count() as count- FROM errors- WHERE pid = {pid:FixedString(12)} AND psid IS NOT NULL+ pid,+ count() as count+ FROM events+ WHERE pid = {pid:FixedString(12)} AND type = 'error' AND psid IS NOT NULLAND created BETWEEN {groupFrom:String} AND {groupTo:String}GROUP BY psidCasted, pid),session_duration_agg AS (- SELECT- CAST(psid, 'String') AS psidCasted,- pid,+ SELECT+ CAST(psid, 'String') AS psidCasted,+ pid,dateDiff('second', min(firstSeen), max(lastSeen)) as avg_duration,argMax(profileId, lastSeen) as profileIdFROM sessions@@ -4816,7 +4767,7 @@ export class AnalyticsService {COALESCE(errc.count, 0) AS errors,dsf.sessionStart,dsf.lastActivity,- if(dateDiff('second', dsf.lastActivity, now()) < ${LIVE_SESSION_THRESHOLD_SECONDS}, 1, 0) AS isLive,+ if(dateDiff('second', toTimeZone(dsf.lastActivity, 'UTC'), toTimeZone(now(), 'UTC')) < ${LIVE_SESSION_THRESHOLD_SECONDS}, 1, 0) AS isLive,sda.avg_duration AS sdur,sda.profileId AS profileId,if(startsWith(sda.profileId, '${AnalyticsService.PROFILE_PREFIX_USER}'), 1, 0) AS isIdentified,@@ -4848,6 +4799,43 @@ export class AnalyticsService {return data}+ private buildSessionsListPrimaryEventsSubquery(+ filtersQuery: string,+ customEVFilterApplied: boolean,+ ): string {+ const scopedSessionFilter = customEVFilterApplied+ ? `+ AND CAST(psid, 'String') IN (+ SELECT DISTINCT CAST(psid, 'String')+ FROM events+ WHERE+ pid = {pid:FixedString(12)}+ AND type = 'custom_event'+ AND psid IS NOT NULL+ AND created BETWEEN {groupFrom:String} AND {groupTo:String}+ ${filtersQuery}+ )+ `+ : filtersQuery++ return `+ SELECT+ CAST(psid, 'String') AS psidCasted,+ pid,+ cc,+ os,+ br,+ toTimeZone(created, {timezone:String}) AS created_for_grouping+ FROM events+ WHERE+ pid = {pid:FixedString(12)}+ AND type IN ('pageview', 'custom_event', 'error')+ AND psid IS NOT NULL+ AND created BETWEEN {groupFrom:String} AND {groupTo:String}+ ${scopedSessionFilter}+ `+ }+async getErrorsList(options: string,filtersQuery: string,@@ -4878,10 +4866,18 @@ export class AnalyticsService {count(DISTINCT psid) as sessions,status.statusFROM (- SELECT eid, name, message, filename, psid, profileId, toTimeZone(errors.created, {timezone:String}) AS created- FROM errors+ SELECT+ eid,+ error_name AS name,+ error_message AS message,+ error_filename AS filename,+ psid,+ profileId,+ toTimeZone(created, {timezone:String}) AS created+ FROM eventsWHERE pid = {pid:FixedString(12)}- AND errors.created BETWEEN {groupFrom:String} AND {groupTo:String}+ AND type = 'error'+ AND created BETWEEN {groupFrom:String} AND {groupTo:String}${filtersQuery}) AS errorsLEFT JOIN (@@ -4895,7 +4891,7 @@ export class AnalyticsService {${parsedOptions?.showResolved? ''- : "WHERE status.status = 'active' OR status.status = 'regressed'"+ : "WHERE status.status = 'active' OR status.status = 'regressed' OR status.status IS NULL"}GROUP BY errors.eid, status.statusORDER BY last_seen DESC@@ -4940,14 +4936,15 @@ export class AnalyticsService {FROM (SELECTeid,- name,- message,- filename,+ error_name AS name,+ error_message AS message,+ error_filename AS filename,colno,lineno,stackTrace- FROM errors+ FROM eventsWHERE pid = {pid:FixedString(12)}+ AND type = 'error'AND eid = {eid:FixedString(32)}AND created BETWEEN {groupFrom:String} AND {groupTo:String}ORDER BY created DESC@@ -4970,9 +4967,10 @@ export class AnalyticsService {max(created) AS last_seen,min(created) AS first_seen,count() AS count- FROM errors+ FROM eventsWHEREpid = {pid:FixedString(12)}+ AND type = 'error'AND eid = {eid:FixedString(32)};`@@ -4985,9 +4983,10 @@ export class AnalyticsService {SELECTmeta.key,meta.value- FROM errors+ FROM eventsWHEREpid = {pid:FixedString(12)}+ AND type = 'error'AND eid = {eid:FixedString(32)}AND created BETWEEN {groupFrom:String} AND {groupTo:String})@@ -5034,8 +5033,8 @@ export class AnalyticsService {timeBucket,groupFrom,groupTo,- `FROM errors WHERE eid = {eid:FixedString(32)} AND created BETWEEN {groupFrom:String} AND {groupTo:String}`,- 'AND eid = {eid:FixedString(32)}',+ `FROM events WHERE type = 'error' AND pid = {pid:FixedString(12)} AND eid = {eid:FixedString(32)} AND created BETWEEN {groupFrom:String} AND {groupTo:String}`,+ 'AND pid = {pid:FixedString(12)} AND eid = {eid:FixedString(32)}',paramsData,safeTimezone,ChartRenderMode.PERIODICAL,@@ -5061,17 +5060,22 @@ export class AnalyticsService {groupTo: string,timeBucket: string,showResolved: boolean,+ sessionFiltersQuery = '',+ sessionFiltersParams: Record<string, unknown> = {},): Promise<any> {const resolvedFilter = showResolved? '': "AND (status.status = 'active' OR status.status = 'regressed' OR status.status IS NULL)"- // Get total sessions from analytics table for the time range+ // Get total sessions from all matching events for the time rangeconst queryTotalSessions = `SELECT count(DISTINCT psid) as totalSessions- FROM analytics+ FROM eventsWHERE pid = {pid:FixedString(12)}+ AND type IN ('pageview', 'custom_event', 'error')+ AND psid IS NOT NULLAND created BETWEEN {groupFrom:String} AND {groupTo:String}+ ${sessionFiltersQuery}`// Get error stats: total errors, unique errors, affected sessions, affected users@@ -5081,7 +5085,7 @@ export class AnalyticsService {count(DISTINCT eid) as uniqueErrors,count(DISTINCT psid) as affectedSessions,count(DISTINCT profileId) as affectedUsers- FROM errors+ FROM events AS errorsLEFT JOIN (SELECT eid, argMax(status, updated) AS statusFROM error_statuses@@ -5089,6 +5093,8 @@ export class AnalyticsService {GROUP BY eid) AS status ON errors.eid = status.eidWHERE pid = {pid:FixedString(12)}+ AND type = 'error'+ AND psid IS NOT NULLAND created BETWEEN {groupFrom:String} AND {groupTo:String}${filtersQuery}${resolvedFilter}@@ -5098,12 +5104,12 @@ export class AnalyticsService {const queryMostFrequentError = `SELECTerrors.eid as eid,- any(errors.name) as name,- any(errors.message) as message,+ any(errors.error_name) as name,+ any(errors.error_message) as message,count(*) as count,count(DISTINCT errors.profileId) as usersAffected,max(errors.created) as lastSeen- FROM errors+ FROM events AS errorsLEFT JOIN (SELECT eid, argMax(status, updated) AS statusFROM error_statuses@@ -5111,6 +5117,7 @@ export class AnalyticsService {GROUP BY eid) AS status ON errors.eid = status.eidWHERE errors.pid = {pid:FixedString(12)}+ AND errors.type = 'error'AND errors.created BETWEEN {groupFrom:String} AND {groupTo:String}${filtersQuery}${resolvedFilter}@@ -5135,7 +5142,7 @@ export class AnalyticsService {SELECTprofileId,${timeBucketFunc}(toTimeZone(created, {timezone:String})) as tz_created- FROM errors+ FROM events AS errorsLEFT JOIN (SELECT eid, argMax(status, updated) AS statusFROM error_statuses@@ -5143,6 +5150,7 @@ export class AnalyticsService {GROUP BY eid) AS status ON errors.eid = status.eidWHERE pid = {pid:FixedString(12)}+ AND type = 'error'AND created BETWEEN {groupFrom:String} AND {groupTo:String}${filtersQuery}${resolvedFilter}@@ -5165,7 +5173,10 @@ export class AnalyticsService {clickhouse.query({query: queryTotalSessions,- query_params: paramsData.params,+ query_params: {+ ...paramsData.params,+ ...sessionFiltersParams,+ },}).then((resultSet) => resultSet.json<any>()).then(({ data }) => data[0]),@@ -5254,9 +5265,11 @@ export class AnalyticsService {): Promise<{ sessions: any[]; total: number }> {const queryCount = `SELECT count(DISTINCT psid) as total- FROM errors+ FROM eventsWHERE pid = {pid:FixedString(12)}+ AND type = 'error'AND eid = {eid:FixedString(32)}+ AND psid IS NOT NULLAND created BETWEEN {groupFrom:String} AND {groupTo:String}`@@ -5264,16 +5277,17 @@ export class AnalyticsService {SELECT DISTINCTCAST(errors.psid, 'String') as psid,any(errors.profileId) as profileId,- any(analytics.cc) as cc,- any(analytics.br) as br,- any(analytics.os) as os,+ any(errors.cc) as cc,+ any(errors.br) as br,+ any(errors.os) as os,min(errors.created) as firstErrorAt,max(errors.created) as lastErrorAt,count(*) as errorCount- FROM errors- LEFT JOIN analytics ON errors.psid = analytics.psid AND errors.pid = analytics.pid+ FROM events AS errorsWHERE errors.pid = {pid:FixedString(12)}+ AND errors.type = 'error'AND errors.eid = {eid:FixedString(32)}+ AND errors.psid IS NOT NULLAND errors.created BETWEEN {groupFrom:String} AND {groupTo:String}GROUP BY errors.psidORDER BY lastErrorAt DESC@@ -5319,7 +5333,7 @@ export class AnalyticsService {{},)- const query = `SELECT count(DISTINCT eid) as count FROM errors WHERE pid = {pid:FixedString(12)} AND eid IN (${_map(+ const query = `SELECT count(DISTINCT eid) as count FROM events WHERE pid = {pid:FixedString(12)} AND type = 'error' AND eid IN (${_map(params,(val, key) => `{${key}:FixedString(32)}`,).join(', ')})`@@ -5482,12 +5496,13 @@ export class AnalyticsService {key,round(sum(${aggFunction}), 2) AS sum,round(avg(${aggFunction}), 2) AS avg- FROM customEV+ FROM eventsARRAY JOIN meta.key AS key, meta.value AS valueWHEREpid = {pid:FixedString(12)}+ AND type = 'custom_event'AND key = {metricKey:String}- AND ev = {customEventName:String}+ AND event_name = {customEventName:String}AND created BETWEEN {periodFormatted:String} AND {now:String}${kvQuery}${filtersQuery}@@ -5500,12 +5515,13 @@ export class AnalyticsService {key,round(sum(${aggFunction}), 2) AS sum,round(avg(${aggFunction}), 2) AS avg- FROM customEV+ FROM eventsARRAY JOIN meta.key AS key, meta.value AS valueWHEREpid = {pid:FixedString(12)}+ AND type = 'custom_event'AND key = {metricKey:String}- AND ev = {customEventName:String}+ AND event_name = {customEventName:String}AND created BETWEEN {periodSubtracted:String} AND {periodFormatted:String}${kvQuery}${filtersQuery}@@ -5599,75 +5615,28 @@ export class AnalyticsService {}}- async getProfilesList(- pid: string,+ private buildProfilesListDataCTE(filtersQuery: string,- paramsData: any,- safeTimezone: string,- take = 30,- skip = 0,- profileType: 'all' | 'anonymous' | 'identified' = 'all',- customEVFilterApplied = false,- ): Promise<object[]> {- let profileTypeFilter = ''- if (profileType === 'anonymous') {- profileTypeFilter = `AND profileId LIKE '${AnalyticsService.PROFILE_PREFIX_ANON}%'`- } else if (profileType === 'identified') {- profileTypeFilter = `AND profileId LIKE '${AnalyticsService.PROFILE_PREFIX_USER}%'`- }-- let allProfileDataCTE: string-- if (customEVFilterApplied) {- allProfileDataCTE = `- all_profile_data AS (- SELECT- profileId,- psid,- cc,- os,- br,- dv,- created,- 1 AS isPageview,- 0 AS isEvent- FROM analytics- WHERE pid = {pid:FixedString(12)}- AND created BETWEEN {groupFrom:String} AND {groupTo:String}- AND profileId IS NOT NULL- AND profileId != ''- ${profileTypeFilter}+ profileTypeFilter: string,+ customEVFilterApplied: boolean,+ ): string {+ const scopedProfileFilter = customEVFilterApplied+ ? `AND profileId IN (SELECT DISTINCT profileId- FROM customEV+ FROM eventsWHERE pid = {pid:FixedString(12)}+ AND type = 'custom_event'AND created BETWEEN {groupFrom:String} AND {groupTo:String}AND profileId IS NOT NULLAND profileId != ''${profileTypeFilter}${filtersQuery})- UNION ALL- SELECT- profileId,- psid,- cc,- os,- br,- dv,- created,- 0 AS isPageview,- 1 AS isEvent- FROM customEV- WHERE pid = {pid:FixedString(12)}- AND created BETWEEN {groupFrom:String} AND {groupTo:String}- AND profileId IS NOT NULL- AND profileId != ''- ${profileTypeFilter}- ${filtersQuery}- )`- } else {- allProfileDataCTE = `+ `+ : filtersQuery++ return `all_profile_data AS (SELECTprofileId,@@ -5677,36 +5646,43 @@ export class AnalyticsService {br,dv,created,- 1 AS isPageview,- 0 AS isEvent- FROM analytics+ if(type = 'pageview', 1, 0) AS isPageview,+ if(type = 'custom_event', 1, 0) AS isEvent,+ if(type = 'error', 1, 0) AS isError+ FROM eventsWHERE pid = {pid:FixedString(12)}+ AND type IN ('pageview', 'custom_event', 'error')AND created BETWEEN {groupFrom:String} AND {groupTo:String}AND profileId IS NOT NULLAND profileId != ''${profileTypeFilter}- ${filtersQuery}- UNION ALL- SELECT- profileId,- psid,- cc,- os,- br,- dv,- created,- 0 AS isPageview,- 1 AS isEvent- FROM customEV- WHERE pid = {pid:FixedString(12)}- AND created BETWEEN {groupFrom:String} AND {groupTo:String}- AND profileId IS NOT NULL- AND profileId != ''- ${profileTypeFilter}- ${filtersQuery}+ ${scopedProfileFilter})`+ }++ async getProfilesList(+ pid: string,+ filtersQuery: string,+ paramsData: any,+ safeTimezone: string,+ take = 30,+ skip = 0,+ profileType: 'all' | 'anonymous' | 'identified' = 'all',+ customEVFilterApplied = false,+ ): Promise<object[]> {+ let profileTypeFilter = ''+ if (profileType === 'anonymous') {+ profileTypeFilter = `AND profileId LIKE '${AnalyticsService.PROFILE_PREFIX_ANON}%'`+ } else if (profileType === 'identified') {+ profileTypeFilter = `AND profileId LIKE '${AnalyticsService.PROFILE_PREFIX_USER}%'`}+ const allProfileDataCTE = this.buildProfilesListDataCTE(+ filtersQuery,+ profileTypeFilter,+ customEVFilterApplied,+ )+const query = `WITH ${allProfileDataCTE},profile_aggregated AS (@@ -5715,6 +5691,7 @@ export class AnalyticsService {sum(isPageview) AS pageviewsCount,countDistinct(psid) AS sessionsCount,sum(isEvent) AS eventsCount,+ sum(isError) AS errorsCount,min(created) AS firstSeen,max(created) AS lastSeen,any(cc) AS cc_agg,@@ -5723,24 +5700,13 @@ export class AnalyticsService {any(dv) AS dv_aggFROM all_profile_dataGROUP BY profileId- ),- profile_errors AS (- SELECT- profileId,- count() AS errorsCount- FROM errors- WHERE pid = {pid:FixedString(12)}- AND profileId IS NOT NULL- AND profileId != ''- ${profileTypeFilter}- GROUP BY profileId)SELECTpa.profileId AS profileId,pa.sessionsCount AS sessionsCount,pa.pageviewsCount AS pageviewsCount,pa.eventsCount AS eventsCount,- COALESCE(perr.errorsCount, 0) AS errorsCount,+ pa.errorsCount AS errorsCount,pa.firstSeen AS firstSeen,pa.lastSeen AS lastSeen,pa.cc_agg AS cc,@@ -5748,7 +5714,6 @@ export class AnalyticsService {pa.br_agg AS br,pa.dv_agg AS dvFROM profile_aggregated AS pa- LEFT JOIN profile_errors AS perr ON pa.profileId = perr.profileIdORDER BY pa.lastSeen DESCLIMIT {take:UInt32}OFFSET {skip:UInt32}@@ -5785,54 +5750,83 @@ export class AnalyticsService {AND profileId = {profileId:String}`- // Query avg duration from analytics table (more accurate than sessions table)+ // Prefer pageview timings, but fall back to sessions for event/error-only profiles.const queryAvgDuration = `- SELECT- avg(session_duration) AS avgDuration- FROM (+ WITH pageview_avg AS (SELECT- psid,- dateDiff('second', min(created), max(created)) AS session_duration- FROM analytics- WHERE pid = {pid:FixedString(12)}- AND profileId = {profileId:String}- AND psid IS NOT NULL- GROUP BY psid- HAVING session_duration > 0+ avgOrNull(session_duration) AS avgDuration+ FROM (+ SELECT+ psid,+ dateDiff('second', min(created), max(created)) AS session_duration+ FROM events+ WHERE pid = {pid:FixedString(12)}+ AND type = 'pageview'+ AND profileId = {profileId:String}+ AND psid IS NOT NULL+ GROUP BY psid+ HAVING session_duration > 0+ )+ ),+ sessions_avg AS (+ SELECT+ avgOrNull(session_duration) AS avgDuration+ FROM (+ SELECT+ psid,+ dateDiff('second', min(firstSeen), max(lastSeen)) AS session_duration+ FROM sessions FINAL+ WHERE pid = {pid:FixedString(12)}+ AND profileId = {profileId:String}+ GROUP BY psid+ HAVING session_duration > 0+ ))+ SELECT+ coalesce(nullIf(pageview_avg.avgDuration, 0), sessions_avg.avgDuration, 0) AS avgDuration+ FROM pageview_avg, sessions_avg`- // Query analytics for accurate pageview countconst queryPageviews = `SELECT count() AS pageviewsCount- FROM analytics+ FROM eventsWHERE pid = {pid:FixedString(12)}+ AND type = 'pageview'AND profileId = {profileId:String}`- // Query customEV for accurate events countconst queryEvents = `SELECT count() AS eventsCount- FROM customEV+ FROM events+ WHERE pid = {pid:FixedString(12)}+ AND type = 'custom_event'+ AND profileId = {profileId:String}+ `++ const queryErrors = `+ SELECT count() AS errorsCount+ FROM eventsWHERE pid = {pid:FixedString(12)}+ AND type = 'error'AND profileId = {profileId:String}`// Query for device/location detailsconst queryDetails = `SELECT- any(cc) AS cc,- any(rg) AS rg,- any(ct) AS ct,- any(os) AS os,- any(osv) AS osv,- any(br) AS br,- any(brv) AS brv,- any(dv) AS dv,- any(lc) AS lc- FROM analytics+ argMax(cc, created) AS cc,+ argMax(rg, created) AS rg,+ argMax(ct, created) AS ct,+ argMax(os, created) AS os,+ argMax(osv, created) AS osv,+ argMax(br, created) AS br,+ argMax(brv, created) AS brv,+ argMax(dv, created) AS dv,+ argMax(lc, created) AS lc+ FROM eventsWHERE pid = {pid:FixedString(12)}AND profileId = {profileId:String}+ AND type IN ('pageview', 'custom_event', 'error')`const params = { pid, profileId }@@ -5842,6 +5836,7 @@ export class AnalyticsService {avgDurationResult,pageviewsResult,eventsResult,+ errorsResult,detailsResult,] = await Promise.all([clickhouse@@ -5856,6 +5851,9 @@ export class AnalyticsService {clickhouse.query({ query: queryEvents, query_params: params }).then((resultSet) => resultSet.json()),+ clickhouse+ .query({ query: queryErrors, query_params: params })+ .then((resultSet) => resultSet.json()),clickhouse.query({ query: queryDetails, query_params: params }).then((resultSet) => resultSet.json()),@@ -5868,6 +5866,7 @@ export class AnalyticsService {const avgDuration = (avgDurationResult.data[0] || {}) as Record<string, any>const pageviews = (pageviewsResult.data[0] || {}) as Record<string, any>const events = (eventsResult.data[0] || {}) as Record<string, any>+ const errors = (errorsResult.data[0] || {}) as Record<string, any>const details = (detailsResult.data[0] || {}) as Record<string, any>return {@@ -5876,6 +5875,7 @@ export class AnalyticsService {sessionsCount: sessionCount.sessionsCount || 0,pageviewsCount: pageviews.pageviewsCount || 0,eventsCount: events.eventsCount || 0,+ errorsCount: errors.errorsCount || 0,firstSeen: sessionCount.firstSeen,lastSeen: sessionCount.lastSeen,avgDuration: avgDuration.avgDuration || 0,@@ -5892,8 +5892,9 @@ export class AnalyticsService {SELECTpg AS page,count() AS count- FROM analytics+ FROM eventsWHERE pid = {pid:FixedString(12)}+ AND type = 'pageview'AND profileId = {profileId:String}GROUP BY pgORDER BY count DESC@@ -5922,30 +5923,15 @@ export class AnalyticsService {.format('YYYY-MM-DD')const query = `- WITH all_activity AS (- SELECT- toDate(created) AS date,- 1 AS isPageview,- 0 AS isEvent- FROM analytics- WHERE pid = {pid:FixedString(12)}- AND profileId = {profileId:String}- AND created >= {startDate:Date}- UNION ALL- SELECT- toDate(created) AS date,- 0 AS isPageview,- 1 AS isEvent- FROM customEV- WHERE pid = {pid:FixedString(12)}- AND profileId = {profileId:String}- AND created >= {startDate:Date}- )SELECT- date,- sum(isPageview) AS pageviews,- sum(isEvent) AS events- FROM all_activity+ toDate(created) AS date,+ countIf(type = 'pageview') AS pageviews,+ countIf(type = 'custom_event') AS events+ FROM events+ WHERE pid = {pid:FixedString(12)}+ AND type IN ('pageview', 'custom_event')+ AND profileId = {profileId:String}+ AND created >= {startDate:Date}GROUP BY dateORDER BY date ASC`@@ -5980,8 +5966,9 @@ export class AnalyticsService {SELECT${timeBucketFunc}(toTimeZone(created, {timezone:String})) AS time,count() AS count- FROM analytics+ FROM eventsWHERE pid = {pid:FixedString(12)}+ AND type = 'pageview'AND profileId = {profileId:String}AND created BETWEEN {from:String} AND {to:String}GROUP BY time@@ -5992,8 +5979,9 @@ export class AnalyticsService {SELECT${timeBucketFunc}(toTimeZone(created, {timezone:String})) AS time,count() AS count- FROM customEV+ FROM eventsWHERE pid = {pid:FixedString(12)}+ AND type = 'custom_event'AND profileId = {profileId:String}AND created BETWEEN {from:String} AND {to:String}GROUP BY time@@ -6004,8 +5992,9 @@ export class AnalyticsService {SELECT${timeBucketFunc}(toTimeZone(created, {timezone:String})) AS time,count() AS count- FROM errors+ FROM eventsWHERE pid = {pid:FixedString(12)}+ AND type = 'error'AND profileId = {profileId:String}AND created BETWEEN {from:String} AND {to:String}GROUP BY time@@ -6066,70 +6055,26 @@ export class AnalyticsService {}}- async getProfileSessionsList(- pid: string,- profileId: string,+ private buildProfileSessionsEventsCTE(filtersQuery: string,- paramsData: any,- safeTimezone: string,- take = 30,- skip = 0,- customEVFilterApplied = false,- ): Promise<object[]> {- let allProfileEventsCTE: string+ customEVFilterApplied: boolean,+ ): string {+ const scopedSessionFilter = customEVFilterApplied+ ? `+ AND CAST(psid, 'String') IN (+ SELECT DISTINCT CAST(psid, 'String')+ FROM events+ WHERE pid = {pid:FixedString(12)}+ AND type = 'custom_event'+ AND profileId = {profileId:String}+ AND psid IS NOT NULL+ AND created BETWEEN {groupFrom:String} AND {groupTo:String}+ ${filtersQuery}+ )+ `+ : filtersQuery- if (customEVFilterApplied) {- allProfileEventsCTE = `- all_profile_events AS (- SELECT- all_events.psidCasted,- all_events.pid,- all_events.profileId,- all_events.cc,- all_events.os,- all_events.br,- all_events.created_tz- FROM (- SELECT- CAST(psid, 'String') AS psidCasted,- pid,- profileId,- cc,- os,- br,- toTimeZone(created, {timezone:String}) AS created_tz- FROM analytics- WHERE pid = {pid:FixedString(12)}- AND profileId = {profileId:String}- AND psid IS NOT NULL- AND created BETWEEN {groupFrom:String} AND {groupTo:String}- UNION ALL- SELECT- CAST(psid, 'String') AS psidCasted,- pid,- profileId,- cc,- os,- br,- toTimeZone(created, {timezone:String}) AS created_tz- FROM customEV- WHERE pid = {pid:FixedString(12)}- AND profileId = {profileId:String}- AND psid IS NOT NULL- AND created BETWEEN {groupFrom:String} AND {groupTo:String}- ) AS all_events- WHERE all_events.psidCasted IN (- SELECT DISTINCT CAST(psid, 'String')- FROM customEV- WHERE pid = {pid:FixedString(12)}- AND profileId = {profileId:String}- AND psid IS NOT NULL- AND created BETWEEN {groupFrom:String} AND {groupTo:String}- ${filtersQuery}- )- )`- } else {- allProfileEventsCTE = `+ return `all_profile_events AS (SELECTCAST(psid, 'String') AS psidCasted,@@ -6139,29 +6084,30 @@ export class AnalyticsService {os,br,toTimeZone(created, {timezone:String}) AS created_tz- FROM analytics- WHERE pid = {pid:FixedString(12)}- AND profileId = {profileId:String}- AND psid IS NOT NULL- AND created BETWEEN {groupFrom:String} AND {groupTo:String}- ${filtersQuery}- UNION ALL- SELECT- CAST(psid, 'String') AS psidCasted,- pid,- profileId,- cc,- os,- br,- toTimeZone(created, {timezone:String}) AS created_tz- FROM customEV+ FROM eventsWHERE pid = {pid:FixedString(12)}+ AND type IN ('pageview', 'custom_event', 'error')AND profileId = {profileId:String}AND psid IS NOT NULLAND created BETWEEN {groupFrom:String} AND {groupTo:String}- ${filtersQuery}+ ${scopedSessionFilter})`- }+ }++ async getProfileSessionsList(+ pid: string,+ profileId: string,+ filtersQuery: string,+ paramsData: any,+ safeTimezone: string,+ take = 30,+ skip = 0,+ customEVFilterApplied = false,+ ): Promise<object[]> {+ const allProfileEventsCTE = this.buildProfileSessionsEventsCTE(+ filtersQuery,+ customEVFilterApplied,+ )const query = `WITH ${allProfileEventsCTE},@@ -6183,8 +6129,9 @@ export class AnalyticsService {CAST(psid, 'String') AS psidCasted,pid,count() as count- FROM analytics+ FROM eventsWHERE pid = {pid:FixedString(12)}+ AND type = 'pageview'AND profileId = {profileId:String}AND psid IS NOT NULLAND created BETWEEN {groupFrom:String} AND {groupTo:String}@@ -6195,8 +6142,9 @@ export class AnalyticsService {CAST(psid, 'String') AS psidCasted,pid,count() as count- FROM customEV+ FROM eventsWHERE pid = {pid:FixedString(12)}+ AND type = 'custom_event'AND profileId = {profileId:String}AND psid IS NOT NULLAND created BETWEEN {groupFrom:String} AND {groupTo:String}@@ -6207,8 +6155,9 @@ export class AnalyticsService {CAST(psid, 'String') AS psidCasted,pid,count() as count- FROM errors+ FROM eventsWHERE pid = {pid:FixedString(12)}+ AND type = 'error'AND profileId = {profileId:String}AND psid IS NOT NULLAND created BETWEEN {groupFrom:String} AND {groupTo:String}@@ -6234,7 +6183,7 @@ export class AnalyticsService {COALESCE(errc.count, 0) AS errors,ps.sessionStart,ps.lastActivity,- if(dateDiff('second', ps.lastActivity, now()) < ${LIVE_SESSION_THRESHOLD_SECONDS}, 1, 0) AS isLive,+ if(dateDiff('second', toTimeZone(ps.lastActivity, 'UTC'), toTimeZone(now(), 'UTC')) < ${LIVE_SESSION_THRESHOLD_SECONDS}, 1, 0) AS isLive,sda.avg_duration AS sdurFROM profile_sessions psLEFT JOIN pageview_counts pc ON ps.psidCasted = pc.psidCasted AND ps.pid = pc.piddiff --git a/backend/apps/community/src/analytics/interfaces/index.ts b/backend/apps/community/src/analytics/interfaces/index.tsindex 84da70732..2dace48be 100644--- a/backend/apps/community/src/analytics/interfaces/index.ts+++ b/backend/apps/community/src/analytics/interfaces/index.ts@@ -183,9 +183,10 @@ export interface IOverallPerformance {}export interface IPageflow {- type: 'pageview' | 'event'+ type: 'pageview' | 'event' | 'error'value: stringcreated: string+ created_utc?: stringmetadata?: [string, string][]}diff --git a/backend/apps/community/src/analytics/utils/transformers.ts b/backend/apps/community/src/analytics/utils/transformers.tsindex b420cdac8..7ec1ed491 100644--- a/backend/apps/community/src/analytics/utils/transformers.ts+++ b/backend/apps/community/src/analytics/utils/transformers.ts@@ -8,7 +8,7 @@ import utc from 'dayjs/plugin/utc'dayjs.extend(utc)const processMetaKV = (- rawMeta: Record<string, string>,+ rawMeta: Record<string, string> | null | undefined,): { 'meta.key': string[] | null; 'meta.value': string[] | null } => {if (!rawMeta || _isEmpty(rawMeta)) {return {@@ -23,10 +23,10 @@ const processMetaKV = (}}-interface TrafficTransformerOptions {- psid: string- profileId: string+interface CommonOptions {pid: string+ psid?: string | null+ profileId?: string | nullhost?: string | nullpg?: string | nulldv?: string | null@@ -52,262 +52,42 @@ interface TrafficTransformerOptions {meta?: Record<string, string> | null}-export const trafficTransformer = ({- psid,- profileId,- pid,- host,- pg,- dv,- br,- brv,- os,- osv,- lc,- ref,- so,- me,- ca,- te,- co,- cc,- rg,- rgc,- ct,- isp,- og,- ut,- ctp,- meta,-}: TrafficTransformerOptions) => {- return {- psid,- profileId,- pid,- host: host || null,- pg: pg || null,- dv: dv || null,- br: br || null,- brv: brv || null,- os: os || null,- osv: osv || null,- lc: lc || null,- ref: ref || null,- so: so || null,- me: me || null,- ca: ca || null,- te: te || null,- co: co || null,- cc: cc || null,- rg: rg || null,- rgc: rgc || null,- ct: ct || null,- isp: isp || null,- og: og || null,- ut: ut || null,- ctp: ctp || null,- ...processMetaKV(meta),- created: dayjs.utc().format('YYYY-MM-DD HH:mm:ss'),- }+interface PageviewOptions extends CommonOptions {+ type: 'pageview'}-interface CustomEventTransformerOptions {- psid: string- profileId: string- pid: string- host?: string | null+interface CustomEventOptions extends CommonOptions {+ type: 'custom_event'ev: string- pg?: string | null- dv?: string | null- br?: string | null- brv?: string | null- os?: string | null- osv?: string | null- lc?: string | null- ref?: string | null- so?: string | null- me?: string | null- ca?: string | null- te?: string | null- co?: string | null- cc?: string | null- rg?: string | null- rgc?: string | null- ct?: string | null- isp?: string | null- og?: string | null- ut?: string | null- ctp?: string | null- meta?: Record<string, string> | null-}--export const customEventTransformer = ({- psid,- profileId,- pid,- host,- ev,- pg,- dv,- br,- brv,- os,- osv,- lc,- ref,- so,- me,- ca,- te,- co,- cc,- rg,- rgc,- ct,- isp,- og,- ut,- ctp,- meta,-}: CustomEventTransformerOptions) => {- return {- psid,- profileId,- pid,- host: host || null,- ev,- pg: pg || null,- dv: dv || null,- br: br || null,- brv: brv || null,- os: os || null,- osv: osv || null,- lc: lc || null,- ref: ref || null,- so: so || null,- me: me || null,- ca: ca || null,- te: te || null,- co: co || null,- cc: cc || null,- rg: rg || null,- rgc: rgc || null,- ct: ct || null,- isp: isp || null,- og: og || null,- ut: ut || null,- ctp: ctp || null,- ...processMetaKV(meta),- created: dayjs.utc().format('YYYY-MM-DD HH:mm:ss'),- }}-interface ErrorEventTransformerOptions {- psid: string- profileId: string+interface ErrorEventOptions extends CommonOptions {+ type: 'error'eid: string- pid: string- host?: string | null- pg?: string | null- dv?: string | null- br?: string | null- brv?: string | null- os?: string | null- osv?: string | null- lc?: string | null- cc?: string | null- rg?: string | null- rgc?: string | null- ct?: string | null- isp?: string | null- og?: string | null- ut?: string | null- ctp?: string | nullname?: string | nullmessage?: string | nulllineno?: number | nullcolno?: number | nullfilename?: string | nullstackTrace?: string | null- meta?: Record<string, string> | null-}--export const errorEventTransformer = ({- psid,- profileId,- eid,- pid,- host,- pg,- dv,- br,- brv,- os,- osv,- lc,- cc,- rg,- rgc,- ct,- isp,- og,- ut,- ctp,- name,- message,- lineno,- colno,- filename,- stackTrace,- meta,-}: ErrorEventTransformerOptions) => {- return {- psid,- profileId,- eid,- pid,- host: host || null,- pg: pg || null,- dv: dv || null,- br: br || null,- brv: brv || null,- os: os || null,- osv: osv || null,- lc: lc || null,- cc: cc || null,- rg: rg || null,- rgc: rgc || null,- ct: ct || null,- isp: isp || null,- og: og || null,- ut: ut || null,- ctp: ctp || null,- name: name || null,- message: message || null,- lineno: lineno || null,- colno: colno || null,- filename: filename || null,- stackTrace: stackTrace || null,- ...processMetaKV(meta),- created: dayjs.utc().format('YYYY-MM-DD HH:mm:ss'),- }}-interface PerformanceTransformerOptions {- pid: string- host?: string | null- pg?: string | null- dv?: string | null- br?: string | null- brv?: string | null- cc?: string | null- rg?: string | null- rgc?: string | null- ct?: string | null- isp?: string | null- og?: string | null- ut?: string | null- ctp?: string | null+interface PerformanceOptions extends Omit<+ CommonOptions,+ | 'psid'+ | 'profileId'+ | 'os'+ | 'osv'+ | 'lc'+ | 'ref'+ | 'so'+ | 'me'+ | 'ca'+ | 'te'+ | 'co'+ | 'meta'+> {+ type: 'performance'dns: numbertls: numberconn: number@@ -318,53 +98,131 @@ interface PerformanceTransformerOptions {ttfb: number}-export const performanceTransformer = ({- pid,- host,- pg,- dv,- br,- brv,- cc,- rg,- rgc,- ct,- isp,- og,- ut,- ctp,- dns,- tls,- conn,- response,- render,- domLoad,- pageLoad,- ttfb,-}: PerformanceTransformerOptions) => {+interface CaptchaOptions {+ type: 'captcha'+ pid: string+ dv?: string | null+ br?: string | null+ os?: string | null+ cc?: string | null+ timestamp?: number+}++type EventTransformerOptions =+ | PageviewOptions+ | CustomEventOptions+ | ErrorEventOptions+ | PerformanceOptions+ | CaptchaOptions++const buildCommon = (opts: CommonOptions) => ({+ pid: opts.pid,+ psid: opts.psid ?? null,+ profileId: opts.profileId ?? null,+ host: opts.host || null,+ pg: opts.pg || null,+ dv: opts.dv || null,+ br: opts.br || null,+ brv: opts.brv || null,+ os: opts.os || null,+ osv: opts.osv || null,+ lc: opts.lc || null,+ ref: opts.ref || null,+ so: opts.so || null,+ me: opts.me || null,+ ca: opts.ca || null,+ te: opts.te || null,+ co: opts.co || null,+ cc: opts.cc || null,+ rg: opts.rg || null,+ rgc: opts.rgc || null,+ ct: opts.ct || null,+ isp: opts.isp || null,+ og: opts.og || null,+ ut: opts.ut || null,+ ctp: opts.ctp || null,+ ...processMetaKV(opts.meta),+})++// Single transformer for all event kinds; the discriminator is `type`.+// Renames at the storage boundary: ev -> event_name, name -> error_name,+// message -> error_message, filename -> error_filename. DTO and API field+// names stay as the legacy short forms.+export const eventTransformer = (opts: EventTransformerOptions) => {+ const created = dayjs.utc().format('YYYY-MM-DD HH:mm:ss')++ if (opts.type === 'pageview') {+ return {+ type: 'pageview' as const,+ ...buildCommon(opts),+ created,+ }+ }++ if (opts.type === 'custom_event') {+ return {+ type: 'custom_event' as const,+ ...buildCommon(opts),+ event_name: opts.ev,+ created,+ }+ }++ if (opts.type === 'error') {+ return {+ type: 'error' as const,+ ...buildCommon(opts),+ eid: opts.eid,+ error_name: opts.name || null,+ error_message: opts.message || null,+ stackTrace: opts.stackTrace || null,+ lineno: opts.lineno ?? null,+ colno: opts.colno ?? null,+ error_filename: opts.filename || null,+ created,+ }+ }++ if (opts.type === 'performance') {+ return {+ type: 'performance' as const,+ pid: opts.pid,+ host: opts.host || null,+ pg: opts.pg || null,+ dv: opts.dv || null,+ br: opts.br || null,+ brv: opts.brv || null,+ cc: opts.cc || null,+ rg: opts.rg || null,+ rgc: opts.rgc || null,+ ct: opts.ct || null,+ isp: opts.isp || null,+ og: opts.og || null,+ ut: opts.ut || null,+ ctp: opts.ctp || null,+ dns: _round(opts.dns),+ tls: _round(opts.tls),+ conn: _round(opts.conn),+ response: _round(opts.response),+ render: _round(opts.render),+ domLoad: _round(opts.domLoad),+ pageLoad: _round(opts.pageLoad),+ ttfb: _round(opts.ttfb),+ created,+ }+ }++ // captchareturn {- pid,- host: host || null,- pg: pg || null,- dv: dv || null,- br: br || null,- brv: brv || null,- cc: cc || null,- rg: rg || null,- rgc: rgc || null,- ct: ct || null,- isp: isp || null,- og: og || null,- ut: ut || null,- ctp: ctp || null,- dns: _round(dns),- tls: _round(tls),- conn: _round(conn),- response: _round(response),- render: _round(render),- domLoad: _round(domLoad),- pageLoad: _round(pageLoad),- ttfb: _round(ttfb),- created: dayjs.utc().format('YYYY-MM-DD HH:mm:ss'),+ type: 'captcha' as const,+ pid: opts.pid,+ dv: opts.dv || null,+ br: opts.br || null,+ os: opts.os || null,+ cc: opts.cc || null,+ created:+ typeof opts.timestamp === 'number'+ ? dayjs.utc(opts.timestamp).format('YYYY-MM-DD HH:mm:ss')+ : created,}}diff --git a/backend/apps/community/src/captcha/captcha.service.ts b/backend/apps/community/src/captcha/captcha.service.tsindex 921e6d7b7..99ac3516c 100644--- a/backend/apps/community/src/captcha/captcha.service.ts+++ b/backend/apps/community/src/captcha/captcha.service.ts@@ -17,7 +17,7 @@ import {} from '../common/constants'import { getIPDetails } from '../common/utils'import { GeneratedChallenge } from './interfaces/generated-captcha'-import { captchaTransformer } from './utils/transformers'+import { eventTransformer } from '../analytics/utils/transformers'import { clickhouse } from '../common/integrations/clickhouse'dayjs.extend(utc)@@ -152,18 +152,19 @@ export class CaptchaService {const osName = ua.os.nameconst { country } = getIPDetails(ip)- const transformed = captchaTransformer(+ const transformed = eventTransformer({+ type: 'captcha',pid,- deviceType,- browserName,- osName,- country,+ dv: deviceType,+ br: browserName,+ os: osName,+ cc: country,timestamp,- )+ })try {await clickhouse.insert({- table: 'captcha',+ table: 'events',format: 'JSONEachRow',values: [transformed],clickhouse_settings: {diff --git a/backend/apps/community/src/captcha/utils/transformers.ts b/backend/apps/community/src/captcha/utils/transformers.tsdeleted file mode 100644index 3a0d21d43..000000000--- a/backend/apps/community/src/captcha/utils/transformers.ts+++ /dev/null@@ -1,22 +0,0 @@-import dayjs from 'dayjs'-import utc from 'dayjs/plugin/utc'--dayjs.extend(utc)--export const captchaTransformer = (- pid: string,- dv: string | null,- br: string | null,- os: string | null,- cc: string | null,- timestamp: number,-) => {- return {- pid,- dv: dv || null,- br: br || null,- os: os || null,- cc: cc || null,- created: dayjs.utc(timestamp).format('YYYY-MM-DD HH:mm:ss'),- }-}diff --git a/backend/apps/community/src/data-import/data-import.processor.ts b/backend/apps/community/src/data-import/data-import.processor.tsindex 0d0646d1b..5f3ebfe97 100644--- a/backend/apps/community/src/data-import/data-import.processor.ts+++ b/backend/apps/community/src/data-import/data-import.processor.ts@@ -8,6 +8,7 @@ import { Job } from 'bullmq'import { DataImportService } from './data-import.service'import { DataImportStatus } from './entity/data-import.entity'import { getMapper } from './mappers'+import { ImportError } from './mappers/mapper.interface'import { clickhouse } from '../common/integrations/clickhouse'const CLICKHOUSE_DB = process.env.CLICKHOUSE_DATABASE || 'analytics'@@ -85,8 +86,7 @@ export class DataImportProcessor extends WorkerHost {let minDate: string | null = nulllet maxDate: string | null = null- const analyticsBatch: Record<string, unknown>[] = []- const customEVBatch: Record<string, unknown>[] = []+ const eventsBatch: Record<string, unknown>[] = []try {for await (const row of mapper.createRowStream(@@ -102,28 +102,17 @@ export class DataImportProcessor extends WorkerHost {if (!maxDate || created > maxDate) maxDate = created}- if (row.table === 'analytics') {- analyticsBatch.push(row.data)- } else {- customEVBatch.push(row.data)- }-- if (analyticsBatch.length >= BATCH_SIZE) {- await this.flushBatch('analytics', analyticsBatch)- importedRows += analyticsBatch.length- analyticsBatch.length = 0- }+ eventsBatch.push({ type: row.type, ...row.data })- if (customEVBatch.length >= BATCH_SIZE) {- await this.flushBatch('customEV', customEVBatch)- importedRows += customEVBatch.length- customEVBatch.length = 0+ if (eventsBatch.length >= BATCH_SIZE) {+ await this.flushBatch(eventsBatch)+ importedRows += eventsBatch.length+ eventsBatch.length = 0}if (totalRows % 10000 === 0) {const progress = {- importedRows:- importedRows + analyticsBatch.length + customEVBatch.length,+ importedRows: importedRows + eventsBatch.length,totalRows,}@@ -141,14 +130,9 @@ export class DataImportProcessor extends WorkerHost {}}- if (analyticsBatch.length > 0) {- await this.flushBatch('analytics', analyticsBatch)- importedRows += analyticsBatch.length- }-- if (customEVBatch.length > 0) {- await this.flushBatch('customEV', customEVBatch)- importedRows += customEVBatch.length+ if (eventsBatch.length > 0) {+ await this.flushBatch(eventsBatch)+ importedRows += eventsBatch.length}await this.dataImportService.markCompleted(importId, projectId, {@@ -176,10 +160,15 @@ export class DataImportProcessor extends WorkerHost {)}+ const userMessage =+ error instanceof ImportError+ ? error.message+ : 'An unexpected error occurred while processing the import. Please try again or contact support.'+await this.dataImportService.markFailed(importId,projectId,- error.message,+ userMessage,)}} finally {@@ -187,12 +176,9 @@ export class DataImportProcessor extends WorkerHost {}}- private async flushBatch(- table: 'analytics' | 'customEV',- batch: Record<string, unknown>[],- ): Promise<void> {+ private async flushBatch(batch: Record<string, unknown>[]): Promise<void> {await clickhouse.insert({- table: `${CLICKHOUSE_DB}.${table}`,+ table: `${CLICKHOUSE_DB}.events`,values: batch,format: 'JSONEachRow',})diff --git a/backend/apps/community/src/data-import/data-import.service.ts b/backend/apps/community/src/data-import/data-import.service.tsindex ac9be0c50..e1453fbe0 100644--- a/backend/apps/community/src/data-import/data-import.service.ts+++ b/backend/apps/community/src/data-import/data-import.service.ts@@ -281,11 +281,7 @@ export class DataImportService {importId: number,): Promise<void> {await clickhouse.command({- query: `ALTER TABLE ${CLICKHOUSE_DB}.analytics DELETE WHERE pid = {pid:FixedString(12)} AND importID = {importID:UInt8}`,- query_params: { pid: projectId, importID: importId },- })- await clickhouse.command({- query: `ALTER TABLE ${CLICKHOUSE_DB}.customEV DELETE WHERE pid = {pid:FixedString(12)} AND importID = {importID:UInt8}`,+ query: `ALTER TABLE ${CLICKHOUSE_DB}.events DELETE WHERE pid = {pid:FixedString(12)} AND importID = {importID:UInt8} AND type IN ('pageview', 'custom_event')`,query_params: { pid: projectId, importID: importId },})}@@ -330,7 +326,7 @@ export class DataImportService {): Promise<boolean> {const result = await clickhouse.query({- query: `SELECT count() as cnt FROM ${CLICKHOUSE_DB}.analytics WHERE pid = {pid:FixedString(12)} AND importID IS NOT NULL AND created >= {from:DateTime} AND created <= {to:DateTime} LIMIT 1`,+ query: `SELECT count() as cnt FROM ${CLICKHOUSE_DB}.events WHERE pid = {pid:FixedString(12)} AND type = 'pageview' AND importID IS NOT NULL AND created >= {from:DateTime} AND created <= {to:DateTime} LIMIT 1`,query_params: { pid: projectId, from, to },}).then((rs) => rs.json<{ cnt: string }>())diff --git a/backend/apps/community/src/data-import/mappers/fathom.mapper.ts b/backend/apps/community/src/data-import/mappers/fathom.mapper.tsindex 55c578f7c..1ae5397ca 100644--- a/backend/apps/community/src/data-import/mappers/fathom.mapper.ts+++ b/backend/apps/community/src/data-import/mappers/fathom.mapper.ts@@ -1,7 +1,11 @@import * as fs from 'fs'import { parse } from 'csv-parse'-import { ImportMapper, AnalyticsImportRow } from './mapper.interface'+import {+ ImportMapper,+ AnalyticsImportRow,+ ImportError,+} from './mapper.interface'import {normalizeNull,truncate,@@ -41,7 +45,7 @@ function detectFileType(headers: string[]): {const missing = COMMON_REQUIRED.filter((c) => !normalized.includes(c))if (missing.length > 0) {- throw new Error(+ throw new ImportError(`CSV does not appear to be a Fathom Analytics export. Missing required columns: ${missing.join(', ')}.`,)}@@ -54,7 +58,7 @@ function detectFileType(headers: string[]): {return { type: 'events', normalized }}- throw new Error(+ throw new ImportError('CSV does not appear to be a Fathom Analytics export. Expected a pageviews export (with "pageviews" column) or an events export (with "event_name" column).',)}@@ -202,19 +206,19 @@ export class FathomMapper implements ImportMapper {if (fileType === 'pageviews') {for (let i = 0; i < count; i++) {- yield { table: 'analytics', data: baseData }+ yield { type: 'pageview', data: baseData }}} else {const eventName = truncate(normalizeNull(row.event_name), 256) || ''- const evData = { ...baseData, ev: eventName }+ const evData = { ...baseData, event_name: eventName }for (let i = 0; i < count; i++) {- yield { table: 'customEV', data: evData }+ yield { type: 'custom_event', data: evData }}}}if (!headerChecked) {- throw new Error(+ throw new ImportError('CSV appears empty or is missing a header row. Please upload a CSV export from Fathom Analytics.',)}diff --git a/backend/apps/community/src/data-import/mappers/mapper.interface.ts b/backend/apps/community/src/data-import/mappers/mapper.interface.tsindex b7bdc9bf0..73ea8538b 100644--- a/backend/apps/community/src/data-import/mappers/mapper.interface.ts+++ b/backend/apps/community/src/data-import/mappers/mapper.interface.ts@@ -6,7 +6,7 @@ export class ImportError extends Error {}export interface AnalyticsImportRow {- table: 'analytics' | 'customEV'+ type: 'pageview' | 'custom_event'data: Record<string, unknown>}diff --git a/backend/apps/community/src/data-import/mappers/plausible.mapper.ts b/backend/apps/community/src/data-import/mappers/plausible.mapper.tsindex 99141bb58..f0c5926d3 100644--- a/backend/apps/community/src/data-import/mappers/plausible.mapper.ts+++ b/backend/apps/community/src/data-import/mappers/plausible.mapper.ts@@ -859,7 +859,7 @@ export class PlausibleMapper implements ImportMapper {created,})- yield { table: 'analytics', data }+ yield { type: 'pageview', data }}}@@ -889,8 +889,8 @@ export class PlausibleMapper implements ImportMapper {page: null,created,})- data.ev = truncate(ev.name, 256) || ''- yield { table: 'customEV', data }+ data.event_name = truncate(ev.name, 256) || ''+ yield { type: 'custom_event', data }}}}diff --git a/backend/apps/community/src/data-import/mappers/simple-analytics.mapper.ts b/backend/apps/community/src/data-import/mappers/simple-analytics.mapper.tsindex e95c04049..b95e37380 100644--- a/backend/apps/community/src/data-import/mappers/simple-analytics.mapper.ts+++ b/backend/apps/community/src/data-import/mappers/simple-analytics.mapper.ts@@ -1,7 +1,11 @@import * as fs from 'fs'import { parse } from 'csv-parse'-import { ImportMapper, AnalyticsImportRow } from './mapper.interface'+import {+ ImportMapper,+ ImportError,+ AnalyticsImportRow,+} from './mapper.interface'import {normalizeNull,truncate,@@ -20,7 +24,7 @@ function validateHeaders(headers: string[]): string[] {)if (missingColumns.length > 0) {- throw new Error(+ throw new ImportError(`CSV does not appear to be a Simple Analytics export. Missing required columns: ${missingColumns.join(', ')}.`,)}@@ -133,7 +137,7 @@ export class SimpleAnalyticsMapper implements ImportMapper {hostname,)- const cc = normalizeNull(row.country_code)+ const cc = normalizeNull(row.country_code)?.toUpperCase() || nullconst validCC = cc && /^[A-Z]{2}$/.test(cc) ? cc : nullconst locale = buildLocale(@@ -172,17 +176,17 @@ export class SimpleAnalyticsMapper implements ImportMapper {const datapoint = normalizeNull(row.datapoint)if (!datapoint || datapoint === 'pageview') {- yield { table: 'analytics', data: baseData }+ yield { type: 'pageview', data: baseData }} else {yield {- table: 'customEV',- data: { ...baseData, ev: truncate(datapoint, 256) || '' },+ type: 'custom_event',+ data: { ...baseData, event_name: truncate(datapoint, 256) || '' },}}}if (!headerChecked) {- throw new Error(+ throw new ImportError('CSV appears empty or is missing a header row. Please upload the raw CSV export from Simple Analytics.',)}diff --git a/backend/apps/community/src/data-import/mappers/umami.mapper.ts b/backend/apps/community/src/data-import/mappers/umami.mapper.tsindex a1514ae46..3d8c9d9b8 100644--- a/backend/apps/community/src/data-import/mappers/umami.mapper.ts+++ b/backend/apps/community/src/data-import/mappers/umami.mapper.ts@@ -4,7 +4,11 @@ import * as os from 'os'import { Unzip, UnzipInflate } from 'fflate'import { parse } from 'csv-parse'-import { ImportMapper, AnalyticsImportRow } from './mapper.interface'+import {+ ImportMapper,+ ImportError,+ AnalyticsImportRow,+} from './mapper.interface'import {normalizeNull,truncate,@@ -188,7 +192,7 @@ export class UmamiMapper implements ImportMapper {file.originalSize > MAX_UMAMI_CSV_BYTES) {fail(- new Error(+ new ImportError(`${WEBSITE_EVENT_CSV} exceeds the ${MAX_UMAMI_CSV_BYTES} byte limit.`,),)@@ -213,7 +217,7 @@ export class UmamiMapper implements ImportMapper {extractedBytes += chunk.lengthif (extractedBytes > MAX_UMAMI_CSV_BYTES) {fail(- new Error(+ new ImportError(`${WEBSITE_EVENT_CSV} exceeds the ${MAX_UMAMI_CSV_BYTES} byte limit.`,),)@@ -263,7 +267,7 @@ export class UmamiMapper implements ImportMapper {if (!foundEntry) {fail(- new Error(+ new ImportError(`ZIP does not contain ${WEBSITE_EVENT_CSV}. Please upload the export ZIP from Umami.`,),)@@ -368,12 +372,12 @@ export class UmamiMapper implements ImportMapper {}if (eventType === '1') {- yield { table: 'analytics', data: baseData }+ yield { type: 'pageview', data: baseData }} else {const eventName = truncate(normalizeNull(row.event_name), 256) || ''yield {- table: 'customEV',- data: { ...baseData, ev: eventName },+ type: 'custom_event',+ data: { ...baseData, event_name: eventName },}}}diff --git a/backend/apps/community/src/goal/goal.controller.ts b/backend/apps/community/src/goal/goal.controller.tsindex 0d9e12b71..37f85f30a 100644--- a/backend/apps/community/src/goal/goal.controller.ts+++ b/backend/apps/community/src/goal/goal.controller.ts@@ -252,38 +252,38 @@ export class GoalController {await this.goalService.delete(id)}- private buildGoalMatchCondition(- goal: Goal,- _table: 'analytics' | 'customEV',- ): { condition: string; params: Record<string, string> } {+ private buildGoalMatchCondition(goal: Goal): {+ condition: string+ params: Record<string, string>+ } {const params: Record<string, string> = {}if (goal.type === GoalType.CUSTOM_EVENT) {- // For custom events, match the event nameif (goal.matchType === GoalMatchType.EXACT) {params.goalValue = goal.value || ''- return { condition: `ev = {goalValue:String}`, params }- } else {- // Contains match- params.goalValue = goal.value || ''- return {- condition: `ev ILIKE concat('%', {goalValue:String}, '%')`,- params,- }+ return { condition: `event_name = {goalValue:String}`, params }}- } else {- // For pageview goals, match the page path- if (goal.matchType === GoalMatchType.EXACT) {- params.goalValue = goal.value || ''- return { condition: `pg = {goalValue:String}`, params }- } else {- // Contains match- params.goalValue = goal.value || ''- return {- condition: `pg ILIKE concat('%', {goalValue:String}, '%')`,- params,- }+ if ((goal.value || '').trim() === '') {+ return { condition: '1=0', params: {} }}+ params.goalValue = goal.value || ''+ return {+ condition: `event_name ILIKE concat('%', {goalValue:String}, '%')`,+ params,+ }+ }++ if (goal.matchType === GoalMatchType.EXACT) {+ params.goalValue = goal.value || ''+ return { condition: `pg = {goalValue:String}`, params }+ }+ if ((goal.value || '').trim() === '') {+ return { condition: '1=0', params: {} }+ }+ params.goalValue = goal.value || ''+ return {+ condition: `pg ILIKE concat('%', {goalValue:String}, '%')`,+ params,}}@@ -347,9 +347,10 @@ export class GoalController {safeTimezone,)- const table = goal.type === GoalType.CUSTOM_EVENT ? 'customEV' : 'analytics'+ const goalType =+ goal.type === GoalType.CUSTOM_EVENT ? 'custom_event' : 'pageview'const { condition: matchCondition, params: matchParams } =- this.buildGoalMatchCondition(goal, table)+ this.buildGoalMatchCondition(goal)const { condition: metaCondition, params: metaParams } =this.buildMetadataCondition(goal)@@ -358,9 +359,10 @@ export class GoalController {SELECTcount(*) as conversions,uniqExact(psid) as uniqueSessions- FROM ${table}+ FROM eventsWHEREpid = {pid:FixedString(12)}+ AND type = {goalType:String}AND ${matchCondition}${metaCondition}AND created BETWEEN {groupFrom:String} AND {groupTo:String}@@ -368,6 +370,7 @@ export class GoalController {const queryParams = {pid: goal.projectId,+ goalType,groupFrom: groupFromUTC,groupTo: groupToUTC,...matchParams,@@ -386,9 +389,10 @@ export class GoalController {// Get total unique sessions for conversion rateconst totalSessionsQuery = `SELECT uniqExact(psid) as totalSessions- FROM analytics+ FROM eventsWHEREpid = {pid:FixedString(12)}+ AND type = 'pageview'AND created BETWEEN {groupFrom:String} AND {groupTo:String}`@@ -566,7 +570,8 @@ export class GoalController {safeTimezone,)- const table = goal.type === GoalType.CUSTOM_EVENT ? 'customEV' : 'analytics'+ const goalType =+ goal.type === GoalType.CUSTOM_EVENT ? 'custom_event' : 'pageview'const timeBucketFunc = Object.prototype.hasOwnProperty.call(timeBucketConversion,resolvedTimeBucket,@@ -576,7 +581,7 @@ export class GoalController {const [selector, groupBy] = this.getGroupSubquery(resolvedTimeBucket)const { condition: matchCondition, params: matchParams } =- this.buildGoalMatchCondition(goal, table)+ this.buildGoalMatchCondition(goal)const { condition: metaCondition, params: metaParams } =this.buildMetadataCondition(goal)@@ -588,9 +593,10 @@ export class GoalController {FROM (SELECT *,${timeBucketFunc}(toTimeZone(created, {timezone:String})) as tz_created- FROM ${table}+ FROM eventsWHEREpid = {pid:FixedString(12)}+ AND type = {goalType:String}AND ${matchCondition}${metaCondition}AND created BETWEEN {groupFrom:String} AND {groupTo:String}@@ -601,6 +607,7 @@ export class GoalController {const queryParams = {pid: goal.projectId,+ goalType,groupFrom: groupFromUTC,groupTo: groupToUTC,timezone: safeTimezone,diff --git a/backend/apps/community/src/project/project.controller.ts b/backend/apps/community/src/project/project.controller.tsindex 757444527..1c98b7939 100644--- a/backend/apps/community/src/project/project.controller.ts+++ b/backend/apps/community/src/project/project.controller.ts@@ -576,11 +576,7 @@ export class ProjectController {try {await clickhouse.command({- query: `ALTER TABLE analytics DELETE WHERE pid={pid:FixedString(12)}`,- query_params: { pid: id },- })- await clickhouse.command({- query: `ALTER TABLE customEV DELETE WHERE pid={pid:FixedString(12)}`,+ query: `ALTER TABLE events DELETE WHERE pid={pid:FixedString(12)} AND type IN ('pageview', 'custom_event')`,query_params: { pid: id },})return 'Project reset successfully'@@ -1021,11 +1017,11 @@ export class ProjectController {try {await deleteProjectSharesByProjectClickhouse(id)await clickhouse.command({- query: `ALTER TABLE analytics DELETE WHERE pid={pid:FixedString(12)}`,+ query: `ALTER TABLE events DELETE WHERE pid={pid:FixedString(12)} AND type IN ('pageview', 'custom_event', 'error', 'performance', 'captcha')`,query_params: { pid: id },})await clickhouse.command({- query: `ALTER TABLE customEV DELETE WHERE pid={pid:FixedString(12)}`,+ query: `ALTER TABLE error_statuses DELETE WHERE pid={pid:FixedString(12)}`,query_params: { pid: id },})await deleteProjectRedis(id)diff --git a/backend/apps/community/src/project/project.service.ts b/backend/apps/community/src/project/project.service.tsindex 0d6147406..160c46ac3 100644--- a/backend/apps/community/src/project/project.service.ts+++ b/backend/apps/community/src/project/project.service.ts@@ -248,36 +248,10 @@ export class ProjectService {)const query = `- SELECT- pid,- CASE- WHEN EXISTS (- SELECT 1- FROM analytics- WHERE pid IN (${pids})- )- OR EXISTS (- SELECT 1- FROM customEV- WHERE pid IN (${pids})- )- THEN 1- ELSE 0- END AS exists- FROM- (- SELECT DISTINCT pid- FROM- (- SELECT pid- FROM analytics- WHERE pid IN (${pids})- UNION ALL- SELECT pid- FROM customEV- WHERE pid IN (${pids})- ) AS t- );+ SELECT DISTINCT pid+ FROM events+ WHERE pid IN (${pids})+ AND type IN ('pageview', 'custom_event')`const { data } = await clickhouse@@ -310,23 +284,10 @@ export class ProjectService {)const query = `- SELECT- pid,- CASE- WHEN EXISTS (- SELECT 1- FROM errors- WHERE pid IN (${pids})- )- THEN 1- ELSE 0- END AS exists- FROM- (- SELECT DISTINCT pid- FROM errors- WHERE pid IN (${pids})- );+ SELECT DISTINCT pid+ FROM events+ WHERE pid IN (${pids})+ AND type = 'error'`const { data } = await clickhouse@@ -359,23 +320,10 @@ export class ProjectService {)const query = `- SELECT- pid,- CASE- WHEN EXISTS (- SELECT 1- FROM captcha- WHERE pid IN (${pids})- )- THEN 1- ELSE 0- END AS exists- FROM- (- SELECT DISTINCT pid- FROM captcha- WHERE pid IN (${pids})- );+ SELECT DISTINCT pid+ FROM events+ WHERE pid IN (${pids})+ AND type = 'captcha'`const { data } = await clickhouse@@ -394,10 +342,7 @@ export class ProjectService {to: string,): Promise<void> {const queries = [- 'ALTER TABLE analytics DELETE WHERE pid = {pid:FixedString(12)} AND created BETWEEN {from:String} AND {to:String}',- 'ALTER TABLE customEV DELETE WHERE pid = {pid:FixedString(12)} AND created BETWEEN {from:String} AND {to:String}',- 'ALTER TABLE performance DELETE WHERE pid = {pid:FixedString(12)} AND created BETWEEN {from:String} AND {to:String}',- 'ALTER TABLE errors DELETE WHERE pid = {pid:FixedString(12)} AND created BETWEEN {from:String} AND {to:String}',+ "ALTER TABLE events DELETE WHERE pid = {pid:FixedString(12)} AND type IN ('pageview', 'custom_event', 'performance', 'error', 'captcha') AND created BETWEEN {from:String} AND {to:String}",'ALTER TABLE error_statuses DELETE WHERE pid = {pid:FixedString(12)} AND created BETWEEN {from:String} AND {to:String}',]diff --git a/backend/apps/community/src/user/user.controller.ts b/backend/apps/community/src/user/user.controller.tsindex 64dd758fd..cb3e4fe8b 100644--- a/backend/apps/community/src/user/user.controller.ts+++ b/backend/apps/community/src/user/user.controller.ts@@ -181,10 +181,7 @@ export class UserController {if (!_isEmpty(projects)) {const pidArray = projects.map((el) => el.id)const queries = [- 'ALTER TABLE analytics DELETE WHERE pid IN ({pids:Array(FixedString(12))})',- 'ALTER TABLE customEV DELETE WHERE pid IN ({pids:Array(FixedString(12))})',- 'ALTER TABLE performance DELETE WHERE pid IN ({pids:Array(FixedString(12))})',- 'ALTER TABLE errors DELETE WHERE pid IN ({pids:Array(FixedString(12))})',+ 'ALTER TABLE events DELETE WHERE pid IN ({pids:Array(FixedString(12))})','ALTER TABLE error_statuses DELETE WHERE pid IN ({pids:Array(FixedString(12))})',]await deleteProjectsByUserIdClickhouse(id)diff --git a/backend/meta/clickhouse/generate-dummy-data.js b/backend/meta/clickhouse/generate-dummy-data.jsindex 1b5f9b8fa..004d81d45 100644--- a/backend/meta/clickhouse/generate-dummy-data.js+++ b/backend/meta/clickhouse/generate-dummy-data.js@@ -115,10 +115,10 @@ const chalk = {cyan: text => `\x1b[36m${text}\x1b[0m`,}-const insertData = async (pid, rowCount, processedRecords, table) => {+const insertData = async (pid, rowCount, processedRecords) => {try {await clickhouse.insert({- table,+ table: 'events',format: 'JSONEachRow',values: processedRecords,clickhouse_settings: {@@ -154,6 +154,7 @@ const generateAnalyticsData = async (pid, rowCount, from, to) => {for (let i = 0; i < rowCount; ++i) {records.push({+ type: 'pageview',psid:Math.random() < 0.05? faker.helpers.arrayElement(PSIDS)@@ -193,7 +194,7 @@ const generateAnalyticsData = async (pid, rowCount, from, to) => {),)- insertData(pid, rowCount, records, 'analytics')+ insertData(pid, rowCount, records)}const generateCaptchaData = async (pid, rowCount, from, to) => {@@ -201,6 +202,7 @@ const generateCaptchaData = async (pid, rowCount, from, to) => {for (let i = 0; i < rowCount; ++i) {records.push({+ type: 'captcha',pid,dv: faker.helpers.arrayElement(DEVICES),br: faker.helpers.arrayElement(BROWSERS),@@ -223,7 +225,7 @@ const generateCaptchaData = async (pid, rowCount, from, to) => {),)- insertData(pid, rowCount, records, 'captcha')+ insertData(pid, rowCount, records)}const generateCustomEventsData = async (pid, rowCount, from, to) => {@@ -231,8 +233,9 @@ const generateCustomEventsData = async (pid, rowCount, from, to) => {for (let i = 0; i < rowCount; ++i) {records.push({+ type: 'custom_event',pid,- ev: faker.helpers.arrayElement(CUSTOM_EVENTS),+ event_name: faker.helpers.arrayElement(CUSTOM_EVENTS),pg: faker.helpers.arrayElement(PAGES),dv: faker.helpers.arrayElement(DEVICES),br: faker.helpers.arrayElement(BROWSERS),@@ -267,7 +270,7 @@ const generateCustomEventsData = async (pid, rowCount, from, to) => {),)- insertData(pid, rowCount, records, 'customEV')+ insertData(pid, rowCount, records)}const generatePerformanceData = async (pid, rowCount, from, to) => {@@ -275,6 +278,7 @@ const generatePerformanceData = async (pid, rowCount, from, to) => {for (let i = 0; i < rowCount; ++i) {records.push({+ type: 'performance',pid,pg: faker.helpers.arrayElement(PAGES),dv: faker.helpers.arrayElement(DEVICES),@@ -308,7 +312,7 @@ const generatePerformanceData = async (pid, rowCount, from, to) => {),)- insertData(pid, rowCount, records, 'performance')+ insertData(pid, rowCount, records)}const main = () => {diff --git a/backend/migrations/clickhouse/2026_05_01_unify_events.js b/backend/migrations/clickhouse/2026_05_01_unify_events.jsnew file mode 100644index 000000000..164c012af--- /dev/null+++ b/backend/migrations/clickhouse/2026_05_01_unify_events.js@@ -0,0 +1,97 @@+const { queriesRunner, dbName } = require('./setup')++const queries = [+ `DROP TABLE IF EXISTS ${dbName}.events_tmp;`,+ `DROP TABLE IF EXISTS ${dbName}.events_backup;`,++ `CREATE TABLE IF NOT EXISTS ${dbName}.events+ (+ type LowCardinality(String),+ pid FixedString(12),+ psid Nullable(UInt64),+ profileId Nullable(String) CODEC(ZSTD(3)),+ host Nullable(String) CODEC(ZSTD(3)),+ pg Nullable(String) CODEC(ZSTD(3)),+ dv LowCardinality(Nullable(String)),+ br LowCardinality(Nullable(String)),+ brv Nullable(String) CODEC(ZSTD(3)),+ os LowCardinality(Nullable(String)),+ osv Nullable(String) CODEC(ZSTD(3)),+ lc LowCardinality(Nullable(String)),+ ref Nullable(String) CODEC(ZSTD(3)),+ so Nullable(String) CODEC(ZSTD(3)),+ me Nullable(String) CODEC(ZSTD(3)),+ ca Nullable(String) CODEC(ZSTD(3)),+ te Nullable(String) CODEC(ZSTD(3)),+ co Nullable(String) CODEC(ZSTD(3)),+ cc Nullable(FixedString(2)),+ rg LowCardinality(Nullable(String)),+ rgc LowCardinality(Nullable(String)),+ ct Nullable(String) CODEC(ZSTD(3)),+ isp LowCardinality(Nullable(String)),+ og Nullable(String) CODEC(ZSTD(3)),+ ut LowCardinality(Nullable(String)),+ ctp LowCardinality(Nullable(String)),+ \`meta.key\` Array(String) CODEC(ZSTD(3)),+ \`meta.value\` Array(String) CODEC(ZSTD(3)),+ importID Nullable(UInt8),+ event_name Nullable(String) CODEC(ZSTD(3)),+ eid Nullable(FixedString(32)),+ error_name Nullable(String) CODEC(ZSTD(3)),+ error_message Nullable(String) CODEC(ZSTD(3)),+ stackTrace Nullable(String) CODEC(ZSTD(3)),+ lineno Nullable(UInt32),+ colno Nullable(UInt32),+ error_filename Nullable(String) CODEC(ZSTD(3)),+ dns Nullable(UInt32),+ tls Nullable(UInt32),+ conn Nullable(UInt32),+ response Nullable(UInt32),+ render Nullable(UInt32),+ domLoad Nullable(UInt32),+ pageLoad Nullable(UInt32),+ ttfb Nullable(UInt32),+ created DateTime('UTC') CODEC(Delta(4), LZ4)+ )+ ENGINE = MergeTree()+ PARTITION BY toYYYYMM(created)+ ORDER BY (pid, type, created);`,++ `CREATE TABLE ${dbName}.events_tmp AS ${dbName}.events;`,++ `INSERT INTO ${dbName}.events_tmp+ (type, pid, psid, profileId, host, pg, dv, br, brv, os, osv, lc, ref, so, me, ca, te, co, cc, rg, rgc, ct, isp, og, ut, ctp, \`meta.key\`, \`meta.value\`, importID, created)+ SELECT+ 'pageview', pid, psid, profileId, host, pg, dv, br, brv, os, osv, lc, ref, so, me, ca, te, co, cc, rg, rgc, ct, isp, og, ut, ctp, \`meta.key\`, \`meta.value\`, importID, created+ FROM ${dbName}.analytics;`,++ `INSERT INTO ${dbName}.events_tmp+ (type, pid, psid, profileId, host, pg, dv, br, brv, os, osv, lc, ref, so, me, ca, te, co, cc, rg, rgc, ct, isp, og, ut, ctp, \`meta.key\`, \`meta.value\`, importID, event_name, created)+ SELECT+ 'custom_event', pid, psid, profileId, host, pg, dv, br, brv, os, osv, lc, ref, so, me, ca, te, co, cc, rg, rgc, ct, isp, og, ut, ctp, \`meta.key\`, \`meta.value\`, importID, ev, created+ FROM ${dbName}.customEV;`,++ `INSERT INTO ${dbName}.events_tmp+ (type, pid, psid, profileId, host, pg, dv, br, brv, os, osv, lc, cc, rg, rgc, ct, isp, og, ut, ctp, \`meta.key\`, \`meta.value\`, eid, error_name, error_message, stackTrace, lineno, colno, error_filename, created)+ SELECT+ 'error', pid, psid, profileId, host, pg, dv, br, brv, os, osv, lc, cc, rg, rgc, ct, isp, og, ut, ctp, \`meta.key\`, \`meta.value\`, eid, name, message, stackTrace, lineno, colno, filename, created+ FROM ${dbName}.errors;`,++ `INSERT INTO ${dbName}.events_tmp+ (type, pid, host, pg, dv, br, brv, cc, rg, rgc, ct, isp, og, ut, ctp, dns, tls, conn, response, render, domLoad, pageLoad, ttfb, created)+ SELECT+ 'performance', pid, host, pg, dv, br, brv, cc, rg, rgc, ct, isp, og, ut, ctp, dns, tls, conn, response, render, domLoad, pageLoad, ttfb, created+ FROM ${dbName}.performance;`,++ `INSERT INTO ${dbName}.events_tmp+ (type, pid, dv, br, os, cc, created)+ SELECT+ 'captcha', pid, dv, br, os, cc, created+ FROM ${dbName}.captcha;`,++ `RENAME TABLE ${dbName}.events TO ${dbName}.events_backup, ${dbName}.events_tmp TO ${dbName}.events;`,++ `DROP TABLE IF EXISTS ${dbName}.events_backup;`,+]++queriesRunner(queries)diff --git a/backend/migrations/clickhouse/2026_05_04_drop_legacy_tables.js b/backend/migrations/clickhouse/2026_05_04_drop_legacy_tables.jsnew file mode 100644index 000000000..4964e865c--- /dev/null+++ b/backend/migrations/clickhouse/2026_05_04_drop_legacy_tables.js@@ -0,0 +1,18 @@+const { queriesRunner, dbName } = require('./setup')++const queries = [+ `SELECT throwIf(+ count() = 0,+ 'The ${dbName}.events table does not exist. Run 2026_05_01_unify_events.js before dropping legacy tables.'+ )+ FROM system.tables+ WHERE database = '${dbName}' AND name = 'events';`,++ `DROP TABLE IF EXISTS ${dbName}.analytics;`,+ `DROP TABLE IF EXISTS ${dbName}.customEV;`,+ `DROP TABLE IF EXISTS ${dbName}.errors;`,+ `DROP TABLE IF EXISTS ${dbName}.performance;`,+ `DROP TABLE IF EXISTS ${dbName}.captcha;`,+]++queriesRunner(queries)diff --git a/backend/migrations/clickhouse/initialise_database.js b/backend/migrations/clickhouse/initialise_database.jsindex 5018ca040..beb10a635 100644--- a/backend/migrations/clickhouse/initialise_database.js+++ b/backend/migrations/clickhouse/initialise_database.js@@ -4,51 +4,14 @@ const { queriesRunner, dbName, databaselessQueriesRunner } = require('./setup')const CLICKHOUSE_DB_INIT_QUERIES = [`CREATE DATABASE IF NOT EXISTS ${dbName}`]const CLICKHOUSE_INIT_QUERIES = [- // The traffic data table- `CREATE TABLE IF NOT EXISTS ${dbName}.analytics- (- psid Nullable(UInt64),- profileId Nullable(String) CODEC(ZSTD(3)),- pid FixedString(12),- host Nullable(String) CODEC(ZSTD(3)),- pg Nullable(String) CODEC(ZSTD(3)),- dv LowCardinality(Nullable(String)),- br LowCardinality(Nullable(String)),- brv Nullable(String) CODEC(ZSTD(3)),- os LowCardinality(Nullable(String)),- osv Nullable(String) CODEC(ZSTD(3)),- lc LowCardinality(Nullable(String)),- ref Nullable(String) CODEC(ZSTD(3)),- so Nullable(String) CODEC(ZSTD(3)),- me Nullable(String) CODEC(ZSTD(3)),- ca Nullable(String) CODEC(ZSTD(3)),- te Nullable(String) CODEC(ZSTD(3)),- co Nullable(String) CODEC(ZSTD(3)),- cc Nullable(FixedString(2)),- rg LowCardinality(Nullable(String)),- rgc LowCardinality(Nullable(String)),- ct Nullable(String) CODEC(ZSTD(3)),- isp LowCardinality(Nullable(String)),- og Nullable(String) CODEC(ZSTD(3)),- ut LowCardinality(Nullable(String)),- ctp LowCardinality(Nullable(String)),- \`meta.key\` Array(String) CODEC(ZSTD(3)),- \`meta.value\` Array(String) CODEC(ZSTD(3)),- importID Nullable(UInt8),- created DateTime('UTC') CODEC(Delta(4), LZ4)- )- ENGINE = MergeTree()- PARTITION BY toYYYYMM(created)- ORDER BY (pid, created);`,- // Custom events table- `CREATE TABLE IF NOT EXISTS ${dbName}.customEV+ `CREATE TABLE IF NOT EXISTS ${dbName}.events(+ type LowCardinality(String),+ pid FixedString(12),psid Nullable(UInt64),profileId Nullable(String) CODEC(ZSTD(3)),- pid FixedString(12),host Nullable(String) CODEC(ZSTD(3)),- ev String CODEC(ZSTD(3)),pg Nullable(String) CODEC(ZSTD(3)),dv LowCardinality(Nullable(String)),br LowCardinality(Nullable(String)),@@ -73,29 +36,14 @@ const CLICKHOUSE_INIT_QUERIES = [\`meta.key\` Array(String) CODEC(ZSTD(3)),\`meta.value\` Array(String) CODEC(ZSTD(3)),importID Nullable(UInt8),- created DateTime('UTC') CODEC(Delta(4), LZ4)- )- ENGINE = MergeTree()- PARTITION BY toYYYYMM(created)- ORDER BY (pid, created);`,-- // The performance data table- `CREATE TABLE IF NOT EXISTS ${dbName}.performance- (- pid FixedString(12),- host Nullable(String) CODEC(ZSTD(3)),- pg Nullable(String) CODEC(ZSTD(3)),- dv LowCardinality(Nullable(String)),- br LowCardinality(Nullable(String)),- brv Nullable(String) CODEC(ZSTD(3)),- cc Nullable(FixedString(2)),- rg LowCardinality(Nullable(String)),- rgc LowCardinality(Nullable(String)),- ct Nullable(String) CODEC(ZSTD(3)),- isp LowCardinality(Nullable(String)),- og Nullable(String) CODEC(ZSTD(3)),- ut LowCardinality(Nullable(String)),- ctp LowCardinality(Nullable(String)),+ event_name Nullable(String) CODEC(ZSTD(3)),+ eid Nullable(FixedString(32)),+ error_name Nullable(String) CODEC(ZSTD(3)),+ error_message Nullable(String) CODEC(ZSTD(3)),+ stackTrace Nullable(String) CODEC(ZSTD(3)),+ lineno Nullable(UInt32),+ colno Nullable(UInt32),+ error_filename Nullable(String) CODEC(ZSTD(3)),dns Nullable(UInt32),tls Nullable(UInt32),conn Nullable(UInt32),@@ -108,44 +56,7 @@ const CLICKHOUSE_INIT_QUERIES = [)ENGINE = MergeTree()PARTITION BY toYYYYMM(created)- ORDER BY (pid, created);`,-- // Error events table- `CREATE TABLE IF NOT EXISTS ${dbName}.errors- (- psid Nullable(UInt64),- profileId Nullable(String) CODEC(ZSTD(3)),- eid FixedString(32),- pid FixedString(12),- host Nullable(String) CODEC(ZSTD(3)),- pg Nullable(String) CODEC(ZSTD(3)),- dv LowCardinality(Nullable(String)),- br LowCardinality(Nullable(String)),- brv Nullable(String) CODEC(ZSTD(3)),- os LowCardinality(Nullable(String)),- osv Nullable(String) CODEC(ZSTD(3)),- lc LowCardinality(Nullable(String)),- cc LowCardinality(Nullable(FixedString(2))),- rg LowCardinality(Nullable(String)),- rgc LowCardinality(Nullable(String)),- ct Nullable(String) CODEC(ZSTD(3)),- isp LowCardinality(Nullable(String)),- og Nullable(String) CODEC(ZSTD(3)),- ut LowCardinality(Nullable(String)),- ctp LowCardinality(Nullable(String)),- name String CODEC(ZSTD(3)),- message Nullable(String) CODEC(ZSTD(3)),- stackTrace Nullable(String) CODEC(ZSTD(3)),- \`meta.key\` Array(String) CODEC(ZSTD(3)),- \`meta.value\` Array(String) CODEC(ZSTD(3)),- lineno Nullable(UInt32),- colno Nullable(UInt32),- filename Nullable(String) CODEC(ZSTD(3)),- created DateTime('UTC') CODEC(Delta(4), LZ4)- )- ENGINE = MergeTree()- PARTITION BY toYYYYMM(created)- ORDER BY (pid, created);`,+ ORDER BY (pid, type, created);`,// Error events status table`CREATE TABLE IF NOT EXISTS ${dbName}.error_statuses (@@ -201,20 +112,6 @@ const CLICKHOUSE_INIT_QUERIES = [ORDER BY (pid, experimentId, created)TTL created + INTERVAL 1 YEAR;`,- // The CAPTCHA data table- `CREATE TABLE IF NOT EXISTS ${dbName}.captcha- (- pid FixedString(12),- dv LowCardinality(Nullable(String)),- br LowCardinality(Nullable(String)),- os Nullable(String) CODEC(ZSTD(3)),- cc Nullable(FixedString(2)),- created DateTime('UTC') CODEC(Delta(4), LZ4)- )- ENGINE = MergeTree()- PARTITION BY toYYYYMM(created)- ORDER BY (pid, created);`,-// Bot block events table for advanced bot protection reporting`CREATE TABLE IF NOT EXISTS ${dbName}.bot_blocks(diff --git a/web/app/api/api.server.ts b/web/app/api/api.server.tsindex 3eb66d8d5..052d09680 100644--- a/web/app/api/api.server.ts+++ b/web/app/api/api.server.ts@@ -1036,10 +1036,12 @@ export async function getFunnelSessionsServer(}interface PageflowItem {- type: 'pageview' | 'event' | 'error'+ type: 'pageview' | 'event' | 'error' | 'sale' | 'refund'value: stringcreated: stringmetadata?: { key: string; value: string }[]+ amount?: number+ currency?: string}export interface SessionDetailsResponse {@@ -1067,6 +1069,7 @@ export interface SessionDetailsResponse {refunds?: numbercreated: stringsdur?: number+ isLive?: booleanprofileId: string | nullisIdentified: 1 | 0isFirstSession: 1 | 0diff --git a/web/app/pages/Project/tabs/Sessions/SessionDetailView.tsx b/web/app/pages/Project/tabs/Sessions/SessionDetailView.tsxindex 0dd99425a..7a3573cc7 100644--- a/web/app/pages/Project/tabs/Sessions/SessionDetailView.tsx+++ b/web/app/pages/Project/tabs/Sessions/SessionDetailView.tsx@@ -42,10 +42,12 @@ import { Pageflow } from './Pageflow'import { SessionChart } from './SessionChart'interface PageflowItem {- type: 'pageview' | 'event' | 'error'+ type: 'pageview' | 'event' | 'error' | 'sale' | 'refund'value: stringcreated: stringmetadata?: { key: string; value: string }[]+ amount?: number+ currency?: string}interface SessionDetailViewProps {diff --git a/web/app/pages/Project/tabs/Sessions/SessionsView.tsx b/web/app/pages/Project/tabs/Sessions/SessionsView.tsxindex c9631ed51..e43ad3018 100644--- a/web/app/pages/Project/tabs/Sessions/SessionsView.tsx+++ b/web/app/pages/Project/tabs/Sessions/SessionsView.tsx@@ -85,10 +85,12 @@ class SessionsErrorBoundary extends Component<const SESSIONS_TAKE = 30interface PageflowItem {- type: 'pageview' | 'event' | 'error'+ type: 'pageview' | 'event' | 'error' | 'sale' | 'refund'value: stringcreated: stringmetadata?: { key: string; value: string }[]+ amount?: number+ currency?: string}interface ActiveSession {@@ -201,7 +203,9 @@ const SessionsViewInner = ({dv: apiDetails.dv,profileId: apiDetails.profileId,sdur: apiDetails.sdur,- isLive: matchingSession?.isLive === 1,+ isLive: matchingSession+ ? matchingSession.isLive === 1+ : apiDetails.isLive,}return {details,