Skip to content

📈 Open source, privacy-first web analytics — a cookie-less Google Analytics alternative with built-in error tracking and performance monitoring. Self-host it or use our cloud. Get started for free!

Unify analytics tables into a single 'events' table (#526)

Andrii Romasiun286428ec730c
diff --git a/admin/src/db/clickhouse.ts b/admin/src/db/clickhouse.ts
index 2e4f960b3..3b76d08eb 100644
--- a/admin/src/db/clickhouse.ts
+++ b/admin/src/db/clickhouse.ts
@@ -34,6 +34,18 @@ export interface ClickHouseStats {
tables: TableStats[];
}
+const EVENT_TABLE = "events";
+const PROJECT_ACTIVITY_EVENT_TYPES = [
+ "pageview",
+ "custom_event",
+ "error",
+ "captcha",
+ "performance",
+];
+const PROJECT_ACTIVITY_TYPE_LIST = PROJECT_ACTIVITY_EVENT_TYPES.map(
+ (type) => `'${type}'`
+).join(", ");
+
function formatBytes(bytes: number): string {
if (bytes === 0) return "0 B";
const k = 1024;
@@ -47,20 +59,16 @@ export async function getClickHouseStats(): Promise<ClickHouseStats> {
const database = process.env.CLICKHOUSE_DATABASE || "analytics";
const tables = [
- "analytics",
- "customEV",
- "performance",
- "errors",
- "captcha",
+ EVENT_TABLE,
"sessions",
"revenue",
"feature_flag_evaluations",
"experiment_exposures",
"error_statuses",
+ "bot_blocks",
];
const tableStats: TableStats[] = [];
- let totalEvents = 0;
for (const table of tables) {
try {
@@ -93,8 +101,6 @@ export async function getClickHouseStats(): Promise<ClickHouseStats> {
bytes,
bytesFormatted: formatBytes(bytes),
});
-
- totalEvents += rows;
} catch {
// Table might not exist, skip it
tableStats.push({
@@ -106,6 +112,9 @@ export async function getClickHouseStats(): Promise<ClickHouseStats> {
}
}
+ const totalEvents =
+ tableStats.find((table) => table.table === EVENT_TABLE)?.rows || 0;
+
return {
totalEvents,
tables: tableStats,
@@ -118,7 +127,12 @@ export async function getProjectEventCount(pid: string): Promise<number> {
try {
const result = await client.query({
- query: `SELECT count() as count FROM ${database}.analytics WHERE pid = {pid:String}`,
+ query: `
+ SELECT count() as count
+ FROM ${database}.${EVENT_TABLE}
+ WHERE pid = {pid:FixedString(12)}
+ AND type IN (${PROJECT_ACTIVITY_TYPE_LIST})
+ `,
query_params: { pid },
format: "JSONEachRow",
});
@@ -158,8 +172,9 @@ export async function getTopProjectsByEvents(
SELECT
pid,
count() as eventCount
- FROM ${database}.analytics
+ FROM ${database}.${EVENT_TABLE}
WHERE created >= now() - INTERVAL ${days} DAY
+ AND type IN (${PROJECT_ACTIVITY_TYPE_LIST})
GROUP BY pid
ORDER BY eventCount DESC
LIMIT ${limit}
@@ -183,8 +198,9 @@ export async function getProjectsWithRecentEvents(
const result = await client.query({
query: `
SELECT DISTINCT pid
- FROM ${database}.analytics
+ FROM ${database}.${EVENT_TABLE}
WHERE created >= now() - INTERVAL ${days} DAY
+ AND type IN (${PROJECT_ACTIVITY_TYPE_LIST})
`,
format: "JSONEachRow",
});
diff --git a/backend/apps/cloud/src/ai/ai.service.ts b/backend/apps/cloud/src/ai/ai.service.ts
index 8671e63fd..5414f6c3c 100644
--- a/backend/apps/cloud/src/ai/ai.service.ts
+++ b/backend/apps/cloud/src/ai/ai.service.ts
@@ -1218,8 +1218,9 @@ Filter modifiers:
SELECT
count(*) as pageviews,
count(DISTINCT psid) as sessions
- FROM analytics
+ FROM events
WHERE pid = {pid:FixedString(12)}
+ AND type = 'pageview'
AND created BETWEEN {groupFrom:String} AND {groupTo:String}
${filterConditions.where}
`
@@ -1242,8 +1243,9 @@ Filter modifiers:
${this.getTimeBucketSelect(timeBucket, timezone)} as date,
count(*) as pageviews,
count(DISTINCT psid) as sessions
- FROM analytics
+ FROM events
WHERE pid = {pid:FixedString(12)}
+ AND type = 'pageview'
AND created BETWEEN {groupFrom:String} AND {groupTo:String}
${filterConditions.where}
GROUP BY date
@@ -1266,8 +1268,9 @@ Filter modifiers:
// Get top pages
const pagesQuery = `
SELECT pg as name, count(*) as count
- FROM analytics
+ FROM events
WHERE pid = {pid:FixedString(12)}
+ AND type = 'pageview'
AND created BETWEEN {groupFrom:String} AND {groupTo:String}
${filterConditions.where}
GROUP BY pg
@@ -1285,8 +1288,9 @@ Filter modifiers:
// Get top countries
const countriesQuery = `
SELECT cc as name, uniqExact(psid) as count
- FROM analytics
+ FROM events
WHERE pid = {pid:FixedString(12)}
+ AND type = 'pageview'
AND created BETWEEN {groupFrom:String} AND {groupTo:String}
AND cc IS NOT NULL AND cc != ''
${filterConditions.where}
@@ -1305,8 +1309,9 @@ Filter modifiers:
// Get top referrers
const referrersQuery = `
SELECT ref as name, uniqExact(psid) as count
- FROM analytics
+ FROM events
WHERE pid = {pid:FixedString(12)}
+ AND type = 'pageview'
AND created BETWEEN {groupFrom:String} AND {groupTo:String}
AND ref IS NOT NULL AND ref != ''
${filterConditions.where}
@@ -1325,8 +1330,9 @@ Filter modifiers:
// Get browsers
const browsersQuery = `
SELECT br as name, uniqExact(psid) as count
- FROM analytics
+ FROM events
WHERE pid = {pid:FixedString(12)}
+ AND type = 'pageview'
AND created BETWEEN {groupFrom:String} AND {groupTo:String}
AND br IS NOT NULL AND br != ''
${filterConditions.where}
@@ -1345,8 +1351,9 @@ Filter modifiers:
// Get devices
const devicesQuery = `
SELECT dv as name, uniqExact(psid) as count
- FROM analytics
+ FROM events
WHERE pid = {pid:FixedString(12)}
+ AND type = 'pageview'
AND created BETWEEN {groupFrom:String} AND {groupTo:String}
AND dv IS NOT NULL AND dv != ''
${filterConditions.where}
@@ -1416,8 +1423,9 @@ Filter modifiers:
round(${measureFn}(render) / 1000, 2) as render,
round(${measureFn}(domLoad) / 1000, 2) as domLoad,
round(${measureFn}(ttfb) / 1000, 2) as ttfb
- FROM performance
+ FROM events
WHERE pid = {pid:FixedString(12)}
+ AND type = 'performance'
AND created BETWEEN {groupFrom:String} AND {groupTo:String}
${filterConditions.where}
`
@@ -1440,8 +1448,9 @@ Filter modifiers:
${this.getTimeBucketSelect(timeBucket, timezone)} as date,
round(${measureFn}(pageLoad) / 1000, 2) as pageLoad,
round(${measureFn}(ttfb) / 1000, 2) as ttfb
- FROM performance
+ FROM events
WHERE pid = {pid:FixedString(12)}
+ AND type = 'performance'
AND created BETWEEN {groupFrom:String} AND {groupTo:String}
${filterConditions.where}
GROUP BY date
@@ -1497,8 +1506,9 @@ Filter modifiers:
SELECT
count(*) as totalErrors,
uniqExact(eid) as uniqueErrors
- FROM errors
+ FROM events
WHERE pid = {pid:FixedString(12)}
+ AND type = 'error'
AND created BETWEEN {groupFrom:String} AND {groupTo:String}
${filterConditions.where}
`
@@ -1512,15 +1522,16 @@ Filter modifiers:
const topErrorsQuery = `
SELECT
- name,
- message,
+ error_name AS name,
+ error_message AS message,
count(*) as count,
max(created) as lastSeen
- FROM errors
+ FROM events
WHERE pid = {pid:FixedString(12)}
+ AND type = 'error'
AND created BETWEEN {groupFrom:String} AND {groupTo:String}
${filterConditions.where}
- GROUP BY name, message
+ GROUP BY error_name, error_message
ORDER BY count DESC
LIMIT 10
`
@@ -1571,22 +1582,24 @@ Filter modifiers:
}
// Query specific goal stats
- const table = goal.type === 'custom_event' ? 'customEV' : 'analytics'
+ const goalType =
+ goal.type === 'custom_event' ? 'custom_event' : 'pageview'
const matchCondition =
goal.matchType === 'exact'
? goal.type === 'custom_event'
- ? `ev = {goalValue:String}`
+ ? `event_name = {goalValue:String}`
: `pg = {goalValue:String}`
: goal.type === 'custom_event'
- ? `ev ILIKE concat('%', {goalValue:String}, '%')`
+ ? `event_name ILIKE concat('%', {goalValue:String}, '%')`
: `pg ILIKE concat('%', {goalValue:String}, '%')`
const query = `
SELECT
count(*) as conversions,
uniqExact(psid) as uniqueSessions
- FROM ${table}
+ FROM events
WHERE pid = {pid:FixedString(12)}
+ AND type = {goalType:String}
AND ${matchCondition}
AND created BETWEEN {groupFrom:String} AND {groupTo:String}
`
@@ -1596,6 +1609,7 @@ Filter modifiers:
query,
query_params: {
pid,
+ goalType,
goalValue: goal.value || '',
groupFrom: groupFromUTC,
groupTo: groupToUTC,
@@ -1617,20 +1631,22 @@ Filter modifiers:
const results = await Promise.all(
goals.map(async (goal) => {
- const table = goal.type === 'custom_event' ? 'customEV' : 'analytics'
+ const goalType =
+ goal.type === 'custom_event' ? 'custom_event' : 'pageview'
const matchCondition =
goal.matchType === 'exact'
? goal.type === 'custom_event'
- ? `ev = {goalValue:String}`
+ ? `event_name = {goalValue:String}`
: `pg = {goalValue:String}`
: goal.type === 'custom_event'
- ? `ev ILIKE concat('%', {goalValue:String}, '%')`
+ ? `event_name ILIKE concat('%', {goalValue:String}, '%')`
: `pg ILIKE concat('%', {goalValue:String}, '%')`
const query = `
SELECT count(*) as conversions
- FROM ${table}
+ FROM events
WHERE pid = {pid:FixedString(12)}
+ AND type = {goalType:String}
AND ${matchCondition}
AND created BETWEEN {groupFrom:String} AND {groupTo:String}
`
@@ -1640,6 +1656,7 @@ Filter modifiers:
query,
query_params: {
pid,
+ goalType,
goalValue: goal.value || '',
groupFrom: groupFromUTC,
groupTo: groupToUTC,
@@ -1747,8 +1764,9 @@ Filter modifiers:
const overallQuery = `
SELECT
count(*) as total
- FROM captcha
+ FROM events
WHERE pid = {pid:FixedString(12)}
+ AND type = 'captcha'
AND created BETWEEN {groupFrom:String} AND {groupTo:String}
${filterConditions.where}
`
@@ -1757,8 +1775,9 @@ Filter modifiers:
SELECT
${this.getTimeBucketSelect(timeBucket, timezone)} as date,
count(*) as challenges
- FROM captcha
+ FROM events
WHERE pid = {pid:FixedString(12)}
+ AND type = 'captcha'
AND created BETWEEN {groupFrom:String} AND {groupTo:String}
${filterConditions.where}
GROUP BY date
@@ -1767,8 +1786,9 @@ Filter modifiers:
const breakdownQuery = (column: string) => `
SELECT ${column} as name, count(*) as count
- FROM captcha
+ FROM events
WHERE pid = {pid:FixedString(12)}
+ AND type = 'captcha'
AND created BETWEEN {groupFrom:String} AND {groupTo:String}
AND ${column} IS NOT NULL AND ${column} != ''
${filterConditions.where}
@@ -1844,21 +1864,23 @@ Filter modifiers:
const overallQuery = `
SELECT
count(*) as totalEvents,
- uniqExact(ev) as uniqueEvents,
+ uniqExact(event_name) as uniqueEvents,
uniqExact(psid) as sessions
- FROM customEV
+ FROM events
WHERE pid = {pid:FixedString(12)}
+ AND type = 'custom_event'
AND created BETWEEN {groupFrom:String} AND {groupTo:String}
${filterConditions.where}
`
const topEventsQuery = `
- SELECT ev as name, count(*) as count, uniqExact(psid) as sessions
- FROM customEV
+ SELECT event_name AS name, count(*) as count, uniqExact(psid) as sessions
+ FROM events
WHERE pid = {pid:FixedString(12)}
+ AND type = 'custom_event'
AND created BETWEEN {groupFrom:String} AND {groupTo:String}
${filterConditions.where}
- GROUP BY ev
+ GROUP BY event_name
ORDER BY count DESC
LIMIT 25
`
@@ -1868,8 +1890,9 @@ Filter modifiers:
${this.getTimeBucketSelect(timeBucket, timezone)} as date,
count(*) as events,
uniqExact(psid) as sessions
- FROM customEV
+ FROM events
WHERE pid = {pid:FixedString(12)}
+ AND type = 'custom_event'
AND created BETWEEN {groupFrom:String} AND {groupTo:String}
${filterConditions.where}
GROUP BY date
@@ -2137,10 +2160,10 @@ Filter modifiers:
let conversions: { variantKey: string; conversions: number }[] = []
if (experiment.goal) {
- const table =
- experiment.goal.type === 'custom_event' ? 'customEV' : 'analytics'
+ const eventType =
+ experiment.goal.type === 'custom_event' ? 'custom_event' : 'pageview'
const matchColumn =
- experiment.goal.type === 'custom_event' ? 'ev' : 'pg'
+ experiment.goal.type === 'custom_event' ? 'event_name' : 'pg'
const matchCondition =
experiment.goal.matchType === 'exact'
? `c.${matchColumn} = {goalValue:String}`
@@ -2149,7 +2172,7 @@ Filter modifiers:
const conversionsQuery = `
SELECT e.variantKey, uniqExact(e.profileId) as conversions
FROM experiment_exposures e
- INNER JOIN ${table} c ON e.pid = c.pid AND e.profileId = assumeNotNull(c.profileId)
+ INNER JOIN events c ON e.pid = c.pid AND e.profileId = assumeNotNull(c.profileId) AND c.type = '${eventType}'
WHERE e.pid = {pid:FixedString(12)}
AND e.experimentId = {experimentId:String}
AND e.created BETWEEN {groupFrom:String} AND {groupTo:String}
@@ -2291,8 +2314,9 @@ Filter modifiers:
min(created) as startedAt,
max(created) as endedAt,
count(*) as pageviews
- FROM analytics
+ FROM events
WHERE pid = {pid:FixedString(12)}
+ AND type = 'pageview'
AND created BETWEEN {groupFrom:String} AND {groupTo:String}
AND psid IS NOT NULL
${extraWhere.join(' ')}
@@ -2356,8 +2380,9 @@ Filter modifiers:
uniqExactIf(profileId, profileId LIKE 'usr_%') as identifiedProfiles,
uniqExact(psid) as sessions,
count(*) as pageviews
- FROM analytics
+ FROM events
WHERE pid = {pid:FixedString(12)}
+ AND type = 'pageview'
AND created BETWEEN {groupFrom:String} AND {groupTo:String}
AND profileId IS NOT NULL
`
@@ -2368,8 +2393,9 @@ Filter modifiers:
uniqExact(psid) as sessions,
count(*) as pageviews,
max(created) as lastSeen
- FROM analytics
+ FROM events
WHERE pid = {pid:FixedString(12)}
+ AND type = 'pageview'
AND created BETWEEN {groupFrom:String} AND {groupTo:String}
AND profileId IS NOT NULL
GROUP BY profileId
@@ -2379,8 +2405,9 @@ Filter modifiers:
const topPagesQuery = `
SELECT pg as name, uniqExact(profileId) as profiles
- FROM analytics
+ FROM events
WHERE pid = {pid:FixedString(12)}
+ AND type = 'pageview'
AND created BETWEEN {groupFrom:String} AND {groupTo:String}
AND profileId IS NOT NULL
AND pg IS NOT NULL AND pg != ''
diff --git a/backend/apps/cloud/src/analytics/analytics.controller.ts b/backend/apps/cloud/src/analytics/analytics.controller.ts
index 8fa6767c0..4f3063996 100644
--- a/backend/apps/cloud/src/analytics/analytics.controller.ts
+++ b/backend/apps/cloud/src/analytics/analytics.controller.ts
@@ -81,12 +81,7 @@ import {
GetErrorOverviewOptions,
} from './dto/get-error-overview.dto'
import { PatchStatusDto } from './dto/patch-status.dto'
-import {
- customEventTransformer,
- errorEventTransformer,
- performanceTransformer,
- trafficTransformer,
-} from './utils/transformers'
+import { eventTransformer } from './utils/transformers'
import { enrichTrafficSource } from './utils/clickIdSources'
import {
MAX_METRICS_IN_VIEW,
@@ -263,10 +258,19 @@ export class AnalyticsController {
let diff
+ const [filtersQuery, filtersParams, appliedFilters, customEVFilterApplied] =
+ this.analyticsService.getFiltersQuery(
+ filters,
+ isCaptcha ? DataType.CAPTCHA : DataType.ANALYTICS,
+ )
+
if (period === 'all') {
+ const eventTypes = isCaptcha
+ ? 'captcha'
+ : (['pageview', 'custom_event', 'error'] as const)
const res = await this.analyticsService.calculateTimeBucketForAllTime(
pid,
- 'analytics',
+ eventTypes,
)
diff = res.diff
@@ -277,12 +281,6 @@ export class AnalyticsController {
allowedTumebucketForPeriodAll = res.timeBucket
}
- const [filtersQuery, filtersParams, appliedFilters, customEVFilterApplied] =
- this.analyticsService.getFiltersQuery(
- filters,
- isCaptcha ? DataType.CAPTCHA : DataType.ANALYTICS,
- )
-
const safeTimezone = this.analyticsService.getSafeTimezone(timezone)
const { groupFrom, groupTo, groupFromUTC, groupToUTC } =
this.analyticsService.getGroupFromTo(
@@ -294,13 +292,11 @@ export class AnalyticsController {
diff,
)
- let subQuery = `FROM ${
- isCaptcha ? 'captcha' : 'analytics'
- } WHERE pid = {pid:FixedString(12)} ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String}`
-
- if (customEVFilterApplied && !isCaptcha) {
- subQuery = `FROM customEV WHERE pid = {pid:FixedString(12)} ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String}`
- }
+ const subQuery = this.analyticsService.buildAnalyticsEventsSubQuery(
+ filtersQuery,
+ customEVFilterApplied,
+ isCaptcha,
+ )
const paramsData = {
params: {
@@ -422,12 +418,15 @@ export class AnalyticsController {
let diff
if (period === 'all') {
- const [analyticsRes, customEVRes] = await Promise.all([
- this.analyticsService.calculateTimeBucketForAllTime(pid, 'analytics'),
- this.analyticsService.calculateTimeBucketForAllTime(pid, 'customEV'),
+ const [pageviewRes, customEventRes] = await Promise.all([
+ this.analyticsService.calculateTimeBucketForAllTime(pid, 'pageview'),
+ this.analyticsService.calculateTimeBucketForAllTime(
+ pid,
+ 'custom_event',
+ ),
])
- diff = Math.max(analyticsRes.diff, customEVRes.diff)
+ diff = Math.max(pageviewRes.diff, customEventRes.diff)
}
const safeTimezone = this.analyticsService.getSafeTimezone(timezone)
@@ -537,12 +536,15 @@ export class AnalyticsController {
let diff
if (period === 'all') {
- const [analyticsRes, customEVRes] = await Promise.all([
- this.analyticsService.calculateTimeBucketForAllTime(pid, 'analytics'),
- this.analyticsService.calculateTimeBucketForAllTime(pid, 'customEV'),
+ const [pageviewRes, customEventRes] = await Promise.all([
+ this.analyticsService.calculateTimeBucketForAllTime(pid, 'pageview'),
+ this.analyticsService.calculateTimeBucketForAllTime(
+ pid,
+ 'custom_event',
+ ),
])
- diff = Math.max(analyticsRes.diff, customEVRes.diff)
+ diff = Math.max(pageviewRes.diff, customEventRes.diff)
}
const safeTimezone = this.analyticsService.getSafeTimezone(timezone)
@@ -707,13 +709,14 @@ export class AnalyticsController {
this.analyticsService.getFiltersQuery(filters, DataType.ANALYTICS)
const safeTimezone = this.analyticsService.getSafeTimezone(timezone)
- const { groupFrom, groupTo } = this.analyticsService.getGroupFromTo(
- from,
- to,
- timeBucket,
- period,
- safeTimezone,
- )
+ const { groupFrom, groupTo, groupFromUTC, groupToUTC } =
+ this.analyticsService.getGroupFromTo(
+ from,
+ to,
+ timeBucket,
+ period,
+ safeTimezone,
+ )
await this.analyticsService.checkProjectAccess(
pid,
uid,
@@ -724,7 +727,14 @@ export class AnalyticsController {
this.logger.log(`pid: ${pid}, period: ${period}`, 'GET /analytics/chart')
- const paramsData = { params: { pid, groupFrom, groupTo, ...filtersParams } }
+ const paramsData = {
+ params: {
+ pid,
+ groupFrom: groupFromUTC,
+ groupTo: groupToUTC,
+ ...filtersParams,
+ },
+ }
const result = await this.analyticsService.groupChartByTimeBucket(
timeBucket,
@@ -795,18 +805,26 @@ export class AnalyticsController {
this.analyticsService.getFiltersQuery(filters, DataType.PERFORMANCE, true)
const safeTimezone = this.analyticsService.getSafeTimezone(timezone)
- const { groupFrom, groupTo } = this.analyticsService.getGroupFromTo(
- from,
- to,
- newTimeBucket,
- period,
- safeTimezone,
- diff,
- )
+ const { groupFrom, groupTo, groupFromUTC, groupToUTC } =
+ this.analyticsService.getGroupFromTo(
+ from,
+ to,
+ newTimeBucket,
+ period,
+ safeTimezone,
+ diff,
+ )
- const subQuery = `FROM performance WHERE pid = {pid:FixedString(12)} ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String}`
+ const subQuery = `FROM events WHERE pid = {pid:FixedString(12)} AND type = 'performance' ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String}`
- const paramsData = { params: { pid, groupFrom, groupTo, ...filtersParams } }
+ const paramsData = {
+ params: {
+ pid,
+ groupFrom: groupFromUTC,
+ groupTo: groupToUTC,
+ ...filtersParams,
+ },
+ }
const result = await this.analyticsService.groupPerfByTimeBucket(
newTimeBucket,
@@ -850,13 +868,14 @@ export class AnalyticsController {
this.analyticsService.getFiltersQuery(filters, DataType.PERFORMANCE, true)
const safeTimezone = this.analyticsService.getSafeTimezone(timezone)
- const { groupFrom, groupTo } = this.analyticsService.getGroupFromTo(
- from,
- to,
- timeBucket,
- period,
- safeTimezone,
- )
+ const { groupFrom, groupTo, groupFromUTC, groupToUTC } =
+ this.analyticsService.getGroupFromTo(
+ from,
+ to,
+ timeBucket,
+ period,
+ safeTimezone,
+ )
await this.analyticsService.checkProjectAccess(
pid,
uid,
@@ -870,12 +889,19 @@ export class AnalyticsController {
'GET /analytics/performance/chart',
)
- const paramsData = { params: { pid, groupFrom, groupTo, ...filtersParams } }
+ const paramsData = {
+ params: {
+ pid,
+ groupFrom: groupFromUTC,
+ groupTo: groupToUTC,
+ ...filtersParams,
+ },
+ }
const chart = await this.analyticsService.getPerfChartData(
timeBucket,
- from,
- to,
+ groupFrom,
+ groupTo,
filtersQuery,
paramsData,
safeTimezone,
@@ -921,7 +947,7 @@ export class AnalyticsController {
if (period === 'all') {
const res = await this.analyticsService.calculateTimeBucketForAllTime(
pid,
- 'analytics',
+ 'pageview',
)
diff = res.diff
@@ -1332,32 +1358,12 @@ export class AnalyticsController {
any(os) AS os,
any(cc) AS cc,
toString(psid) AS psid
- FROM
- (
- SELECT
- psid,
- dv,
- br,
- os,
- cc
- FROM analytics
- WHERE
- pid = {pid:FixedString(12)}
- AND created >= {since:DateTime}
- AND psid IS NOT NULL
- UNION ALL
- SELECT
- psid,
- dv,
- br,
- os,
- cc
- FROM customEV
- WHERE
- pid = {pid:FixedString(12)}
- AND created >= {since:DateTime}
- AND psid IS NOT NULL
- )
+ FROM events
+ WHERE
+ pid = {pid:FixedString(12)}
+ AND type IN ('pageview', 'custom_event')
+ AND created >= {since:DateTime}
+ AND psid IS NOT NULL
GROUP BY psid
`
@@ -1464,7 +1470,8 @@ export class AnalyticsController {
const { name, message, lineno, colno, filename, stackTrace, meta } =
errorDTO
- const transformed = errorEventTransformer({
+ const transformed = eventTransformer({
+ type: 'error',
psid,
profileId,
eid: this.analyticsService.getErrorID(errorDTO),
@@ -1496,7 +1503,7 @@ export class AnalyticsController {
try {
await clickhouse.insert({
- table: 'errors',
+ table: 'events',
format: 'JSONEachRow',
values: [transformed],
clickhouse_settings: { async_insert: 1 },
@@ -1618,7 +1625,8 @@ export class AnalyticsController {
enrichTrafficSource(eventsDTO)
- const transformed = customEventTransformer({
+ const transformed = eventTransformer({
+ type: 'custom_event',
psid,
profileId,
pid: eventsDTO.pid,
@@ -1650,7 +1658,7 @@ export class AnalyticsController {
try {
await clickhouse.insert({
- table: 'customEV',
+ table: 'events',
format: 'JSONEachRow',
values: [transformed],
clickhouse_settings: { async_insert: 1 },
@@ -1789,7 +1797,8 @@ export class AnalyticsController {
enrichTrafficSource(logDTO)
- const transformed = trafficTransformer({
+ const transformed = eventTransformer({
+ type: 'pageview',
psid,
profileId,
pid: logDTO.pid,
@@ -1832,7 +1841,8 @@ export class AnalyticsController {
ttfb,
} = logDTO.perf
- perfTransformed = performanceTransformer({
+ perfTransformed = eventTransformer({
+ type: 'performance',
pid: logDTO.pid,
host: this.analyticsService.getHostFromOrigin(headers.origin),
pg: logDTO.pg,
@@ -1860,7 +1870,7 @@ export class AnalyticsController {
try {
await clickhouse.insert({
- table: 'analytics',
+ table: 'events',
format: 'JSONEachRow',
values: [transformed],
clickhouse_settings: { async_insert: 1 },
@@ -1868,7 +1878,7 @@ export class AnalyticsController {
if (!_isEmpty(perfTransformed)) {
await clickhouse.insert({
- table: 'performance',
+ table: 'events',
format: 'JSONEachRow',
values: [perfTransformed],
clickhouse_settings: { async_insert: 1 },
@@ -1955,7 +1965,8 @@ export class AnalyticsController {
const { deviceType, browserName, browserVersion, osName, osVersion } =
await this.analyticsService.getRequestInformation(headers)
- const transformed = trafficTransformer({
+ const transformed = eventTransformer({
+ type: 'pageview',
psid,
profileId,
pid: logDTO.pid,
@@ -1986,7 +1997,7 @@ export class AnalyticsController {
try {
await clickhouse.insert({
- table: 'analytics',
+ table: 'events',
format: 'JSONEachRow',
values: [transformed],
clickhouse_settings: { async_insert: 1 },
@@ -2039,7 +2050,7 @@ export class AnalyticsController {
if (period === 'all') {
const res = await this.analyticsService.calculateTimeBucketForAllTime(
pid,
- customEVFilterApplied ? 'customEV' : 'analytics',
+ ['pageview', 'custom_event', 'error'],
)
timeBucket = res.timeBucket[0]
@@ -2124,7 +2135,7 @@ export class AnalyticsController {
if (period === 'all') {
const res = await this.analyticsService.calculateTimeBucketForAllTime(
pid,
- 'errors',
+ 'error',
)
timeBucket = res.timeBucket[0]
@@ -2203,7 +2214,7 @@ export class AnalyticsController {
if (period === 'all') {
const res = await this.analyticsService.calculateTimeBucketForAllTime(
pid,
- 'errors',
+ 'error',
)
newTimeBucket = res.timeBucket[0]
@@ -2281,7 +2292,7 @@ export class AnalyticsController {
if (period === 'all') {
const res = await this.analyticsService.calculateTimeBucketForAllTime(
pid,
- 'errors',
+ 'error',
)
newTimeBucket = res.timeBucket[0]
@@ -2293,6 +2304,8 @@ export class AnalyticsController {
DataType.ERRORS,
true,
)
+ const [sessionFiltersQuery, sessionFiltersParams] =
+ this.analyticsService.getFiltersQuery(filters, DataType.ANALYTICS, true)
const safeTimezone = this.analyticsService.getSafeTimezone(timezone)
const { groupFromUTC, groupToUTC } = this.analyticsService.getGroupFromTo(
@@ -2322,6 +2335,8 @@ export class AnalyticsController {
groupToUTC,
newTimeBucket,
parsedOptions.showResolved || false,
+ sessionFiltersQuery,
+ sessionFiltersParams,
)
}
@@ -2362,7 +2377,7 @@ export class AnalyticsController {
if (period === 'all') {
const res = await this.analyticsService.calculateTimeBucketForAllTime(
pid,
- 'errors',
+ 'error',
)
newTimeBucket = res.timeBucket[0]
@@ -2467,7 +2482,7 @@ export class AnalyticsController {
if (period === 'all') {
const res = await this.analyticsService.calculateTimeBucketForAllTime(
pid,
- customEVFilterApplied ? 'customEV' : 'analytics',
+ ['pageview', 'custom_event', 'error'],
)
timeBucket = res.timeBucket[0]
@@ -2546,7 +2561,7 @@ export class AnalyticsController {
if (period === 'all') {
const res = await this.analyticsService.calculateTimeBucketForAllTime(
pid,
- 'analytics',
+ ['pageview', 'custom_event', 'error'],
)
timeBucket = res.timeBucket[0]
diff = res.diff
@@ -2634,7 +2649,7 @@ export class AnalyticsController {
if (period === 'all') {
const res = await this.analyticsService.calculateTimeBucketForAllTime(
pid,
- customEVFilterApplied ? 'customEV' : 'analytics',
+ ['pageview', 'custom_event', 'error'],
)
timeBucket = res.timeBucket[0]
@@ -2714,7 +2729,7 @@ export class AnalyticsController {
if (period === VALID_PERIODS[VALID_PERIODS.length - 1]) {
const res = await this.analyticsService.calculateTimeBucketForAllTime(
pid,
- 'customEV',
+ 'custom_event',
)
newTimeBucket = _includes(res.timeBucket, timeBucket)
@@ -2728,16 +2743,24 @@ export class AnalyticsController {
this.analyticsService.getFiltersQuery(filters, DataType.ANALYTICS)
const safeTimezone = this.analyticsService.getSafeTimezone(timezone)
- const { groupFrom, groupTo } = this.analyticsService.getGroupFromTo(
- from,
- to,
- newTimeBucket,
- period,
- safeTimezone,
- diff,
- )
+ const { groupFrom, groupTo, groupFromUTC, groupToUTC } =
+ this.analyticsService.getGroupFromTo(
+ from,
+ to,
+ newTimeBucket,
+ period,
+ safeTimezone,
+ diff,
+ )
- const paramsData = { params: { pid, groupFrom, groupTo, ...filtersParams } }
+ const paramsData = {
+ params: {
+ pid,
+ groupFrom: groupFromUTC,
+ groupTo: groupToUTC,
+ ...filtersParams,
+ },
+ }
// customEvents comes as a JSON.stringified array from the frontend
let customEventsList: string[] = []
diff --git a/backend/apps/cloud/src/analytics/analytics.service.ts b/backend/apps/cloud/src/analytics/analytics.service.ts
index 80c4a8b6a..ba4e2dbcb 100644
--- a/backend/apps/cloud/src/analytics/analytics.service.ts
+++ b/backend/apps/cloud/src/analytics/analytics.service.ts
@@ -278,6 +278,14 @@ const EXCLUDE_NULL_FOR = [
'ctp',
]
+const ERROR_COLUMN_MAP: Record<string, string> = {
+ name: 'error_name',
+ message: 'error_message',
+ filename: 'error_filename',
+}
+
+const mapErrorColumn = (type: string): string => ERROR_COLUMN_MAP[type] || type
+
const generateParamsQuery = (
col: string,
subQuery: string,
@@ -285,7 +293,8 @@ const generateParamsQuery = (
type: 'traffic' | 'performance' | 'captcha' | 'errors',
measure?: PerfMeasure,
): string => {
- let columns = [`${col} as name`]
+ const sqlCol = type === 'errors' ? mapErrorColumn(col) : col
+ let columns = [`${sqlCol} as name`]
// For regions and cities we'll return an array of objects, that will also include the country code and region code
// We need the conutry code to display the flag next to the region/city name
@@ -309,15 +318,15 @@ const generateParamsQuery = (
const fn = MEASURES_MAP[processedMeasure]
- if (col === 'pg' || col === 'host') {
+ if (sqlCol === 'pg' || sqlCol === 'host') {
return `SELECT ${columnsQuery}, round(divide(${fn}(pageLoad), 1000), 2) as count ${subQuery} GROUP BY ${columnsQuery}`
}
- return `SELECT ${columnsQuery}, round(divide(${fn}(pageLoad), 1000), 2) as count ${subQuery} ${EXCLUDE_NULL_FOR.includes(col) ? `AND ${col} IS NOT NULL` : ''} GROUP BY ${columnsQuery}`
+ return `SELECT ${columnsQuery}, round(divide(${fn}(pageLoad), 1000), 2) as count ${subQuery} ${EXCLUDE_NULL_FOR.includes(sqlCol) ? `AND ${sqlCol} IS NOT NULL` : ''} GROUP BY ${columnsQuery}`
}
if (type === 'errors') {
- return `SELECT ${columnsQuery}, count(*) as count ${subQuery} ${EXCLUDE_NULL_FOR.includes(col) ? `AND ${col} IS NOT NULL` : ''} GROUP BY ${columnsQuery}`
+ return `SELECT ${columnsQuery}, count(*) as count ${subQuery} ${EXCLUDE_NULL_FOR.includes(sqlCol) ? `AND ${sqlCol} IS NOT NULL` : ''} GROUP BY ${columnsQuery}`
}
if (type === 'captcha') {
@@ -338,6 +347,13 @@ export enum DataType {
ERRORS = 'errors',
}
+type AnalyticsEventType = 'pageview' | 'custom_event'
+type EventsAllTimeType =
+ | AnalyticsEventType
+ | 'performance'
+ | 'error'
+ | 'captcha'
+
const isValidOrigin = (origins: string[], origin: string) => {
const escapeRegex = (str: string) =>
str.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')
@@ -704,6 +720,22 @@ export class AnalyticsService {
return CAPTCHA_COLUMNS
}
+ getAnalyticsEventType(customEVFilterApplied: boolean): AnalyticsEventType {
+ return customEVFilterApplied ? 'custom_event' : 'pageview'
+ }
+
+ buildAnalyticsEventsSubQuery(
+ filtersQuery: string,
+ customEVFilterApplied: boolean,
+ isCaptcha = false,
+ ): string {
+ const eventType = isCaptcha
+ ? 'captcha'
+ : this.getAnalyticsEventType(customEVFilterApplied)
+
+ return `FROM events WHERE pid = {pid:FixedString(12)} AND type = '${eventType}' ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String}`
+ }
+
getGroupFromTo(
from: string,
to: string,
@@ -857,9 +889,10 @@ export class AnalyticsService {
pg,
created,
lagInFrame(pg) OVER (PARTITION BY psid ORDER BY created) AS prev_page
- FROM analytics
+ FROM events
WHERE
pid = {pid:FixedString(12)}
+ AND type = 'pageview'
AND created BETWEEN {groupFrom:String} AND {groupTo:String}
AND pg IS NOT NULL
${filtersQuery}
@@ -970,15 +1003,17 @@ export class AnalyticsService {
async calculateTimeBucketForAllTime(
pid: string,
- table: 'analytics' | 'customEV' | 'performance' | 'errors',
+ eventTypes: EventsAllTimeType | readonly EventsAllTimeType[],
): Promise<{
timeBucket: TimeBucketType[]
diff: number
}> {
+ const types = Array.isArray(eventTypes) ? [...eventTypes] : [eventTypes]
+
const { data: fromData } = await clickhouse
.query({
- query: `SELECT min(created) AS firstCreated FROM ${table} WHERE pid = {pid:FixedString(12)}`,
- query_params: { pid },
+ query: `SELECT min(created) AS firstCreated FROM events WHERE pid = {pid:FixedString(12)} AND type IN ({types:Array(String)})`,
+ query_params: { pid, types },
})
.then((res) => res.json<{ firstCreated?: string }>())
@@ -1212,8 +1247,8 @@ export class AnalyticsService {
? 'argMin(pg, created)'
: 'argMax(pg, created)'
const subQueryForPages = isContains
- ? `SELECT psid FROM (SELECT psid, ${pageSelector} as page FROM analytics WHERE pid = {pid:FixedString(12)} GROUP BY psid) WHERE page ILIKE concat('%', {${param}:String}, '%')`
- : `SELECT psid FROM (SELECT psid, ${pageSelector} as page FROM analytics WHERE pid = {pid:FixedString(12)} GROUP BY psid) WHERE page = {${param}:String}`
+ ? `SELECT psid FROM (SELECT psid, ${pageSelector} as page FROM events WHERE pid = {pid:FixedString(12)} AND type = 'pageview' AND created BETWEEN {groupFrom:String} AND {groupTo:String} GROUP BY psid) WHERE page ILIKE concat('%', {${param}:String}, '%')`
+ : `SELECT psid FROM (SELECT psid, ${pageSelector} as page FROM events WHERE pid = {pid:FixedString(12)} AND type = 'pageview' AND created BETWEEN {groupFrom:String} AND {groupTo:String} GROUP BY psid) WHERE page = {${param}:String}`
// For exclusive filter (isNot) we exclude sessions with matching entry/exit page
query += `psid ${isExclusive ? 'NOT IN' : 'IN'} (${subQueryForPages})`
@@ -1254,6 +1289,11 @@ export class AnalyticsService {
) {
sqlColumn = 'meta.value'
isArrayDataset = true
+ } else if (column === 'ev') {
+ // Public filter contract `ev` maps to the renamed `event_name` column
+ sqlColumn = 'event_name'
+ } else if (dataType === DataType.ERRORS) {
+ sqlColumn = mapErrorColumn(column)
}
const isNullFilter =
@@ -1631,21 +1671,11 @@ export class AnalyticsService {
FROM (
SELECT
psid,
- pg AS value,
- created
- FROM analytics
- WHERE pid = {pid:FixedString(12)}
- AND psid != 0
- AND created BETWEEN {groupFrom:String} AND {groupTo:String}
-
- UNION ALL
-
- SELECT
- psid,
- ev AS value,
+ if(type = 'pageview', pg, event_name) AS value,
created
- FROM customEV
+ FROM events
WHERE pid = {pid:FixedString(12)}
+ AND type IN ('pageview', 'custom_event')
AND psid != 0
AND created BETWEEN {groupFrom:String} AND {groupTo:String}
)
@@ -1693,14 +1723,14 @@ export class AnalyticsService {
psid,
CAST(windowFunnel(86400)(created, ${pagesStr}) AS UInt64) AS level
FROM (
- SELECT psid, pg AS value, created
- FROM analytics
- WHERE pid = {pid:FixedString(12)} AND psid != 0
- AND created BETWEEN {groupFrom:String} AND {groupTo:String}
- UNION ALL
- SELECT psid, ev AS value, created
- FROM customEV
- WHERE pid = {pid:FixedString(12)} AND psid != 0
+ SELECT
+ psid,
+ if(type = 'pageview', pg, event_name) AS value,
+ created
+ FROM events
+ WHERE pid = {pid:FixedString(12)}
+ AND type IN ('pageview', 'custom_event')
+ AND psid != 0
AND created BETWEEN {groupFrom:String} AND {groupTo:String}
)
GROUP BY psid
@@ -1715,17 +1745,11 @@ export class AnalyticsService {
psid,
argMin(cc, created) AS cc,
argMin(if(domain(ref) != '', domain(ref), 'Direct / None'), created) AS source
- FROM (
- SELECT psid, cc, ref, created
- FROM analytics
- WHERE pid = {pid:FixedString(12)} AND psid != 0
- AND created BETWEEN {groupFrom:String} AND {groupTo:String}
- UNION ALL
- SELECT psid, cc, ref, created
- FROM customEV
- WHERE pid = {pid:FixedString(12)} AND psid != 0
- AND created BETWEEN {groupFrom:String} AND {groupTo:String}
- )
+ FROM events
+ WHERE pid = {pid:FixedString(12)}
+ AND type IN ('pageview', 'custom_event')
+ AND psid != 0
+ AND created BETWEEN {groupFrom:String} AND {groupTo:String}
GROUP BY psid
)
SELECT step, type, val, cnt FROM (
@@ -1803,14 +1827,14 @@ export class AnalyticsService {
psid,
windowFunnel(86400)(created, ${pagesStr}) AS level
FROM (
- SELECT psid, pg AS value, created
- FROM analytics
- WHERE pid = {pid:FixedString(12)} AND psid != 0
- AND created BETWEEN {groupFrom:String} AND {groupTo:String}
- UNION ALL
- SELECT psid, ev AS value, created
- FROM customEV
- WHERE pid = {pid:FixedString(12)} AND psid != 0
+ SELECT
+ psid,
+ if(type = 'pageview', pg, event_name) AS value,
+ created
+ FROM events
+ WHERE pid = {pid:FixedString(12)}
+ AND type IN ('pageview', 'custom_event')
+ AND psid != 0
AND created BETWEEN {groupFrom:String} AND {groupTo:String}
)
GROUP BY psid
@@ -1826,19 +1850,12 @@ export class AnalyticsService {
any(br) AS br,
min(toTimeZone(created, {timezone:String})) AS sessionStart,
max(toTimeZone(created, {timezone:String})) AS lastActivity
- FROM (
- SELECT psid, pid, cc, os, br, created
- FROM analytics
- WHERE pid = {pid:FixedString(12)} AND psid IS NOT NULL
- AND created BETWEEN {groupFrom:String} AND {groupTo:String}
- AND psid IN (SELECT psid FROM funnel_qualified)
- UNION ALL
- SELECT psid, pid, cc, os, br, created
- FROM customEV
- WHERE pid = {pid:FixedString(12)} AND psid IS NOT NULL
+ FROM events
+ WHERE pid = {pid:FixedString(12)}
+ AND type IN ('pageview', 'custom_event')
+ AND psid IS NOT NULL
AND created BETWEEN {groupFrom:String} AND {groupTo:String}
AND psid IN (SELECT psid FROM funnel_qualified)
- )
GROUP BY psidCasted, pid
),
pageview_counts AS (
@@ -1846,8 +1863,8 @@ export class AnalyticsService {
CAST(psid, 'String') AS psidCasted,
pid,
count() as count
- FROM analytics
- WHERE pid = {pid:FixedString(12)} AND psid IS NOT NULL
+ FROM events
+ WHERE pid = {pid:FixedString(12)} AND type = 'pageview' AND psid IS NOT NULL
AND created BETWEEN {groupFrom:String} AND {groupTo:String}
AND psid IN (SELECT psid FROM funnel_qualified)
GROUP BY psidCasted, pid
@@ -1857,8 +1874,8 @@ export class AnalyticsService {
CAST(psid, 'String') AS psidCasted,
pid,
count() as count
- FROM customEV
- WHERE pid = {pid:FixedString(12)} AND psid IS NOT NULL
+ FROM events
+ WHERE pid = {pid:FixedString(12)} AND type = 'custom_event' AND psid IS NOT NULL
AND created BETWEEN {groupFrom:String} AND {groupTo:String}
AND psid IN (SELECT psid FROM funnel_qualified)
GROUP BY psidCasted, pid
@@ -1868,8 +1885,8 @@ export class AnalyticsService {
CAST(psid, 'String') AS psidCasted,
pid,
count() as count
- FROM errors
- WHERE pid = {pid:FixedString(12)} AND psid IS NOT NULL
+ FROM events
+ WHERE pid = {pid:FixedString(12)} AND type = 'error' AND psid IS NOT NULL
AND created BETWEEN {groupFrom:String} AND {groupTo:String}
AND psid IN (SELECT psid FROM funnel_qualified)
GROUP BY psidCasted, pid
@@ -1947,8 +1964,9 @@ export class AnalyticsService {
const query = `
SELECT
count() as c
- FROM analytics
+ FROM events
WHERE pid = {pid:FixedString(12)}
+ AND type = 'pageview'
AND created BETWEEN {groupFrom:String} AND {groupTo:String}
`
@@ -2044,12 +2062,13 @@ export class AnalyticsService {
groupTo: string,
): Promise<{ cc: string; count: number } | null> {
const query = `
- SELECT
+ SELECT
cc,
count() as count
- FROM analytics
- WHERE
+ FROM events
+ WHERE
pid = {pid:FixedString(12)}
+ AND type = 'pageview'
AND created BETWEEN {groupFrom:String} AND {groupTo:String}
AND cc IS NOT NULL
AND cc != ''
@@ -2079,13 +2098,14 @@ export class AnalyticsService {
const query = `
WITH counts AS (
- SELECT
+ SELECT
pid,
cc,
count() as cnt
- FROM analytics
- WHERE
+ FROM events
+ WHERE
pid IN {pids:Array(FixedString(12))}
+ AND type = 'pageview'
AND created BETWEEN {groupFrom:String} AND {groupTo:String}
AND cc IS NOT NULL
AND cc != ''
@@ -2128,12 +2148,13 @@ export class AnalyticsService {
groupTo: string,
): Promise<{ count: number; uniqueErrors: number }> {
const query = `
- SELECT
+ SELECT
count() as count,
count(DISTINCT eid) as uniqueErrors
- FROM errors
- WHERE
+ FROM events
+ WHERE
pid = {pid:FixedString(12)}
+ AND type = 'error'
AND created BETWEEN {groupFrom:String} AND {groupTo:String}
`
@@ -2159,13 +2180,14 @@ export class AnalyticsService {
}
const query = `
- SELECT
+ SELECT
pid,
count() as count,
count(DISTINCT eid) as uniqueErrors
- FROM errors
- WHERE
+ FROM events
+ WHERE
pid IN {pids:Array(FixedString(12))}
+ AND type = 'error'
AND created BETWEEN {groupFrom:String} AND {groupTo:String}
GROUP BY pid
`
@@ -2207,12 +2229,15 @@ export class AnalyticsService {
}
const query = `
- SELECT
+ SELECT
pid,
uniqExact(psid) as totalSessions
- FROM analytics
- WHERE
+ FROM events
+ WHERE
pid IN {pids:Array(FixedString(12))}
+ AND type IN ('pageview', 'custom_event', 'error')
+ AND psid IS NOT NULL
+ AND psid != ''
AND created BETWEEN {groupFrom:String} AND {groupTo:String}
GROUP BY pid
`
@@ -2277,15 +2302,18 @@ export class AnalyticsService {
const promises = pids.map(async (pid) => {
try {
if (period === 'all') {
- let queryAll = `
+ const allTimeType = this.getAnalyticsEventType(customEVFilterApplied)
+
+ const queryAll = `
WITH analytics_counts AS (
SELECT
count(*) AS all,
count(DISTINCT psid) AS unique,
count(DISTINCT profileId) AS users
- FROM analytics
+ FROM events
WHERE
pid = {pid:FixedString(12)}
+ AND type = '${allTimeType}'
${filtersQuery}
),
duration_avg AS (
@@ -2296,6 +2324,14 @@ export class AnalyticsService {
dateDiff('second', min(firstSeen), max(lastSeen)) as duration
FROM sessions
WHERE pid = {pid:FixedString(12)}
+ AND psid IN (
+ SELECT DISTINCT psid
+ FROM events
+ WHERE pid = {pid:FixedString(12)}
+ AND type = '${allTimeType}'
+ AND psid IS NOT NULL
+ ${filtersQuery}
+ )
GROUP BY psid
)
)
@@ -2305,36 +2341,6 @@ export class AnalyticsService {
FROM analytics_counts, duration_avg
`
- if (customEVFilterApplied) {
- queryAll = `
- WITH analytics_counts AS (
- SELECT
- count(*) AS all,
- count(DISTINCT psid) AS unique,
- count(DISTINCT profileId) AS users
- FROM customEV
- WHERE
- pid = {pid:FixedString(12)}
- ${filtersQuery}
- ),
- duration_avg AS (
- SELECT avgOrNull(duration) as sdur
- FROM (
- SELECT
- psid,
- dateDiff('second', min(firstSeen), max(lastSeen)) as duration
- FROM sessions
- WHERE pid = {pid:FixedString(12)}
- GROUP BY psid
- )
- )
- SELECT
- analytics_counts.*,
- duration_avg.sdur
- FROM analytics_counts, duration_avg
- `
- }
-
const { data } = await clickhouse
.query({
query: queryAll,
@@ -2378,7 +2384,7 @@ export class AnalyticsService {
const { diff: allDiff, timeBucket: allowedBuckets } =
await this.calculateTimeBucketForAllTime(
pid,
- customEVFilterApplied ? 'customEV' : 'analytics',
+ this.getAnalyticsEventType(customEVFilterApplied),
)
const allTimeChartBucket = _includes(
allowedBuckets,
@@ -2419,16 +2425,19 @@ export class AnalyticsService {
)
.format('YYYY-MM-DD HH:mm:ss')
- let queryCurrent = `
+ const periodType = this.getAnalyticsEventType(customEVFilterApplied)
+
+ const queryCurrent = `
WITH analytics_counts AS (
SELECT
1 AS sortOrder,
count(*) AS all,
count(DISTINCT psid) AS unique,
count(DISTINCT profileId) AS users
- FROM analytics
+ FROM events
WHERE
pid = {pid:FixedString(12)}
+ AND type = '${periodType}'
AND created BETWEEN {groupFromUTC:String} AND {groupToUTC:String}
${filtersQuery}
),
@@ -2442,8 +2451,9 @@ export class AnalyticsService {
WHERE pid = {pid:FixedString(12)}
AND psid IN (
SELECT DISTINCT psid
- FROM analytics
+ FROM events
WHERE pid = {pid:FixedString(12)}
+ AND type = '${periodType}'
AND psid IS NOT NULL
AND created BETWEEN {groupFromUTC:String} AND {groupToUTC:String}
${filtersQuery}
@@ -2457,16 +2467,17 @@ export class AnalyticsService {
FROM analytics_counts, duration_avg
`
- let queryPrevious = `
+ const queryPrevious = `
WITH analytics_counts AS (
SELECT
2 AS sortOrder,
count(*) AS all,
count(DISTINCT psid) AS unique,
count(DISTINCT profileId) AS users
- FROM analytics
+ FROM events
WHERE
pid = {pid:FixedString(12)}
+ AND type = '${periodType}'
AND created BETWEEN {periodSubtracted:String} AND {groupFromUTC:String}
${filtersQuery}
),
@@ -2480,8 +2491,9 @@ export class AnalyticsService {
WHERE pid = {pid:FixedString(12)}
AND psid IN (
SELECT DISTINCT psid
- FROM analytics
+ FROM events
WHERE pid = {pid:FixedString(12)}
+ AND type = '${periodType}'
AND psid IS NOT NULL
AND created BETWEEN {periodSubtracted:String} AND {groupFromUTC:String}
${filtersQuery}
@@ -2495,83 +2507,6 @@ export class AnalyticsService {
FROM analytics_counts, duration_avg
`
- if (customEVFilterApplied) {
- queryCurrent = `
- WITH analytics_counts AS (
- SELECT
- 1 AS sortOrder,
- count(*) AS all,
- count(DISTINCT psid) AS unique,
- count(DISTINCT profileId) AS users
- FROM customEV
- WHERE
- pid = {pid:FixedString(12)}
- AND created BETWEEN {groupFromUTC:String} AND {groupToUTC:String}
- ${filtersQuery}
- ),
- duration_avg AS (
- SELECT avgOrNull(duration) as sdur
- FROM (
- SELECT
- psid,
- dateDiff('second', min(firstSeen), max(lastSeen)) as duration
- FROM sessions
- WHERE pid = {pid:FixedString(12)}
- AND psid IN (
- SELECT DISTINCT psid
- FROM customEV
- WHERE pid = {pid:FixedString(12)}
- AND psid IS NOT NULL
- AND created BETWEEN {groupFromUTC:String} AND {groupToUTC:String}
- ${filtersQuery}
- )
- GROUP BY psid
- )
- )
- SELECT
- analytics_counts.*,
- duration_avg.sdur
- FROM analytics_counts, duration_avg
- `
- queryPrevious = `
- WITH analytics_counts AS (
- SELECT
- 2 AS sortOrder,
- count(*) AS all,
- count(DISTINCT psid) AS unique,
- count(DISTINCT profileId) AS users
- FROM customEV
- WHERE
- pid = {pid:FixedString(12)}
- AND created BETWEEN {periodSubtracted:String} AND {groupFromUTC:String}
- ${filtersQuery}
- ),
- duration_avg AS (
- SELECT avgOrNull(duration) as sdur
- FROM (
- SELECT
- psid,
- dateDiff('second', min(firstSeen), max(lastSeen)) as duration
- FROM sessions
- WHERE pid = {pid:FixedString(12)}
- AND psid IN (
- SELECT DISTINCT psid
- FROM customEV
- WHERE pid = {pid:FixedString(12)}
- AND psid IS NOT NULL
- AND created BETWEEN {periodSubtracted:String} AND {groupFromUTC:String}
- ${filtersQuery}
- )
- GROUP BY psid
- )
- )
- SELECT
- analytics_counts.*,
- duration_avg.sdur
- FROM analytics_counts, duration_avg
- `
- }
-
const query = `${queryCurrent} UNION ALL ${queryPrevious}`
let { data } = await clickhouse
@@ -2666,6 +2601,55 @@ export class AnalyticsService {
return result
}
+ private generateAnalyticsAggregationQueryForScope(
+ timeBucket: TimeBucketType,
+ filtersQuery: string,
+ mode: ChartRenderMode,
+ customEVFilterApplied: boolean,
+ ): string {
+ return customEVFilterApplied
+ ? this.generateCustomEventsAggregationQuery(
+ timeBucket,
+ filtersQuery,
+ mode,
+ )
+ : this.generateAnalyticsAggregationQuery(timeBucket, filtersQuery, mode)
+ }
+
+ private extractAnalyticsChartDataForScope(
+ result: Array<TrafficCHResponse | TrafficCEFilterCHResponse>,
+ xShifted: string[],
+ customEVFilterApplied: boolean,
+ mode: ChartRenderMode,
+ ): IExtractChartData {
+ const chartData = this.extractChartData(result, xShifted)
+
+ if (customEVFilterApplied) {
+ const visits =
+ this.extractCustomEventsChartData(result, xShifted)?._unknown_event ||
+ []
+
+ chartData.visits = Array.from(
+ { length: _size(xShifted) },
+ (_, index) => visits[index] || 0,
+ )
+ }
+
+ // Propagate the previous cumulative value forward for empty buckets
+ if (mode === ChartRenderMode.CUMULATIVE) {
+ for (let i = 1; i < chartData.visits.length; ++i) {
+ if (chartData.visits[i] === 0) {
+ chartData.visits[i] = chartData.visits[i - 1]
+ }
+ if (chartData.uniques[i] === 0) {
+ chartData.uniques[i] = chartData.uniques[i - 1]
+ }
+ }
+ }
+
+ return chartData
+ }
+
/**
* Get simplified chart data for dashboard cards
* Returns only x (dates) and visits (pageviews) for a lightweight chart
@@ -2696,30 +2680,11 @@ export class AnalyticsService {
},
}
- if (customEVFilterApplied) {
- const query = this.generateCustomEventsAggregationQuery(
- timeBucket,
- filtersQuery,
- ChartRenderMode.PERIODICAL,
- )
-
- const { data } = await clickhouse
- .query({
- query,
- query_params: { ...paramsData.params, timezone: safeTimezone },
- })
- .then((resultSet) => resultSet.json<TrafficCEFilterCHResponse>())
-
- const visits =
- this.extractCustomEventsChartData(data, xShifted)?._unknown_event || []
-
- return { x: xShifted, visits }
- }
-
- const query = this.generateAnalyticsAggregationQuery(
+ const query = this.generateAnalyticsAggregationQueryForScope(
timeBucket,
filtersQuery,
ChartRenderMode.PERIODICAL,
+ customEVFilterApplied,
)
const { data } = await clickhouse
@@ -2727,9 +2692,16 @@ export class AnalyticsService {
query,
query_params: { ...paramsData.params, timezone: safeTimezone },
})
- .then((resultSet) => resultSet.json<TrafficCHResponse>())
+ .then((resultSet) =>
+ resultSet.json<TrafficCHResponse | TrafficCEFilterCHResponse>(),
+ )
- const { visits } = this.extractChartData(data, xShifted)
+ const { visits } = this.extractAnalyticsChartDataForScope(
+ data,
+ xShifted,
+ customEVFilterApplied,
+ ChartRenderMode.PERIODICAL,
+ )
return { x: xShifted, visits }
}
@@ -2750,7 +2722,7 @@ export class AnalyticsService {
if (_isEmpty(period) || ['today', 'yesterday', 'custom'].includes(period)) {
const safeTimezone = this.getSafeTimezone(timezone)
- const { groupFrom, groupTo } = this.getGroupFromTo(
+ const { groupFromUTC, groupToUTC } = this.getGroupFromTo(
from,
to,
['today', 'yesterday'].includes(period) ? TimeBucketType.HOUR : null,
@@ -2758,8 +2730,8 @@ export class AnalyticsService {
safeTimezone,
)
- _from = groupFrom
- _to = groupTo
+ _from = groupFromUTC
+ _to = groupToUTC
}
const result = {}
@@ -2779,7 +2751,7 @@ export class AnalyticsService {
const promises = pids.map(async (pid) => {
try {
if (period === 'all') {
- const queryAll = `SELECT ${columnSelectors} FROM performance WHERE pid = {pid:FixedString(12)} ${filtersQuery}`
+ const queryAll = `SELECT ${columnSelectors} FROM events WHERE pid = {pid:FixedString(12)} AND type = 'performance' ${filtersQuery}`
const { data } = await clickhouse
.query({
@@ -2823,11 +2795,12 @@ export class AnalyticsService {
if (_from && _to) {
// diff may be 0 (when selecting data for 1 day), so let's make it 1 to grab some data for the prev day as well
- const diff = dayjs(_to).diff(dayjs(_from), 'days') || 1
+ const diff = dayjs.utc(_to).diff(dayjs.utc(_from), 'days') || 1
now = _to
periodFormatted = _from
- periodSubtracted = dayjs(_from)
+ periodSubtracted = dayjs
+ .utc(_from)
.subtract(diff, 'days')
.format('YYYY-MM-DD HH:mm:ss')
} else {
@@ -2842,8 +2815,8 @@ export class AnalyticsService {
.format('YYYY-MM-DD HH:mm:ss')
}
- const queryCurrent = `SELECT 1 AS sortOrder, ${columnSelectors} FROM performance WHERE pid = {pid:FixedString(12)} AND created BETWEEN {periodFormatted:String} AND {now:String} ${filtersQuery}`
- const queryPrevious = `SELECT 2 AS sortOrder, ${columnSelectors} FROM performance WHERE pid = {pid:FixedString(12)} AND created BETWEEN {periodSubtracted:String} AND {periodFormatted:String} ${filtersQuery}`
+ const queryCurrent = `SELECT 1 AS sortOrder, ${columnSelectors} FROM events WHERE pid = {pid:FixedString(12)} AND type = 'performance' AND created BETWEEN {periodFormatted:String} AND {now:String} ${filtersQuery}`
+ const queryPrevious = `SELECT 2 AS sortOrder, ${columnSelectors} FROM events WHERE pid = {pid:FixedString(12)} AND type = 'performance' AND created BETWEEN {periodSubtracted:String} AND {periodFormatted:String} ${filtersQuery}`
const query = `${queryCurrent} UNION ALL ${queryPrevious}`
@@ -2940,8 +2913,8 @@ export class AnalyticsService {
const query = `
WITH session_pages AS (
SELECT psid, ${selector} as page
- FROM analytics
- WHERE pid = {pid:FixedString(12)}
+ FROM events
+ WHERE pid = {pid:FixedString(12)} AND type = 'pageview'
GROUP BY psid
)
SELECT page FROM session_pages WHERE page IS NOT NULL GROUP BY page
@@ -2957,11 +2930,11 @@ export class AnalyticsService {
return _map(data, 'page')
}
- let query = `SELECT ${type} FROM analytics WHERE pid={pid:FixedString(12)} AND ${type} IS NOT NULL GROUP BY ${type}`
-
- if (type === 'ev') {
- query = `SELECT ${type} FROM customEV WHERE pid={pid:FixedString(12)} AND ${type} IS NOT NULL GROUP BY ${type}`
- }
+ // Public filter contract `ev` reads from the renamed event_name column.
+ let query =
+ type === 'ev'
+ ? `SELECT event_name AS ev FROM events WHERE pid={pid:FixedString(12)} AND type = 'custom_event' AND event_name IS NOT NULL GROUP BY event_name`
+ : `SELECT ${type} FROM events WHERE pid={pid:FixedString(12)} AND type IN ('pageview', 'custom_event') AND ${type} IS NOT NULL GROUP BY ${type}`
const { data } = await clickhouse
.query({
@@ -2982,7 +2955,9 @@ export class AnalyticsService {
)
}
- const query = `SELECT ${type} FROM errors WHERE pid={pid:FixedString(12)} AND ${type} IS NOT NULL GROUP BY ${type}`
+ const sqlCol = mapErrorColumn(type)
+
+ const query = `SELECT ${sqlCol} AS ${type} FROM events WHERE pid={pid:FixedString(12)} AND type = 'error' AND ${sqlCol} IS NOT NULL GROUP BY ${sqlCol}`
const { data } = await clickhouse
.query({
@@ -3001,13 +2976,16 @@ export class AnalyticsService {
type: 'traffic' | 'errors',
column: 'br' | 'os',
): Promise<Array<{ name: string; version: string }>> {
- const safeTable = type === 'errors' ? 'errors' : 'analytics'
+ const safeType =
+ type === 'errors'
+ ? "type = 'error'"
+ : "type IN ('pageview', 'custom_event')"
const safeVersionCol = column === 'br' ? 'brv' : 'osv'
const query = `
- SELECT ${column}, ${safeVersionCol}
- FROM ${safeTable}
- WHERE pid={pid:FixedString(12)} AND ${column} IS NOT NULL AND ${safeVersionCol} IS NOT NULL
+ SELECT ${column}, ${safeVersionCol}
+ FROM events
+ WHERE pid={pid:FixedString(12)} AND ${safeType} AND ${column} IS NOT NULL AND ${safeVersionCol} IS NOT NULL
GROUP BY ${column}, ${safeVersionCol}
`
@@ -3067,7 +3045,7 @@ export class AnalyticsService {
const query = `
WITH ${withClauses.join(',\n')}
- SELECT
+ SELECT
column_name,
name,
count,
@@ -3088,7 +3066,7 @@ export class AnalyticsService {
}
return `
- SELECT
+ SELECT
'${col}' as column_name,
name,
count,
@@ -3385,8 +3363,8 @@ export class AnalyticsService {
pid,
psid,
${timeBucketFunc}(toTimeZone(created, {timezone:String})) as tz_created
- FROM analytics
- PREWHERE pid = {pid:FixedString(12)}
+ FROM events
+ PREWHERE pid = {pid:FixedString(12)} AND type = 'pageview'
WHERE created BETWEEN ${tzFromDate} AND ${tzToDate}
${filtersQuery}
) as subquery
@@ -3434,8 +3412,8 @@ export class AnalyticsService {
FROM (
SELECT
${timeBucketFunc}(toTimeZone(created, {timezone:String})) as tz_created
- FROM analytics
- PREWHERE pid = {pid:FixedString(12)}
+ FROM events
+ PREWHERE pid = {pid:FixedString(12)} AND type = 'pageview'
WHERE created BETWEEN ${tzFromDate} AND ${tzToDate}
${filtersQuery}
) as subquery
@@ -3462,8 +3440,8 @@ export class AnalyticsService {
FROM (
SELECT
${timeBucketFunc}(toTimeZone(created, {timezone:String})) as tz_created
- FROM customEV
- PREWHERE pid = {pid:FixedString(12)}
+ FROM events
+ PREWHERE pid = {pid:FixedString(12)} AND type = 'custom_event'
WHERE created BETWEEN ${tzFromDate} AND ${tzToDate}
${filtersQuery}
) as subquery
@@ -3488,8 +3466,8 @@ export class AnalyticsService {
FROM (
SELECT
${timeBucketFunc}(toTimeZone(created, {timezone:String})) as tz_created
- FROM errors
- PREWHERE pid = {pid:FixedString(12)}
+ FROM events
+ PREWHERE pid = {pid:FixedString(12)} AND type = 'error'
WHERE created BETWEEN ${tzFromDate} AND ${tzToDate}
${filtersQuery}
) as subquery
@@ -3511,15 +3489,31 @@ export class AnalyticsService {
const baseQuery = `
SELECT
${selector},
+ avgOrNull(sessions_data.duration) as sdur,
+ count(DISTINCT psid) as uniques,
count() as count
FROM (
- SELECT *,
- ${timeBucketFunc}(toTimeZone(created, {timezone:String})) as tz_created
- FROM customEV
+ SELECT
+ pid,
+ psid,
+ ${timeBucketFunc}(toTimeZone(created, {timezone:String})) as tz_created
+ FROM events
WHERE pid = {pid:FixedString(12)}
+ AND type = 'custom_event'
AND created BETWEEN ${tzFromDate} AND ${tzToDate}
${filtersQuery}
) as subquery
+ LEFT JOIN (
+ SELECT
+ pid,
+ psid,
+ dateDiff('second', min(firstSeen), max(lastSeen)) as duration
+ FROM sessions
+ WHERE pid = {pid:FixedString(12)}
+ GROUP BY pid, psid
+ ) as sessions_data
+ ON subquery.pid = sessions_data.pid
+ AND subquery.psid = sessions_data.psid
GROUP BY ${groupBy}
ORDER BY ${groupBy}
`
@@ -3528,7 +3522,8 @@ export class AnalyticsService {
return `
SELECT
*,
- sum(count) OVER (ORDER BY ${groupBy}) as count
+ sum(count) OVER (ORDER BY ${groupBy}) as count,
+ sum(uniques) OVER (ORDER BY ${groupBy}) as uniques
FROM (${baseQuery})
`
}
@@ -3559,9 +3554,10 @@ export class AnalyticsService {
FROM (
SELECT *,
${timeBucketFunc}(toTimeZone(created, {timezone:String})) as tz_created
- FROM performance
+ FROM events
WHERE
pid = {pid:FixedString(12)}
+ AND type = 'performance'
AND created BETWEEN ${tzFromDate} AND ${tzToDate}
${filtersQuery}
) as subquery
@@ -3594,9 +3590,10 @@ export class AnalyticsService {
FROM (
SELECT *,
${timeBucketFunc}(toTimeZone(created, {timezone:String})) as tz_created
- FROM performance
+ FROM events
WHERE
pid = {pid:FixedString(12)}
+ AND type = 'performance'
AND created BETWEEN ${tzFromDate} AND ${tzToDate}
${filtersQuery}
) as subquery
@@ -3622,9 +3619,10 @@ export class AnalyticsService {
FROM (
SELECT *,
${timeBucketFunc}(toTimeZone(created, {timezone:String})) as tz_created
- FROM captcha
+ FROM events
WHERE
pid = {pid:FixedString(12)}
+ AND type = 'captcha'
AND created BETWEEN ${tzFromDate} AND ${tzToDate}
${filtersQuery}
) as subquery
@@ -3661,9 +3659,10 @@ export class AnalyticsService {
FROM (
SELECT pg, dv, br, os, lc, cc, rg, ct, profileId,
${timeBucketFunc}(created) as tz_created
- FROM errors
+ FROM events
WHERE
pid = {pid:FixedString(12)}
+ AND type = 'error'
AND created BETWEEN {groupFrom:String} AND {groupTo:String}
${filtersQuery}
) as subquery
@@ -3674,10 +3673,11 @@ export class AnalyticsService {
if (mode === ChartRenderMode.CUMULATIVE) {
return `
SELECT
- *,
- sum(pageviews) OVER (ORDER BY ${groupBy}) as pageviews,
- sum(uniques) OVER (ORDER BY ${groupBy}) as uniques
+ ${groupBy},
+ sum(count) OVER (ORDER BY ${groupBy}) as count,
+ sum(affectedUsers) OVER (ORDER BY ${groupBy}) as affectedUsers
FROM (${baseQuery})
+ ORDER BY ${groupBy}
`
}
@@ -3742,7 +3742,7 @@ export class AnalyticsService {
): Promise<{ name: string; count: number }[]> {
const query = `
WITH session_first_pages AS (
- SELECT
+ SELECT
psid,
argMin(pg, created) as entry_page
${subQuery}
@@ -3837,39 +3837,11 @@ export class AnalyticsService {
): Promise<object | void> {
const { xShifted } = this.generateXAxis(timeBucket, from, to, safeTimezone)
- if (customEVFilterApplied) {
- const query = this.generateCustomEventsAggregationQuery(
- timeBucket,
- filtersQuery,
- mode,
- )
-
- const { data } = await clickhouse
- .query({
- query,
- query_params: { ...paramsData.params, timezone: safeTimezone },
- })
- .then((resultSet) => resultSet.json<TrafficCEFilterCHResponse>())
-
- const uniques =
- this.extractCustomEventsChartData(data, xShifted)?._unknown_event || []
-
- const sdur = Array(_size(xShifted)).fill(0)
-
- return Promise.resolve({
- chart: {
- x: xShifted,
- visits: uniques,
- uniques,
- sdur,
- },
- })
- }
-
- const query = this.generateAnalyticsAggregationQuery(
+ const query = this.generateAnalyticsAggregationQueryForScope(
timeBucket,
filtersQuery,
mode,
+ customEVFilterApplied,
)
const { data } = await clickhouse
@@ -3877,21 +3849,16 @@ export class AnalyticsService {
query,
query_params: { ...paramsData.params, timezone: safeTimezone },
})
- .then((resultSet) => resultSet.json<TrafficCHResponse>())
-
- const { visits, uniques, sdur } = this.extractChartData(data, xShifted)
+ .then((resultSet) =>
+ resultSet.json<TrafficCHResponse | TrafficCEFilterCHResponse>(),
+ )
- // Propagate the previous cumulative value forward for empty buckets
- if (mode === ChartRenderMode.CUMULATIVE) {
- for (let i = 1; i < visits.length; ++i) {
- if (visits[i] === 0) {
- visits[i] = visits[i - 1]
- }
- if (uniques[i] === 0) {
- uniques[i] = uniques[i - 1]
- }
- }
- }
+ const { visits, uniques, sdur } = this.extractAnalyticsChartDataForScope(
+ data,
+ xShifted,
+ customEVFilterApplied,
+ mode,
+ )
return Promise.resolve({
chart: {
@@ -4431,7 +4398,7 @@ export class AnalyticsService {
filtersQuery: string,
params: any,
): Promise<ICustomEvent> {
- const query = `SELECT ev, count() FROM customEV WHERE pid = {pid:FixedString(12)} ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String} GROUP BY ev`
+ const query = `SELECT event_name AS ev, count() FROM events WHERE pid = {pid:FixedString(12)} AND type = 'custom_event' ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String} GROUP BY event_name`
const result = {}
const { data } = await clickhouse
@@ -4461,10 +4428,10 @@ export class AnalyticsService {
FROM (
SELECT
meta.key AS key
- FROM
- analytics
+ FROM events
WHERE
pid = {pid:FixedString(12)}
+ AND type = 'pageview'
${filtersQuery}
AND created BETWEEN {groupFrom:String} AND {groupTo:String}
)
@@ -4518,7 +4485,7 @@ export class AnalyticsService {
)
if (period === 'all') {
- const res = await this.calculateTimeBucketForAllTime(pid, 'customEV')
+ const res = await this.calculateTimeBucketForAllTime(pid, 'custom_event')
diff = res.diff
@@ -4546,12 +4513,12 @@ export class AnalyticsService {
SELECT
meta.key,
meta.value
- FROM
- customEV
+ FROM events
WHERE
pid = {pid:FixedString(12)}
+ AND type = 'custom_event'
AND created BETWEEN {groupFrom:String} AND {groupTo:String}
- AND ev = {event:String}
+ AND event_name = {event:String}
${filtersQuery}
)
ARRAY JOIN meta.key, meta.value
@@ -4610,7 +4577,7 @@ export class AnalyticsService {
const [filtersQuery, filtersParams, appliedFilters, customEVFilterApplied] =
this.getFiltersQuery(filters, DataType.ANALYTICS)
- // We cannot make a query to customEV table using analytics table properties
+ // Page properties are only present on pageview-scoped analytics rows.
if (customEVFilterApplied) {
return {
result: [],
@@ -4619,7 +4586,7 @@ export class AnalyticsService {
}
if (period === 'all') {
- const res = await this.calculateTimeBucketForAllTime(pid, 'analytics')
+ const res = await this.calculateTimeBucketForAllTime(pid, 'pageview')
diff = res.diff
@@ -4647,10 +4614,10 @@ export class AnalyticsService {
SELECT
meta.key,
meta.value
- FROM
- analytics
+ FROM events
WHERE
pid = {pid:FixedString(12)}
+ AND type = 'pageview'
AND created BETWEEN {groupFrom:String} AND {groupTo:String}
AND indexOf(meta.key, {property:String}) > 0
${filtersQuery}
@@ -4700,17 +4667,11 @@ export class AnalyticsService {
const query = `
SELECT uniqExact(psid) as count
- FROM (
- SELECT psid FROM analytics
- WHERE pid = {pid:FixedString(12)}
- AND created >= {since:DateTime}
- AND psid IS NOT NULL
- UNION ALL
- SELECT psid FROM customEV
- WHERE pid = {pid:FixedString(12)}
- AND created >= {since:DateTime}
- AND psid IS NOT NULL
- )
+ FROM events
+ WHERE pid = {pid:FixedString(12)}
+ AND type IN ('pageview', 'custom_event')
+ AND created >= {since:DateTime}
+ AND psid IS NOT NULL
`
try {
@@ -4748,25 +4709,26 @@ export class AnalyticsService {
const tzToDate = `toTimeZone(parseDateTimeBestEffort({groupTo:String}), {timezone:String})`
const customEventsFilter =
customEvents && customEvents.length > 0
- ? 'AND ev IN {customEvents:Array(String)}'
+ ? 'AND event_name IN {customEvents:Array(String)}'
: ''
const query = `
SELECT
${selector},
- ev,
+ event_name AS ev,
count() as count
FROM (
SELECT *,
${timeBucketFunc}(toTimeZone(created, {timezone:String})) as tz_created
- FROM customEV
+ FROM events
WHERE
pid = {pid:FixedString(12)}
+ AND type = 'custom_event'
AND created BETWEEN ${tzFromDate} AND ${tzToDate}
${filtersQuery}
${customEventsFilter}
) as subquery
- GROUP BY ${groupBy}, ev
+ GROUP BY ${groupBy}, event_name
ORDER BY ${groupBy}
`
@@ -4864,52 +4826,47 @@ export class AnalyticsService {
const queryPages = `
WITH events_with_meta AS (
SELECT
- 'pageview' AS type,
- toString(pg) AS value,
- toTimeZone(analytics.created, {timezone:String}) AS created,
- pid,
- toString(analytics.psid) AS psid,
- arrayFilter(x -> x.1 != '' AND x.2 != '', arrayZip(meta.key, meta.value)) AS metadata
- FROM analytics
- WHERE
- pid = {pid:FixedString(12)}
- AND analytics.psid IS NOT NULL
- AND toString(analytics.psid) = {psid:String}
-
- UNION ALL
-
- SELECT
- 'event' AS type,
- toString(ev) AS value,
- toTimeZone(customEV.created, {timezone:String}) AS created,
- pid,
- toString(customEV.psid) AS psid,
- arrayFilter(x -> x.1 != '' AND x.2 != '', arrayZip(meta.key, meta.value)) AS metadata
- FROM customEV
- WHERE
- pid = {pid:FixedString(12)}
- AND customEV.psid IS NOT NULL
- AND toString(customEV.psid) = {psid:String}
-
- UNION ALL
-
- SELECT
- 'error' AS type,
- toString(errors.name) AS value,
- toTimeZone(errors.created, {timezone:String}) AS created,
+ multiIf(event_type = 'pageview', 'pageview', event_type = 'custom_event', 'event', 'error') AS type,
+ multiIf(
+ event_type = 'pageview', toString(pg),
+ event_type = 'custom_event', toString(event_name),
+ toString(error_name)
+ ) AS value,
+ toTimeZone(created, {timezone:String}) AS created,
pid,
- toString(errors.psid) AS psid,
- [
- tuple('message', COALESCE(errors.message, '')),
- tuple('lineno', toString(COALESCE(errors.lineno, 0))),
- tuple('colno', toString(COALESCE(errors.colno, 0))),
- tuple('filename', COALESCE(errors.filename, ''))
- ] AS metadata
- FROM errors
- WHERE
- pid = {pid:FixedString(12)}
- AND errors.psid IS NOT NULL
- AND toString(errors.psid) = {psid:String}
+ toString(psid) AS psid,
+ if(
+ event_type = 'error',
+ arrayFilter(x -> x.2 != '' AND x.2 != '0', [
+ tuple('message', COALESCE(error_message, '')),
+ tuple('lineno', ifNull(toString(lineno), '')),
+ tuple('colno', ifNull(toString(colno), '')),
+ tuple('filename', COALESCE(error_filename, ''))
+ ]),
+ arrayFilter(x -> x.1 != '' AND x.2 != '', arrayZip(meta_key, meta_value))
+ ) AS metadata
+ FROM (
+ SELECT
+ type AS event_type,
+ pg,
+ event_name,
+ error_name,
+ error_message,
+ lineno,
+ colno,
+ error_filename,
+ created,
+ pid,
+ psid,
+ meta.key AS meta_key,
+ meta.value AS meta_value
+ FROM events
+ WHERE
+ pid = {pid:FixedString(12)}
+ AND type IN ('pageview', 'custom_event', 'error')
+ AND psid IS NOT NULL
+ AND toString(psid) = {psid:String}
+ )
UNION ALL
@@ -4961,12 +4918,19 @@ export class AnalyticsService {
const querySessionDetails = `
SELECT
dv, br, brv, os, osv, lc, ref, so, me, ca, te, co, cc, rg, ct, profileId
- FROM analytics
+ FROM events
WHERE
pid = {pid:FixedString(12)}
+ AND type IN ('pageview', 'custom_event', 'error')
AND psid IS NOT NULL
AND toString(psid) = {psid:String}
- ORDER BY created ASC
+ ORDER BY
+ CASE
+ WHEN type = 'pageview' THEN 0
+ WHEN type = 'custom_event' THEN 1
+ ELSE 2
+ END,
+ created ASC
LIMIT 1;
`
@@ -5057,9 +5021,10 @@ export class AnalyticsService {
const querySessionDetailsFromCustomEV = `
SELECT
dv, br, brv, os, osv, lc, ref, so, me, ca, te, co, cc, rg, ct, profileId
- FROM customEV
+ FROM events
WHERE
pid = {pid:FixedString(12)}
+ AND type IN ('custom_event', 'error')
AND psid IS NOT NULL
AND toString(psid) = {psid:String}
ORDER BY created ASC
@@ -5149,94 +5114,16 @@ export class AnalyticsService {
skip = 0,
customEVFilterApplied = false,
): Promise<object | void> {
- let primaryEventsSubquery: string
-
- if (customEVFilterApplied) {
- // When filtering by custom events, we need to:
- // 1. Identify sessions (psids) that have matching custom events
- // 2. Calculate session boundaries from ALL analytics events for those sessions
- primaryEventsSubquery = `
- SELECT
- all_events.psidCasted,
- all_events.pid,
- all_events.cc,
- all_events.os,
- all_events.br,
- all_events.created_for_grouping
- FROM (
- SELECT
- CAST(analytics.psid, 'String') AS psidCasted,
- analytics.pid,
- analytics.cc,
- analytics.os,
- analytics.br,
- toTimeZone(analytics.created, {timezone:String}) AS created_for_grouping
- FROM analytics
- WHERE
- analytics.pid = {pid:FixedString(12)}
- AND analytics.psid IS NOT NULL
- AND analytics.created BETWEEN {groupFrom:String} AND {groupTo:String}
- UNION ALL
- SELECT
- CAST(customEV.psid, 'String') AS psidCasted,
- customEV.pid,
- customEV.cc,
- customEV.os,
- customEV.br,
- toTimeZone(customEV.created, {timezone:String}) AS created_for_grouping
- FROM customEV
- WHERE
- customEV.pid = {pid:FixedString(12)}
- AND customEV.psid IS NOT NULL
- AND customEV.created BETWEEN {groupFrom:String} AND {groupTo:String}
- ) AS all_events
- WHERE all_events.psidCasted IN (
- SELECT DISTINCT CAST(customEV.psid, 'String')
- FROM customEV
- WHERE
- customEV.pid = {pid:FixedString(12)}
- AND customEV.psid IS NOT NULL
- AND customEV.created BETWEEN {groupFrom:String} AND {groupTo:String}
- ${filtersQuery}
- )
- `
- } else {
- primaryEventsSubquery = `
- SELECT
- CAST(analytics.psid, 'String') AS psidCasted,
- analytics.pid,
- analytics.cc,
- analytics.os,
- analytics.br,
- toTimeZone(analytics.created, {timezone:String}) AS created_for_grouping
- FROM analytics
- WHERE
- analytics.pid = {pid:FixedString(12)}
- AND analytics.psid IS NOT NULL
- AND analytics.created BETWEEN {groupFrom:String} AND {groupTo:String}
- ${filtersQuery}
- UNION ALL
- SELECT
- CAST(customEV.psid, 'String') AS psidCasted,
- customEV.pid,
- customEV.cc,
- customEV.os,
- customEV.br,
- toTimeZone(customEV.created, {timezone:String}) AS created_for_grouping
- FROM customEV
- WHERE
- customEV.pid = {pid:FixedString(12)}
- AND customEV.psid IS NOT NULL
- AND customEV.created BETWEEN {groupFrom:String} AND {groupTo:String}
- ${filtersQuery}
- `
- }
+ const primaryEventsSubquery = this.buildSessionsListPrimaryEventsSubquery(
+ filtersQuery,
+ customEVFilterApplied,
+ )
const query = `
WITH distinct_sessions_filtered AS (
SELECT
psidCasted,
- pid,
+ pid,
any(cc) AS cc,
any(os) AS os,
any(br) AS br,
@@ -5246,32 +5133,32 @@ export class AnalyticsService {
GROUP BY psidCasted, pid
),
pageview_counts AS (
- SELECT
+ SELECT
CAST(psid, 'String') AS psidCasted,
- pid,
- count() as count
- FROM analytics
- WHERE pid = {pid:FixedString(12)} AND psid IS NOT NULL
+ pid,
+ count() as count
+ FROM events
+ WHERE pid = {pid:FixedString(12)} AND type = 'pageview' AND psid IS NOT NULL
AND created BETWEEN {groupFrom:String} AND {groupTo:String}
GROUP BY psidCasted, pid
),
event_counts AS (
- SELECT
+ SELECT
CAST(psid, 'String') AS psidCasted,
- pid,
- count() as count
- FROM customEV
- WHERE pid = {pid:FixedString(12)} AND psid IS NOT NULL
+ pid,
+ count() as count
+ FROM events
+ WHERE pid = {pid:FixedString(12)} AND type = 'custom_event' AND psid IS NOT NULL
AND created BETWEEN {groupFrom:String} AND {groupTo:String}
GROUP BY psidCasted, pid
),
error_counts AS (
- SELECT
+ SELECT
CAST(psid, 'String') AS psidCasted,
- pid,
- count() as count
- FROM errors
- WHERE pid = {pid:FixedString(12)} AND psid IS NOT NULL
+ pid,
+ count() as count
+ FROM events
+ WHERE pid = {pid:FixedString(12)} AND type = 'error' AND psid IS NOT NULL
AND created BETWEEN {groupFrom:String} AND {groupTo:String}
GROUP BY psidCasted, pid
),
@@ -5296,9 +5183,9 @@ export class AnalyticsService {
GROUP BY psidCasted, pid
),
session_duration_agg AS (
- SELECT
- CAST(psid, 'String') AS psidCasted,
- pid,
+ SELECT
+ CAST(psid, 'String') AS psidCasted,
+ pid,
dateDiff('second', min(firstSeen), max(lastSeen)) as avg_duration,
argMax(profileId, lastSeen) as profileId
FROM sessions
@@ -5360,75 +5247,65 @@ export class AnalyticsService {
return data
}
- async getProfilesList(
- pid: string,
+ private buildSessionsListPrimaryEventsSubquery(
filtersQuery: string,
- paramsData: any,
- safeTimezone: string,
- take = 30,
- skip = 0,
- profileType: 'all' | 'anonymous' | 'identified' = 'all',
- customEVFilterApplied = false,
- ): Promise<object[]> {
- let profileTypeFilter = ''
- if (profileType === 'anonymous') {
- profileTypeFilter = `AND profileId LIKE '${AnalyticsService.PROFILE_PREFIX_ANON}%'`
- } else if (profileType === 'identified') {
- profileTypeFilter = `AND profileId LIKE '${AnalyticsService.PROFILE_PREFIX_USER}%'`
- }
+ customEVFilterApplied: boolean,
+ ): string {
+ const scopedSessionFilter = customEVFilterApplied
+ ? `
+ AND CAST(psid, 'String') IN (
+ SELECT DISTINCT CAST(psid, 'String')
+ FROM events
+ WHERE
+ pid = {pid:FixedString(12)}
+ AND type = 'custom_event'
+ AND psid IS NOT NULL
+ AND created BETWEEN {groupFrom:String} AND {groupTo:String}
+ ${filtersQuery}
+ )
+ `
+ : filtersQuery
- let allProfileDataCTE: string
+ return `
+ SELECT
+ CAST(psid, 'String') AS psidCasted,
+ pid,
+ cc,
+ os,
+ br,
+ toTimeZone(created, {timezone:String}) AS created_for_grouping
+ FROM events
+ WHERE
+ pid = {pid:FixedString(12)}
+ AND type IN ('pageview', 'custom_event', 'error')
+ AND psid IS NOT NULL
+ AND created BETWEEN {groupFrom:String} AND {groupTo:String}
+ ${scopedSessionFilter}
+ `
+ }
- if (customEVFilterApplied) {
- allProfileDataCTE = `
- all_profile_data AS (
- SELECT
- profileId,
- psid,
- cc,
- os,
- br,
- dv,
- created,
- 1 AS isPageview,
- 0 AS isEvent
- FROM analytics
- WHERE pid = {pid:FixedString(12)}
- AND created BETWEEN {groupFrom:String} AND {groupTo:String}
- AND profileId IS NOT NULL
- AND profileId != ''
- ${profileTypeFilter}
+ private buildProfilesListDataCTE(
+ filtersQuery: string,
+ profileTypeFilter: string,
+ customEVFilterApplied: boolean,
+ ): string {
+ const scopedProfileFilter = customEVFilterApplied
+ ? `
AND profileId IN (
SELECT DISTINCT profileId
- FROM customEV
+ FROM events
WHERE pid = {pid:FixedString(12)}
+ AND type = 'custom_event'
AND created BETWEEN {groupFrom:String} AND {groupTo:String}
AND profileId IS NOT NULL
AND profileId != ''
${profileTypeFilter}
${filtersQuery}
)
- UNION ALL
- SELECT
- profileId,
- psid,
- cc,
- os,
- br,
- dv,
- created,
- 0 AS isPageview,
- 1 AS isEvent
- FROM customEV
- WHERE pid = {pid:FixedString(12)}
- AND created BETWEEN {groupFrom:String} AND {groupTo:String}
- AND profileId IS NOT NULL
- AND profileId != ''
- ${profileTypeFilter}
- ${filtersQuery}
- )`
- } else {
- allProfileDataCTE = `
+ `
+ : filtersQuery
+
+ return `
all_profile_data AS (
SELECT
profileId,
@@ -5438,36 +5315,43 @@ export class AnalyticsService {
br,
dv,
created,
- 1 AS isPageview,
- 0 AS isEvent
- FROM analytics
+ if(type = 'pageview', 1, 0) AS isPageview,
+ if(type = 'custom_event', 1, 0) AS isEvent,
+ if(type = 'error', 1, 0) AS isError
+ FROM events
WHERE pid = {pid:FixedString(12)}
+ AND type IN ('pageview', 'custom_event', 'error')
AND created BETWEEN {groupFrom:String} AND {groupTo:String}
AND profileId IS NOT NULL
AND profileId != ''
${profileTypeFilter}
- ${filtersQuery}
- UNION ALL
- SELECT
- profileId,
- psid,
- cc,
- os,
- br,
- dv,
- created,
- 0 AS isPageview,
- 1 AS isEvent
- FROM customEV
- WHERE pid = {pid:FixedString(12)}
- AND created BETWEEN {groupFrom:String} AND {groupTo:String}
- AND profileId IS NOT NULL
- AND profileId != ''
- ${profileTypeFilter}
- ${filtersQuery}
+ ${scopedProfileFilter}
)`
+ }
+
+ async getProfilesList(
+ pid: string,
+ filtersQuery: string,
+ paramsData: any,
+ safeTimezone: string,
+ take = 30,
+ skip = 0,
+ profileType: 'all' | 'anonymous' | 'identified' = 'all',
+ customEVFilterApplied = false,
+ ): Promise<object[]> {
+ let profileTypeFilter = ''
+ if (profileType === 'anonymous') {
+ profileTypeFilter = `AND profileId LIKE '${AnalyticsService.PROFILE_PREFIX_ANON}%'`
+ } else if (profileType === 'identified') {
+ profileTypeFilter = `AND profileId LIKE '${AnalyticsService.PROFILE_PREFIX_USER}%'`
}
+ const allProfileDataCTE = this.buildProfilesListDataCTE(
+ filtersQuery,
+ profileTypeFilter,
+ customEVFilterApplied,
+ )
+
const query = `
WITH ${allProfileDataCTE},
profile_aggregated AS (
@@ -5476,6 +5360,7 @@ export class AnalyticsService {
sum(isPageview) AS pageviewsCount,
countDistinct(psid) AS sessionsCount,
sum(isEvent) AS eventsCount,
+ sum(isError) AS errorsCount,
min(created) AS firstSeen,
max(created) AS lastSeen,
any(cc) AS cc_agg,
@@ -5484,25 +5369,13 @@ export class AnalyticsService {
any(dv) AS dv_agg
FROM all_profile_data
GROUP BY profileId
- ),
- profile_errors AS (
- SELECT
- profileId,
- count() AS errorsCount
- FROM errors
- WHERE pid = {pid:FixedString(12)}
- AND errors.created BETWEEN {groupFrom:String} AND {groupTo:String}
- AND profileId IS NOT NULL
- AND profileId != ''
- ${profileTypeFilter}
- GROUP BY profileId
)
SELECT
pa.profileId AS profileId,
pa.sessionsCount AS sessionsCount,
pa.pageviewsCount AS pageviewsCount,
pa.eventsCount AS eventsCount,
- COALESCE(perr.errorsCount, 0) AS errorsCount,
+ pa.errorsCount AS errorsCount,
pa.firstSeen AS firstSeen,
pa.lastSeen AS lastSeen,
pa.cc_agg AS cc,
@@ -5510,7 +5383,6 @@ export class AnalyticsService {
pa.br_agg AS br,
pa.dv_agg AS dv
FROM profile_aggregated AS pa
- LEFT JOIN profile_errors AS perr ON pa.profileId = perr.profileId
ORDER BY pa.lastSeen DESC
LIMIT {take:UInt32}
OFFSET {skip:UInt32}
@@ -5547,40 +5419,59 @@ export class AnalyticsService {
AND profileId = {profileId:String}
`
- // Query avg duration from analytics table (more accurate than sessions table)
+ // Prefer pageview timings, but fall back to sessions for event/error-only profiles.
const queryAvgDuration = `
- SELECT
- avg(session_duration) AS avgDuration
- FROM (
+ WITH pageview_avg AS (
SELECT
- psid,
- dateDiff('second', min(created), max(created)) AS session_duration
- FROM analytics
- WHERE pid = {pid:FixedString(12)}
- AND profileId = {profileId:String}
- AND psid IS NOT NULL
- GROUP BY psid
- HAVING session_duration > 0
+ avgOrNull(session_duration) AS avgDuration
+ FROM (
+ SELECT
+ psid,
+ dateDiff('second', min(created), max(created)) AS session_duration
+ FROM events
+ WHERE pid = {pid:FixedString(12)}
+ AND type = 'pageview'
+ AND profileId = {profileId:String}
+ AND psid IS NOT NULL
+ GROUP BY psid
+ HAVING session_duration > 0
+ )
+ ),
+ sessions_avg AS (
+ SELECT
+ avgOrNull(session_duration) AS avgDuration
+ FROM (
+ SELECT
+ psid,
+ dateDiff('second', min(firstSeen), max(lastSeen)) AS session_duration
+ FROM sessions FINAL
+ WHERE pid = {pid:FixedString(12)}
+ AND profileId = {profileId:String}
+ GROUP BY psid
+ HAVING session_duration > 0
+ )
)
+ SELECT
+ coalesce(nullIf(pageview_avg.avgDuration, 0), sessions_avg.avgDuration, 0) AS avgDuration
+ FROM pageview_avg, sessions_avg
`
- // Query analytics for accurate pageview count
const queryPageviews = `
SELECT count() AS pageviewsCount
- FROM analytics
+ FROM events
WHERE pid = {pid:FixedString(12)}
+ AND type = 'pageview'
AND profileId = {profileId:String}
`
- // Query customEV for accurate events count
const queryEvents = `
SELECT count() AS eventsCount
- FROM customEV
+ FROM events
WHERE pid = {pid:FixedString(12)}
+ AND type = 'custom_event'
AND profileId = {profileId:String}
`
- // Query for device/location details
const queryDetails = `
SELECT
any(cc) AS cc,
@@ -5592,9 +5483,10 @@ export class AnalyticsService {
any(brv) AS brv,
any(dv) AS dv,
any(lc) AS lc
- FROM analytics
+ FROM events
WHERE pid = {pid:FixedString(12)}
AND profileId = {profileId:String}
+ AND type IN ('pageview', 'custom_event', 'error')
`
// Query for total revenue from profile (only sales, not refunds)
@@ -5679,8 +5571,9 @@ export class AnalyticsService {
SELECT
pg AS page,
count() AS count
- FROM analytics
+ FROM events
WHERE pid = {pid:FixedString(12)}
+ AND type = 'pageview'
AND profileId = {profileId:String}
GROUP BY pg
ORDER BY count DESC
@@ -5709,30 +5602,15 @@ export class AnalyticsService {
.format('YYYY-MM-DD')
const query = `
- WITH all_activity AS (
- SELECT
- toDate(created) AS date,
- 1 AS isPageview,
- 0 AS isEvent
- FROM analytics
- WHERE pid = {pid:FixedString(12)}
- AND profileId = {profileId:String}
- AND created >= {startDate:Date}
- UNION ALL
- SELECT
- toDate(created) AS date,
- 0 AS isPageview,
- 1 AS isEvent
- FROM customEV
- WHERE pid = {pid:FixedString(12)}
- AND profileId = {profileId:String}
- AND created >= {startDate:Date}
- )
SELECT
- date,
- sum(isPageview) AS pageviews,
- sum(isEvent) AS events
- FROM all_activity
+ toDate(created) AS date,
+ countIf(type = 'pageview') AS pageviews,
+ countIf(type = 'custom_event') AS events
+ FROM events
+ WHERE pid = {pid:FixedString(12)}
+ AND type IN ('pageview', 'custom_event')
+ AND profileId = {profileId:String}
+ AND created >= {startDate:Date}
GROUP BY date
ORDER BY date ASC
`
@@ -5767,8 +5645,9 @@ export class AnalyticsService {
SELECT
${timeBucketFunc}(toTimeZone(created, {timezone:String})) AS time,
count() AS count
- FROM analytics
+ FROM events
WHERE pid = {pid:FixedString(12)}
+ AND type = 'pageview'
AND profileId = {profileId:String}
AND created BETWEEN {from:String} AND {to:String}
GROUP BY time
@@ -5779,8 +5658,9 @@ export class AnalyticsService {
SELECT
${timeBucketFunc}(toTimeZone(created, {timezone:String})) AS time,
count() AS count
- FROM customEV
+ FROM events
WHERE pid = {pid:FixedString(12)}
+ AND type = 'custom_event'
AND profileId = {profileId:String}
AND created BETWEEN {from:String} AND {to:String}
GROUP BY time
@@ -5791,8 +5671,9 @@ export class AnalyticsService {
SELECT
${timeBucketFunc}(toTimeZone(created, {timezone:String})) AS time,
count() AS count
- FROM errors
+ FROM events
WHERE pid = {pid:FixedString(12)}
+ AND type = 'error'
AND profileId = {profileId:String}
AND created BETWEEN {from:String} AND {to:String}
GROUP BY time
@@ -5853,70 +5734,26 @@ export class AnalyticsService {
}
}
- async getProfileSessionsList(
- pid: string,
- profileId: string,
+ private buildProfileSessionsEventsCTE(
filtersQuery: string,
- paramsData: any,
- safeTimezone: string,
- take = 30,
- skip = 0,
- customEVFilterApplied = false,
- ): Promise<object[]> {
- let allProfileEventsCTE: string
+ customEVFilterApplied: boolean,
+ ): string {
+ const scopedSessionFilter = customEVFilterApplied
+ ? `
+ AND CAST(psid, 'String') IN (
+ SELECT DISTINCT CAST(psid, 'String')
+ FROM events
+ WHERE pid = {pid:FixedString(12)}
+ AND type = 'custom_event'
+ AND profileId = {profileId:String}
+ AND psid IS NOT NULL
+ AND created BETWEEN {groupFrom:String} AND {groupTo:String}
+ ${filtersQuery}
+ )
+ `
+ : filtersQuery
- if (customEVFilterApplied) {
- allProfileEventsCTE = `
- all_profile_events AS (
- SELECT
- all_events.psidCasted,
- all_events.pid,
- all_events.profileId,
- all_events.cc,
- all_events.os,
- all_events.br,
- all_events.created_tz
- FROM (
- SELECT
- CAST(psid, 'String') AS psidCasted,
- pid,
- profileId,
- cc,
- os,
- br,
- toTimeZone(created, {timezone:String}) AS created_tz
- FROM analytics
- WHERE pid = {pid:FixedString(12)}
- AND profileId = {profileId:String}
- AND psid IS NOT NULL
- AND created BETWEEN {groupFrom:String} AND {groupTo:String}
- UNION ALL
- SELECT
- CAST(psid, 'String') AS psidCasted,
- pid,
- profileId,
- cc,
- os,
- br,
- toTimeZone(created, {timezone:String}) AS created_tz
- FROM customEV
- WHERE pid = {pid:FixedString(12)}
- AND profileId = {profileId:String}
- AND psid IS NOT NULL
- AND created BETWEEN {groupFrom:String} AND {groupTo:String}
- ) AS all_events
- WHERE all_events.psidCasted IN (
- SELECT DISTINCT CAST(psid, 'String')
- FROM customEV
- WHERE pid = {pid:FixedString(12)}
- AND profileId = {profileId:String}
- AND psid IS NOT NULL
- AND created BETWEEN {groupFrom:String} AND {groupTo:String}
- ${filtersQuery}
- )
- )`
- } else {
- allProfileEventsCTE = `
+ return `
all_profile_events AS (
SELECT
CAST(psid, 'String') AS psidCasted,
@@ -5926,29 +5763,30 @@ export class AnalyticsService {
os,
br,
toTimeZone(created, {timezone:String}) AS created_tz
- FROM analytics
- WHERE pid = {pid:FixedString(12)}
- AND profileId = {profileId:String}
- AND psid IS NOT NULL
- AND created BETWEEN {groupFrom:String} AND {groupTo:String}
- ${filtersQuery}
- UNION ALL
- SELECT
- CAST(psid, 'String') AS psidCasted,
- pid,
- profileId,
- cc,
- os,
- br,
- toTimeZone(created, {timezone:String}) AS created_tz
- FROM customEV
+ FROM events
WHERE pid = {pid:FixedString(12)}
+ AND type IN ('pageview', 'custom_event', 'error')
AND profileId = {profileId:String}
AND psid IS NOT NULL
AND created BETWEEN {groupFrom:String} AND {groupTo:String}
- ${filtersQuery}
+ ${scopedSessionFilter}
)`
- }
+ }
+
+ async getProfileSessionsList(
+ pid: string,
+ profileId: string,
+ filtersQuery: string,
+ paramsData: any,
+ safeTimezone: string,
+ take = 30,
+ skip = 0,
+ customEVFilterApplied = false,
+ ): Promise<object[]> {
+ const allProfileEventsCTE = this.buildProfileSessionsEventsCTE(
+ filtersQuery,
+ customEVFilterApplied,
+ )
const query = `
WITH ${allProfileEventsCTE},
@@ -5970,8 +5808,9 @@ export class AnalyticsService {
CAST(psid, 'String') AS psidCasted,
pid,
count() as count
- FROM analytics
+ FROM events
WHERE pid = {pid:FixedString(12)}
+ AND type = 'pageview'
AND profileId = {profileId:String}
AND psid IS NOT NULL
AND created BETWEEN {groupFrom:String} AND {groupTo:String}
@@ -5982,8 +5821,9 @@ export class AnalyticsService {
CAST(psid, 'String') AS psidCasted,
pid,
count() as count
- FROM customEV
+ FROM events
WHERE pid = {pid:FixedString(12)}
+ AND type = 'custom_event'
AND profileId = {profileId:String}
AND psid IS NOT NULL
AND created BETWEEN {groupFrom:String} AND {groupTo:String}
@@ -5994,8 +5834,9 @@ export class AnalyticsService {
CAST(psid, 'String') AS psidCasted,
pid,
count() as count
- FROM errors
+ FROM events
WHERE pid = {pid:FixedString(12)}
+ AND type = 'error'
AND profileId = {profileId:String}
AND psid IS NOT NULL
AND created BETWEEN {groupFrom:String} AND {groupTo:String}
@@ -6081,10 +5922,18 @@ export class AnalyticsService {
count(DISTINCT psid) as sessions,
status.status
FROM (
- SELECT eid, name, message, filename, psid, profileId, toTimeZone(errors.created, {timezone:String}) AS created
- FROM errors
+ SELECT
+ eid,
+ error_name AS name,
+ error_message AS message,
+ error_filename AS filename,
+ psid,
+ profileId,
+ toTimeZone(created, {timezone:String}) AS created
+ FROM events
WHERE pid = {pid:FixedString(12)}
- AND errors.created BETWEEN {groupFrom:String} AND {groupTo:String}
+ AND type = 'error'
+ AND created BETWEEN {groupFrom:String} AND {groupTo:String}
${filtersQuery}
) AS errors
LEFT JOIN (
@@ -6098,7 +5947,7 @@ export class AnalyticsService {
${
parsedOptions?.showResolved
? ''
- : "WHERE status.status = 'active' OR status.status = 'regressed'"
+ : "WHERE status.status = 'active' OR status.status = 'regressed' OR status.status IS NULL"
}
GROUP BY errors.eid, status.status
ORDER BY last_seen DESC
@@ -6143,14 +5992,15 @@ export class AnalyticsService {
FROM (
SELECT
eid,
- name,
- message,
- filename,
+ error_name AS name,
+ error_message AS message,
+ error_filename AS filename,
colno,
lineno,
stackTrace
- FROM errors
+ FROM events
WHERE pid = {pid:FixedString(12)}
+ AND type = 'error'
AND eid = {eid:FixedString(32)}
AND created BETWEEN {groupFrom:String} AND {groupTo:String}
ORDER BY created DESC
@@ -6173,10 +6023,12 @@ export class AnalyticsService {
max(created) AS last_seen,
min(created) AS first_seen,
count() AS count
- FROM errors
+ FROM events
WHERE
pid = {pid:FixedString(12)}
- AND eid = {eid:FixedString(32)};
+ AND type = 'error'
+ AND eid = {eid:FixedString(32)}
+ AND created BETWEEN {groupFrom:String} AND {groupTo:String};
`
const queryMetadata = `
@@ -6188,9 +6040,10 @@ export class AnalyticsService {
SELECT
meta.key,
meta.value
- FROM errors
+ FROM events
WHERE
pid = {pid:FixedString(12)}
+ AND type = 'error'
AND eid = {eid:FixedString(32)}
AND created BETWEEN {groupFrom:String} AND {groupTo:String}
)
@@ -6237,8 +6090,8 @@ export class AnalyticsService {
timeBucket,
groupFrom,
groupTo,
- `FROM errors WHERE eid = {eid:FixedString(32)} AND created BETWEEN {groupFrom:String} AND {groupTo:String}`,
- 'AND eid = {eid:FixedString(32)}',
+ `FROM events WHERE type = 'error' AND pid = {pid:FixedString(12)} AND eid = {eid:FixedString(32)} AND created BETWEEN {groupFrom:String} AND {groupTo:String}`,
+ 'AND pid = {pid:FixedString(12)} AND eid = {eid:FixedString(32)}',
paramsData,
safeTimezone,
ChartRenderMode.PERIODICAL,
@@ -6264,17 +6117,23 @@ export class AnalyticsService {
groupTo: string,
timeBucket: string,
showResolved: boolean,
+ sessionFiltersQuery = '',
+ sessionFiltersParams: Record<string, unknown> = {},
): Promise<any> {
const resolvedFilter = showResolved
? ''
: "AND (status.status = 'active' OR status.status = 'regressed' OR status.status IS NULL)"
- // Get total sessions from analytics table for the time range
+ // Get total sessions from all matching events for the time range
const queryTotalSessions = `
SELECT count(DISTINCT psid) as totalSessions
- FROM analytics
+ FROM events
WHERE pid = {pid:FixedString(12)}
+ AND type IN ('pageview', 'custom_event', 'error')
+ AND psid IS NOT NULL
+ AND psid != ''
AND created BETWEEN {groupFrom:String} AND {groupTo:String}
+ ${sessionFiltersQuery}
`
// Get error stats: total errors, unique errors, affected sessions, affected users
@@ -6284,7 +6143,7 @@ export class AnalyticsService {
count(DISTINCT eid) as uniqueErrors,
count(DISTINCT psid) as affectedSessions,
count(DISTINCT profileId) as affectedUsers
- FROM errors
+ FROM events AS errors
LEFT JOIN (
SELECT eid, argMax(status, updated) AS status
FROM error_statuses
@@ -6292,6 +6151,7 @@ export class AnalyticsService {
GROUP BY eid
) AS status ON errors.eid = status.eid
WHERE pid = {pid:FixedString(12)}
+ AND type = 'error'
AND created BETWEEN {groupFrom:String} AND {groupTo:String}
${filtersQuery}
${resolvedFilter}
@@ -6301,12 +6161,12 @@ export class AnalyticsService {
const queryMostFrequentError = `
SELECT
errors.eid as eid,
- any(errors.name) as name,
- any(errors.message) as message,
+ any(errors.error_name) as name,
+ any(errors.error_message) as message,
count(*) as count,
count(DISTINCT errors.profileId) as usersAffected,
max(errors.created) as lastSeen
- FROM errors
+ FROM events AS errors
LEFT JOIN (
SELECT eid, argMax(status, updated) AS status
FROM error_statuses
@@ -6314,6 +6174,7 @@ export class AnalyticsService {
GROUP BY eid
) AS status ON errors.eid = status.eid
WHERE errors.pid = {pid:FixedString(12)}
+ AND errors.type = 'error'
AND errors.created BETWEEN {groupFrom:String} AND {groupTo:String}
${filtersQuery}
${resolvedFilter}
@@ -6338,7 +6199,7 @@ export class AnalyticsService {
SELECT
profileId,
${timeBucketFunc}(toTimeZone(created, {timezone:String})) as tz_created
- FROM errors
+ FROM events AS errors
LEFT JOIN (
SELECT eid, argMax(status, updated) AS status
FROM error_statuses
@@ -6346,6 +6207,7 @@ export class AnalyticsService {
GROUP BY eid
) AS status ON errors.eid = status.eid
WHERE pid = {pid:FixedString(12)}
+ AND type = 'error'
AND created BETWEEN {groupFrom:String} AND {groupTo:String}
${filtersQuery}
${resolvedFilter}
@@ -6368,7 +6230,10 @@ export class AnalyticsService {
clickhouse
.query({
query: queryTotalSessions,
- query_params: paramsData.params,
+ query_params: {
+ ...paramsData.params,
+ ...sessionFiltersParams,
+ },
})
.then((resultSet) => resultSet.json<any>())
.then(({ data }) => data[0]),
@@ -6457,29 +6322,62 @@ export class AnalyticsService {
): Promise<{ sessions: any[]; total: number }> {
const queryCount = `
SELECT count(DISTINCT psid) as total
- FROM errors
+ FROM events
WHERE pid = {pid:FixedString(12)}
+ AND type = 'error'
AND eid = {eid:FixedString(32)}
AND created BETWEEN {groupFrom:String} AND {groupTo:String}
`
const querySessions = `
- SELECT DISTINCT
+ SELECT
CAST(errors.psid, 'String') as psid,
- any(errors.profileId) as profileId,
+ errors.profileId as profileId,
any(analytics.cc) as cc,
any(analytics.br) as br,
any(analytics.os) as os,
- min(errors.created) as firstErrorAt,
- max(errors.created) as lastErrorAt,
- count(*) as errorCount
- FROM errors
- LEFT JOIN analytics ON errors.psid = analytics.psid AND errors.pid = analytics.pid
- WHERE errors.pid = {pid:FixedString(12)}
- AND errors.eid = {eid:FixedString(32)}
- AND errors.created BETWEEN {groupFrom:String} AND {groupTo:String}
- GROUP BY errors.psid
- ORDER BY lastErrorAt DESC
+ errors.firstErrorAt as firstErrorAt,
+ errors.lastErrorAt as lastErrorAt,
+ errors.errorCount as errorCount
+ FROM (
+ SELECT
+ pid,
+ psid,
+ any(profileId) AS profileId,
+ min(created) AS firstErrorAt,
+ max(created) AS lastErrorAt,
+ count(*) AS errorCount
+ FROM events
+ WHERE pid = {pid:FixedString(12)}
+ AND type = 'error'
+ AND eid = {eid:FixedString(32)}
+ AND created BETWEEN {groupFrom:String} AND {groupTo:String}
+ GROUP BY pid, psid
+ ) AS errors
+ LEFT JOIN (
+ SELECT
+ pid,
+ psid,
+ any(cc) AS cc,
+ any(br) AS br,
+ any(os) AS os,
+ countIf(type = 'pageview') AS pageviews
+ FROM events
+ WHERE pid = {pid:FixedString(12)}
+ AND type IN ('pageview', 'error', 'custom_event')
+ AND psid IS NOT NULL
+ AND created BETWEEN {groupFrom:String} AND {groupTo:String}
+ GROUP BY pid, psid
+ ) AS analytics
+ ON errors.psid = analytics.psid
+ AND errors.pid = analytics.pid
+ GROUP BY
+ errors.psid,
+ errors.profileId,
+ errors.firstErrorAt,
+ errors.lastErrorAt,
+ errors.errorCount
+ ORDER BY errors.lastErrorAt DESC
LIMIT {take:UInt32}
OFFSET {skip:UInt32}
`
@@ -6513,6 +6411,10 @@ export class AnalyticsService {
}
async validateEIDs(eids: string[], pid: string) {
+ if (_isEmpty(eids)) {
+ throw new BadRequestException('At least one error ID is required')
+ }
+
const params = _reduce(
eids,
(acc, curr, index) => ({
@@ -6522,7 +6424,7 @@ export class AnalyticsService {
{},
)
- const query = `SELECT count(DISTINCT eid) as count FROM errors WHERE pid = {pid:FixedString(12)} AND eid IN (${_map(
+ const query = `SELECT count(DISTINCT eid) as count FROM events WHERE pid = {pid:FixedString(12)} AND type = 'error' AND eid IN (${_map(
params,
(val, key) => `{${key}:FixedString(32)}`,
).join(', ')})`
@@ -6604,15 +6506,7 @@ export class AnalyticsService {
events: number
}> {
const query = `
- SELECT 'traffic' AS type, count(*) AS count FROM analytics
- UNION ALL
- SELECT 'customEV' AS type, count(*) AS count FROM customEV
- UNION ALL
- SELECT 'performance' AS type, count(*) AS count FROM performance
- UNION ALL
- SELECT 'captcha' AS type, count(*) AS count FROM captcha
- UNION ALL
- SELECT 'errors' AS type, count(*) AS count FROM errors
+ SELECT count(*) AS count FROM events
`
const users = await this.userService.count({
@@ -6732,12 +6626,13 @@ export class AnalyticsService {
key,
round(sum(${aggFunction}), 2) AS sum,
round(avg(${aggFunction}), 2) AS avg
- FROM customEV
+ FROM events
ARRAY JOIN meta.key AS key, meta.value AS value
WHERE
pid = {pid:FixedString(12)}
+ AND type = 'custom_event'
AND key = {metricKey:String}
- AND ev = {customEventName:String}
+ AND event_name = {customEventName:String}
AND created BETWEEN {periodFormatted:String} AND {now:String}
${kvQuery}
${filtersQuery}
@@ -6750,12 +6645,13 @@ export class AnalyticsService {
key,
round(sum(${aggFunction}), 2) AS sum,
round(avg(${aggFunction}), 2) AS avg
- FROM customEV
+ FROM events
ARRAY JOIN meta.key AS key, meta.value AS value
WHERE
pid = {pid:FixedString(12)}
+ AND type = 'custom_event'
AND key = {metricKey:String}
- AND ev = {customEventName:String}
+ AND event_name = {customEventName:String}
AND created BETWEEN {periodSubtracted:String} AND {periodFormatted:String}
${kvQuery}
${filtersQuery}
diff --git a/backend/apps/cloud/src/analytics/utils/transformers.ts b/backend/apps/cloud/src/analytics/utils/transformers.ts
index b420cdac8..7ec1ed491 100644
--- a/backend/apps/cloud/src/analytics/utils/transformers.ts
+++ b/backend/apps/cloud/src/analytics/utils/transformers.ts
@@ -8,7 +8,7 @@ import utc from 'dayjs/plugin/utc'
dayjs.extend(utc)
const processMetaKV = (
- rawMeta: Record<string, string>,
+ rawMeta: Record<string, string> | null | undefined,
): { 'meta.key': string[] | null; 'meta.value': string[] | null } => {
if (!rawMeta || _isEmpty(rawMeta)) {
return {
@@ -23,10 +23,10 @@ const processMetaKV = (
}
}
-interface TrafficTransformerOptions {
- psid: string
- profileId: string
+interface CommonOptions {
pid: string
+ psid?: string | null
+ profileId?: string | null
host?: string | null
pg?: string | null
dv?: string | null
@@ -52,262 +52,42 @@ interface TrafficTransformerOptions {
meta?: Record<string, string> | null
}
-export const trafficTransformer = ({
- psid,
- profileId,
- pid,
- host,
- pg,
- dv,
- br,
- brv,
- os,
- osv,
- lc,
- ref,
- so,
- me,
- ca,
- te,
- co,
- cc,
- rg,
- rgc,
- ct,
- isp,
- og,
- ut,
- ctp,
- meta,
-}: TrafficTransformerOptions) => {
- return {
- psid,
- profileId,
- pid,
- host: host || null,
- pg: pg || null,
- dv: dv || null,
- br: br || null,
- brv: brv || null,
- os: os || null,
- osv: osv || null,
- lc: lc || null,
- ref: ref || null,
- so: so || null,
- me: me || null,
- ca: ca || null,
- te: te || null,
- co: co || null,
- cc: cc || null,
- rg: rg || null,
- rgc: rgc || null,
- ct: ct || null,
- isp: isp || null,
- og: og || null,
- ut: ut || null,
- ctp: ctp || null,
- ...processMetaKV(meta),
- created: dayjs.utc().format('YYYY-MM-DD HH:mm:ss'),
- }
+interface PageviewOptions extends CommonOptions {
+ type: 'pageview'
}
-interface CustomEventTransformerOptions {
- psid: string
- profileId: string
- pid: string
- host?: string | null
+interface CustomEventOptions extends CommonOptions {
+ type: 'custom_event'
ev: string
- pg?: string | null
- dv?: string | null
- br?: string | null
- brv?: string | null
- os?: string | null
- osv?: string | null
- lc?: string | null
- ref?: string | null
- so?: string | null
- me?: string | null
- ca?: string | null
- te?: string | null
- co?: string | null
- cc?: string | null
- rg?: string | null
- rgc?: string | null
- ct?: string | null
- isp?: string | null
- og?: string | null
- ut?: string | null
- ctp?: string | null
- meta?: Record<string, string> | null
-}
-
-export const customEventTransformer = ({
- psid,
- profileId,
- pid,
- host,
- ev,
- pg,
- dv,
- br,
- brv,
- os,
- osv,
- lc,
- ref,
- so,
- me,
- ca,
- te,
- co,
- cc,
- rg,
- rgc,
- ct,
- isp,
- og,
- ut,
- ctp,
- meta,
-}: CustomEventTransformerOptions) => {
- return {
- psid,
- profileId,
- pid,
- host: host || null,
- ev,
- pg: pg || null,
- dv: dv || null,
- br: br || null,
- brv: brv || null,
- os: os || null,
- osv: osv || null,
- lc: lc || null,
- ref: ref || null,
- so: so || null,
- me: me || null,
- ca: ca || null,
- te: te || null,
- co: co || null,
- cc: cc || null,
- rg: rg || null,
- rgc: rgc || null,
- ct: ct || null,
- isp: isp || null,
- og: og || null,
- ut: ut || null,
- ctp: ctp || null,
- ...processMetaKV(meta),
- created: dayjs.utc().format('YYYY-MM-DD HH:mm:ss'),
- }
}
-interface ErrorEventTransformerOptions {
- psid: string
- profileId: string
+interface ErrorEventOptions extends CommonOptions {
+ type: 'error'
eid: string
- pid: string
- host?: string | null
- pg?: string | null
- dv?: string | null
- br?: string | null
- brv?: string | null
- os?: string | null
- osv?: string | null
- lc?: string | null
- cc?: string | null
- rg?: string | null
- rgc?: string | null
- ct?: string | null
- isp?: string | null
- og?: string | null
- ut?: string | null
- ctp?: string | null
name?: string | null
message?: string | null
lineno?: number | null
colno?: number | null
filename?: string | null
stackTrace?: string | null
- meta?: Record<string, string> | null
-}
-
-export const errorEventTransformer = ({
- psid,
- profileId,
- eid,
- pid,
- host,
- pg,
- dv,
- br,
- brv,
- os,
- osv,
- lc,
- cc,
- rg,
- rgc,
- ct,
- isp,
- og,
- ut,
- ctp,
- name,
- message,
- lineno,
- colno,
- filename,
- stackTrace,
- meta,
-}: ErrorEventTransformerOptions) => {
- return {
- psid,
- profileId,
- eid,
- pid,
- host: host || null,
- pg: pg || null,
- dv: dv || null,
- br: br || null,
- brv: brv || null,
- os: os || null,
- osv: osv || null,
- lc: lc || null,
- cc: cc || null,
- rg: rg || null,
- rgc: rgc || null,
- ct: ct || null,
- isp: isp || null,
- og: og || null,
- ut: ut || null,
- ctp: ctp || null,
- name: name || null,
- message: message || null,
- lineno: lineno || null,
- colno: colno || null,
- filename: filename || null,
- stackTrace: stackTrace || null,
- ...processMetaKV(meta),
- created: dayjs.utc().format('YYYY-MM-DD HH:mm:ss'),
- }
}
-interface PerformanceTransformerOptions {
- pid: string
- host?: string | null
- pg?: string | null
- dv?: string | null
- br?: string | null
- brv?: string | null
- cc?: string | null
- rg?: string | null
- rgc?: string | null
- ct?: string | null
- isp?: string | null
- og?: string | null
- ut?: string | null
- ctp?: string | null
+interface PerformanceOptions extends Omit<
+ CommonOptions,
+ | 'psid'
+ | 'profileId'
+ | 'os'
+ | 'osv'
+ | 'lc'
+ | 'ref'
+ | 'so'
+ | 'me'
+ | 'ca'
+ | 'te'
+ | 'co'
+ | 'meta'
+> {
+ type: 'performance'
dns: number
tls: number
conn: number
@@ -318,53 +98,131 @@ interface PerformanceTransformerOptions {
ttfb: number
}
-export const performanceTransformer = ({
- pid,
- host,
- pg,
- dv,
- br,
- brv,
- cc,
- rg,
- rgc,
- ct,
- isp,
- og,
- ut,
- ctp,
- dns,
- tls,
- conn,
- response,
- render,
- domLoad,
- pageLoad,
- ttfb,
-}: PerformanceTransformerOptions) => {
+interface CaptchaOptions {
+ type: 'captcha'
+ pid: string
+ dv?: string | null
+ br?: string | null
+ os?: string | null
+ cc?: string | null
+ timestamp?: number
+}
+
+type EventTransformerOptions =
+ | PageviewOptions
+ | CustomEventOptions
+ | ErrorEventOptions
+ | PerformanceOptions
+ | CaptchaOptions
+
+const buildCommon = (opts: CommonOptions) => ({
+ pid: opts.pid,
+ psid: opts.psid ?? null,
+ profileId: opts.profileId ?? null,
+ host: opts.host || null,
+ pg: opts.pg || null,
+ dv: opts.dv || null,
+ br: opts.br || null,
+ brv: opts.brv || null,
+ os: opts.os || null,
+ osv: opts.osv || null,
+ lc: opts.lc || null,
+ ref: opts.ref || null,
+ so: opts.so || null,
+ me: opts.me || null,
+ ca: opts.ca || null,
+ te: opts.te || null,
+ co: opts.co || null,
+ cc: opts.cc || null,
+ rg: opts.rg || null,
+ rgc: opts.rgc || null,
+ ct: opts.ct || null,
+ isp: opts.isp || null,
+ og: opts.og || null,
+ ut: opts.ut || null,
+ ctp: opts.ctp || null,
+ ...processMetaKV(opts.meta),
+})
+
+// Single transformer for all event kinds; the discriminator is `type`.
+// Renames at the storage boundary: ev -> event_name, name -> error_name,
+// message -> error_message, filename -> error_filename. DTO and API field
+// names stay as the legacy short forms.
+export const eventTransformer = (opts: EventTransformerOptions) => {
+ const created = dayjs.utc().format('YYYY-MM-DD HH:mm:ss')
+
+ if (opts.type === 'pageview') {
+ return {
+ type: 'pageview' as const,
+ ...buildCommon(opts),
+ created,
+ }
+ }
+
+ if (opts.type === 'custom_event') {
+ return {
+ type: 'custom_event' as const,
+ ...buildCommon(opts),
+ event_name: opts.ev,
+ created,
+ }
+ }
+
+ if (opts.type === 'error') {
+ return {
+ type: 'error' as const,
+ ...buildCommon(opts),
+ eid: opts.eid,
+ error_name: opts.name || null,
+ error_message: opts.message || null,
+ stackTrace: opts.stackTrace || null,
+ lineno: opts.lineno ?? null,
+ colno: opts.colno ?? null,
+ error_filename: opts.filename || null,
+ created,
+ }
+ }
+
+ if (opts.type === 'performance') {
+ return {
+ type: 'performance' as const,
+ pid: opts.pid,
+ host: opts.host || null,
+ pg: opts.pg || null,
+ dv: opts.dv || null,
+ br: opts.br || null,
+ brv: opts.brv || null,
+ cc: opts.cc || null,
+ rg: opts.rg || null,
+ rgc: opts.rgc || null,
+ ct: opts.ct || null,
+ isp: opts.isp || null,
+ og: opts.og || null,
+ ut: opts.ut || null,
+ ctp: opts.ctp || null,
+ dns: _round(opts.dns),
+ tls: _round(opts.tls),
+ conn: _round(opts.conn),
+ response: _round(opts.response),
+ render: _round(opts.render),
+ domLoad: _round(opts.domLoad),
+ pageLoad: _round(opts.pageLoad),
+ ttfb: _round(opts.ttfb),
+ created,
+ }
+ }
+
+ // captcha
return {
- pid,
- host: host || null,
- pg: pg || null,
- dv: dv || null,
- br: br || null,
- brv: brv || null,
- cc: cc || null,
- rg: rg || null,
- rgc: rgc || null,
- ct: ct || null,
- isp: isp || null,
- og: og || null,
- ut: ut || null,
- ctp: ctp || null,
- dns: _round(dns),
- tls: _round(tls),
- conn: _round(conn),
- response: _round(response),
- render: _round(render),
- domLoad: _round(domLoad),
- pageLoad: _round(pageLoad),
- ttfb: _round(ttfb),
- created: dayjs.utc().format('YYYY-MM-DD HH:mm:ss'),
+ type: 'captcha' as const,
+ pid: opts.pid,
+ dv: opts.dv || null,
+ br: opts.br || null,
+ os: opts.os || null,
+ cc: opts.cc || null,
+ created:
+ typeof opts.timestamp === 'number'
+ ? dayjs.utc(opts.timestamp).format('YYYY-MM-DD HH:mm:ss')
+ : created,
}
}
diff --git a/backend/apps/cloud/src/captcha/captcha.service.ts b/backend/apps/cloud/src/captcha/captcha.service.ts
index 921e6d7b7..99ac3516c 100644
--- a/backend/apps/cloud/src/captcha/captcha.service.ts
+++ b/backend/apps/cloud/src/captcha/captcha.service.ts
@@ -17,7 +17,7 @@ import {
} from '../common/constants'
import { getIPDetails } from '../common/utils'
import { GeneratedChallenge } from './interfaces/generated-captcha'
-import { captchaTransformer } from './utils/transformers'
+import { eventTransformer } from '../analytics/utils/transformers'
import { clickhouse } from '../common/integrations/clickhouse'
dayjs.extend(utc)
@@ -152,18 +152,19 @@ export class CaptchaService {
const osName = ua.os.name
const { country } = getIPDetails(ip)
- const transformed = captchaTransformer(
+ const transformed = eventTransformer({
+ type: 'captcha',
pid,
- deviceType,
- browserName,
- osName,
- country,
+ dv: deviceType,
+ br: browserName,
+ os: osName,
+ cc: country,
timestamp,
- )
+ })
try {
await clickhouse.insert({
- table: 'captcha',
+ table: 'events',
format: 'JSONEachRow',
values: [transformed],
clickhouse_settings: {
diff --git a/backend/apps/cloud/src/captcha/utils/transformers.ts b/backend/apps/cloud/src/captcha/utils/transformers.ts
deleted file mode 100644
index 3a0d21d43..000000000
--- a/backend/apps/cloud/src/captcha/utils/transformers.ts
+++ /dev/null
@@ -1,22 +0,0 @@
-import dayjs from 'dayjs'
-import utc from 'dayjs/plugin/utc'
-
-dayjs.extend(utc)
-
-export const captchaTransformer = (
- pid: string,
- dv: string | null,
- br: string | null,
- os: string | null,
- cc: string | null,
- timestamp: number,
-) => {
- return {
- pid,
- dv: dv || null,
- br: br || null,
- os: os || null,
- cc: cc || null,
- created: dayjs.utc(timestamp).format('YYYY-MM-DD HH:mm:ss'),
- }
-}
diff --git a/backend/apps/cloud/src/data-import/data-import.processor.ts b/backend/apps/cloud/src/data-import/data-import.processor.ts
index 18799910d..019945a24 100644
--- a/backend/apps/cloud/src/data-import/data-import.processor.ts
+++ b/backend/apps/cloud/src/data-import/data-import.processor.ts
@@ -107,8 +107,7 @@ export class DataImportProcessor extends WorkerHost {
let minDate: string | null = null
let maxDate: string | null = null
- const analyticsBatch: Record<string, unknown>[] = []
- const customEVBatch: Record<string, unknown>[] = []
+ const eventsBatch: Record<string, unknown>[] = []
const context = isApiBased
? {
@@ -142,28 +141,17 @@ export class DataImportProcessor extends WorkerHost {
if (!maxDate || created > maxDate) maxDate = created
}
- if (row.table === 'analytics') {
- analyticsBatch.push(row.data)
- } else {
- customEVBatch.push(row.data)
- }
-
- if (analyticsBatch.length >= BATCH_SIZE) {
- await this.flushBatch('analytics', analyticsBatch)
- importedRows += analyticsBatch.length
- analyticsBatch.length = 0
- }
+ eventsBatch.push({ type: row.type, ...row.data })
- if (customEVBatch.length >= BATCH_SIZE) {
- await this.flushBatch('customEV', customEVBatch)
- importedRows += customEVBatch.length
- customEVBatch.length = 0
+ if (eventsBatch.length >= BATCH_SIZE) {
+ await this.flushBatch(eventsBatch)
+ importedRows += eventsBatch.length
+ eventsBatch.length = 0
}
if (totalRows % 10000 === 0) {
const progress = {
- importedRows:
- importedRows + analyticsBatch.length + customEVBatch.length,
+ importedRows: importedRows + eventsBatch.length,
totalRows,
}
@@ -181,14 +169,9 @@ export class DataImportProcessor extends WorkerHost {
}
}
- if (analyticsBatch.length > 0) {
- await this.flushBatch('analytics', analyticsBatch)
- importedRows += analyticsBatch.length
- }
-
- if (customEVBatch.length > 0) {
- await this.flushBatch('customEV', customEVBatch)
- importedRows += customEVBatch.length
+ if (eventsBatch.length > 0) {
+ await this.flushBatch(eventsBatch)
+ importedRows += eventsBatch.length
}
await this.dataImportService.markCompleted(id, {
@@ -235,12 +218,9 @@ export class DataImportProcessor extends WorkerHost {
}
}
- private async flushBatch(
- table: 'analytics' | 'customEV',
- batch: Record<string, unknown>[],
- ): Promise<void> {
+ private async flushBatch(batch: Record<string, unknown>[]): Promise<void> {
await clickhouse.insert({
- table: `${CLICKHOUSE_DB}.${table}`,
+ table: `${CLICKHOUSE_DB}.events`,
values: batch,
format: 'JSONEachRow',
})
diff --git a/backend/apps/cloud/src/data-import/data-import.service.ts b/backend/apps/cloud/src/data-import/data-import.service.ts
index 63f60b59e..834b53086 100644
--- a/backend/apps/cloud/src/data-import/data-import.service.ts
+++ b/backend/apps/cloud/src/data-import/data-import.service.ts
@@ -161,11 +161,7 @@ export class DataImportService {
importId: number,
): Promise<void> {
await clickhouse.command({
- query: `ALTER TABLE ${CLICKHOUSE_DB}.analytics DELETE WHERE pid = {pid:FixedString(12)} AND importID = {importID:UInt8}`,
- query_params: { pid: projectId, importID: importId },
- })
- await clickhouse.command({
- query: `ALTER TABLE ${CLICKHOUSE_DB}.customEV DELETE WHERE pid = {pid:FixedString(12)} AND importID = {importID:UInt8}`,
+ query: `ALTER TABLE ${CLICKHOUSE_DB}.events DELETE WHERE pid = {pid:FixedString(12)} AND importID = {importID:UInt8} AND type IN ('pageview', 'custom_event')`,
query_params: { pid: projectId, importID: importId },
})
}
@@ -207,7 +203,7 @@ export class DataImportService {
): Promise<boolean> {
const result = await clickhouse
.query({
- query: `SELECT count() as cnt FROM ${CLICKHOUSE_DB}.analytics WHERE pid = {pid:FixedString(12)} AND importID IS NOT NULL AND created >= {from:DateTime} AND created <= {to:DateTime} LIMIT 1`,
+ query: `SELECT count() as cnt FROM ${CLICKHOUSE_DB}.events WHERE pid = {pid:FixedString(12)} AND type = 'pageview' AND importID IS NOT NULL AND created >= {from:DateTime} AND created <= {to:DateTime} LIMIT 1`,
query_params: { pid: projectId, from, to },
})
.then((resultSet) => resultSet.json<{ cnt: string }>())
diff --git a/backend/apps/cloud/src/data-import/mappers/fathom.mapper.ts b/backend/apps/cloud/src/data-import/mappers/fathom.mapper.ts
index f74c72122..b9dce040d 100644
--- a/backend/apps/cloud/src/data-import/mappers/fathom.mapper.ts
+++ b/backend/apps/cloud/src/data-import/mappers/fathom.mapper.ts
@@ -206,13 +206,13 @@ export class FathomMapper implements ImportMapper {
if (fileType === 'pageviews') {
for (let i = 0; i < count; i++) {
- yield { table: 'analytics', data: baseData }
+ yield { type: 'pageview', data: baseData }
}
} else {
const eventName = truncate(normalizeNull(row.event_name), 256) || ''
- const evData = { ...baseData, ev: eventName }
+ const evData = { ...baseData, event_name: eventName }
for (let i = 0; i < count; i++) {
- yield { table: 'customEV', data: evData }
+ yield { type: 'custom_event', data: evData }
}
}
}
diff --git a/backend/apps/cloud/src/data-import/mappers/ga4.mapper.ts b/backend/apps/cloud/src/data-import/mappers/ga4.mapper.ts
index 5cd625c58..2ee657aa9 100644
--- a/backend/apps/cloud/src/data-import/mappers/ga4.mapper.ts
+++ b/backend/apps/cloud/src/data-import/mappers/ga4.mapper.ts
@@ -357,7 +357,7 @@ export class Ga4Mapper implements ImportMapper {
}
for (let i = 0; i < count; i++) {
- yield { table: 'analytics', data }
+ yield { type: 'pageview', data }
}
}
@@ -441,7 +441,7 @@ export class Ga4Mapper implements ImportMapper {
pid,
host: truncate(host, 253),
pg: truncate(pagePath, 2048),
- ev: truncate(eventName, 256),
+ event_name: truncate(eventName, 256),
dv: device,
br: truncate(browser, 30),
brv: null,
@@ -465,7 +465,7 @@ export class Ga4Mapper implements ImportMapper {
}
for (let i = 0; i < count; i++) {
- yield { table: 'customEV', data }
+ yield { type: 'custom_event', data }
}
}
diff --git a/backend/apps/cloud/src/data-import/mappers/mapper.interface.ts b/backend/apps/cloud/src/data-import/mappers/mapper.interface.ts
index 0351b00b0..cf19c69c8 100644
--- a/backend/apps/cloud/src/data-import/mappers/mapper.interface.ts
+++ b/backend/apps/cloud/src/data-import/mappers/mapper.interface.ts
@@ -6,7 +6,7 @@ export class ImportError extends Error {
}
export interface AnalyticsImportRow {
- table: 'analytics' | 'customEV'
+ type: 'pageview' | 'custom_event'
data: Record<string, unknown>
}
diff --git a/backend/apps/cloud/src/data-import/mappers/plausible.mapper.ts b/backend/apps/cloud/src/data-import/mappers/plausible.mapper.ts
index 99141bb58..f0c5926d3 100644
--- a/backend/apps/cloud/src/data-import/mappers/plausible.mapper.ts
+++ b/backend/apps/cloud/src/data-import/mappers/plausible.mapper.ts
@@ -859,7 +859,7 @@ export class PlausibleMapper implements ImportMapper {
created,
})
- yield { table: 'analytics', data }
+ yield { type: 'pageview', data }
}
}
@@ -889,8 +889,8 @@ export class PlausibleMapper implements ImportMapper {
page: null,
created,
})
- data.ev = truncate(ev.name, 256) || ''
- yield { table: 'customEV', data }
+ data.event_name = truncate(ev.name, 256) || ''
+ yield { type: 'custom_event', data }
}
}
}
diff --git a/backend/apps/cloud/src/data-import/mappers/simple-analytics.mapper.ts b/backend/apps/cloud/src/data-import/mappers/simple-analytics.mapper.ts
index 880b863b9..b95e37380 100644
--- a/backend/apps/cloud/src/data-import/mappers/simple-analytics.mapper.ts
+++ b/backend/apps/cloud/src/data-import/mappers/simple-analytics.mapper.ts
@@ -176,11 +176,11 @@ export class SimpleAnalyticsMapper implements ImportMapper {
const datapoint = normalizeNull(row.datapoint)
if (!datapoint || datapoint === 'pageview') {
- yield { table: 'analytics', data: baseData }
+ yield { type: 'pageview', data: baseData }
} else {
yield {
- table: 'customEV',
- data: { ...baseData, ev: truncate(datapoint, 256) || '' },
+ type: 'custom_event',
+ data: { ...baseData, event_name: truncate(datapoint, 256) || '' },
}
}
}
diff --git a/backend/apps/cloud/src/data-import/mappers/umami.mapper.ts b/backend/apps/cloud/src/data-import/mappers/umami.mapper.ts
index 192a9638b..3d8c9d9b8 100644
--- a/backend/apps/cloud/src/data-import/mappers/umami.mapper.ts
+++ b/backend/apps/cloud/src/data-import/mappers/umami.mapper.ts
@@ -372,12 +372,12 @@ export class UmamiMapper implements ImportMapper {
}
if (eventType === '1') {
- yield { table: 'analytics', data: baseData }
+ yield { type: 'pageview', data: baseData }
} else {
const eventName = truncate(normalizeNull(row.event_name), 256) || ''
yield {
- table: 'customEV',
- data: { ...baseData, ev: eventName },
+ type: 'custom_event',
+ data: { ...baseData, event_name: eventName },
}
}
}
diff --git a/backend/apps/cloud/src/experiment/experiment.controller.ts b/backend/apps/cloud/src/experiment/experiment.controller.ts
index 52f230267..869176359 100644
--- a/backend/apps/cloud/src/experiment/experiment.controller.ts
+++ b/backend/apps/cloud/src/experiment/experiment.controller.ts
@@ -60,6 +60,7 @@ import {
} from './dto/experiment.dto'
import { ExperimentService } from './experiment.service'
import { GoalService } from '../goal/goal.service'
+import { Goal, GoalMatchType, GoalType } from '../goal/entity/goal.entity'
import { FeatureFlagService } from '../feature-flag/feature-flag.service'
import {
FeatureFlag,
@@ -73,6 +74,15 @@ import { Pagination } from '../common/pagination'
const EXPERIMENTS_MAXIMUM = 20 // Maximum experiments per project
const FEATURE_FLAG_KEY_REGEX = /^[a-zA-Z0-9_-]+$/
+type GoalEventConditions = {
+ eventType: 'pageview' | 'custom_event'
+ matchColumn: 'pg' | 'event_name'
+ matchCondition: string
+ metaCondition: string
+ metaParams: Record<string, string>
+ goalValue: string
+}
+
@ApiTags('Experiment')
@Controller(['experiment', 'v1/experiment'])
export class ExperimentController {
@@ -881,11 +891,14 @@ export class ExperimentController {
timeBucketParam || getLowestPossibleTimeBucket(period, from, to)
let allowedTimeBucketForPeriodAll: TimeBucketType[] | undefined
let diff: number | undefined
+ const goalConditions = experiment.goal
+ ? this.buildGoalEventConditions(experiment.goal)
+ : null
if (period === 'all') {
const res = await this.analyticsService.calculateTimeBucketForAllTime(
experiment.project.id,
- 'analytics',
+ goalConditions?.eventType || 'pageview',
)
diff = res.diff
@@ -938,43 +951,21 @@ export class ExperimentController {
}
let conversionsData: { variantKey: string; conversions: number }[] = []
- if (experiment.goal) {
- const goalType = experiment.goal.type
- const table = goalType === 'custom_event' ? 'customEV' : 'analytics'
- const matchColumn = goalType === 'custom_event' ? 'ev' : 'pg'
- const goalValue = experiment.goal.value || ''
-
- let matchCondition = ''
- if (experiment.goal.matchType === 'exact') {
- matchCondition = `c.${matchColumn} = {goalValue:String}`
- } else {
- matchCondition = `c.${matchColumn} ILIKE concat('%', {goalValue:String}, '%')`
- }
-
- let metaCondition = ''
- const metaParams: Record<string, string> = {}
- const metadataFilters = experiment.goal.metadataFilters
- if (metadataFilters && metadataFilters.length > 0) {
- const conditions: string[] = []
- metadataFilters.forEach((filter, index) => {
- const keyParam = `metaKey${index}`
- const valueParam = `metaValue${index}`
- metaParams[keyParam] = filter.key
- metaParams[valueParam] = filter.value
- conditions.push(
- `has(c.meta.key, {${keyParam}:String}) AND c.meta.value[indexOf(c.meta.key, {${keyParam}:String})] = {${valueParam}:String}`,
- )
- })
-
- metaCondition = `AND (${conditions.join(' AND ')})`
- }
+ if (goalConditions) {
+ const {
+ eventType,
+ matchCondition,
+ metaCondition,
+ metaParams,
+ goalValue,
+ } = goalConditions
const conversionsQuery = `
SELECT
e.variantKey,
uniqExact(e.profileId) as conversions
FROM experiment_exposures e
- INNER JOIN ${table} c ON e.pid = c.pid AND e.profileId = assumeNotNull(c.profileId)
+ INNER JOIN events c ON e.pid = c.pid AND e.profileId = assumeNotNull(c.profileId) AND c.type = '${eventType}'
WHERE
e.pid = {pid:FixedString(12)}
AND e.experimentId = {experimentId:String}
@@ -1105,6 +1096,50 @@ export class ExperimentController {
}
}
+ private buildGoalEventConditions(goal: Goal): GoalEventConditions {
+ const eventType =
+ goal.type === GoalType.CUSTOM_EVENT ? 'custom_event' : 'pageview'
+ const matchColumn =
+ goal.type === GoalType.CUSTOM_EVENT ? 'event_name' : 'pg'
+ const goalValue = goal.value || ''
+ const metaParams: Record<string, string> = {}
+
+ let matchCondition = ''
+ if (goal.matchType === GoalMatchType.EXACT) {
+ matchCondition = `c.${matchColumn} = {goalValue:String}`
+ } else if (goalValue.trim() === '') {
+ matchCondition = '1=0'
+ } else {
+ matchCondition = `c.${matchColumn} ILIKE concat('%', {goalValue:String}, '%')`
+ }
+
+ let metaCondition = ''
+ if (goal.metadataFilters && goal.metadataFilters.length > 0) {
+ const conditions: string[] = []
+
+ goal.metadataFilters.forEach((filter, index) => {
+ const keyParam = `metaKey${index}`
+ const valueParam = `metaValue${index}`
+ metaParams[keyParam] = filter.key
+ metaParams[valueParam] = filter.value
+ conditions.push(
+ `has(c.meta.key, {${keyParam}:String}) AND c.meta.value[indexOf(c.meta.key, {${keyParam}:String})] = {${valueParam}:String}`,
+ )
+ })
+
+ metaCondition = `AND (${conditions.join(' AND ')})`
+ }
+
+ return {
+ eventType,
+ matchColumn,
+ matchCondition,
+ metaCondition,
+ metaParams,
+ goalValue,
+ }
+ }
+
/**
* Generate time-series chart data for experiment win probabilities
*/
@@ -1177,39 +1212,18 @@ export class ExperimentController {
let conversionsData: any[] = []
if (experiment.goal) {
- const goalType = experiment.goal.type
- const table = goalType === 'custom_event' ? 'customEV' : 'analytics'
- const matchColumn = goalType === 'custom_event' ? 'ev' : 'pg'
- const goalValue = experiment.goal.value || ''
+ const {
+ eventType,
+ matchCondition,
+ metaCondition,
+ metaParams,
+ goalValue,
+ } = this.buildGoalEventConditions(experiment.goal)
const conversionsDateColumnsSelect = this.getTimeBucketDateColumnsSelect(
timeBucket,
'firstConversion',
)
- let matchCondition = ''
- if (experiment.goal.matchType === 'exact') {
- matchCondition = `c.${matchColumn} = {goalValue:String}`
- } else {
- matchCondition = `c.${matchColumn} ILIKE concat('%', {goalValue:String}, '%')`
- }
-
- let metaCondition = ''
- const metaParams: Record<string, string> = {}
- const metadataFilters = experiment.goal.metadataFilters
- if (metadataFilters && metadataFilters.length > 0) {
- const conditions: string[] = []
- metadataFilters.forEach((filter, index) => {
- const keyParam = `metaKey${index}`
- const valueParam = `metaValue${index}`
- metaParams[keyParam] = filter.key
- metaParams[valueParam] = filter.value
- conditions.push(
- `has(c.meta.key, {${keyParam}:String}) AND c.meta.value[indexOf(c.meta.key, {${keyParam}:String})] = {${valueParam}:String}`,
- )
- })
- metaCondition = `AND (${conditions.join(' AND ')})`
- }
-
const conversionsQuery = `
SELECT
${conversionsDateColumnsSelect},
@@ -1221,7 +1235,7 @@ export class ExperimentController {
e.profileId as profileId,
min(c.created) as firstConversion
FROM experiment_exposures e
- INNER JOIN ${table} c ON e.pid = c.pid AND e.profileId = assumeNotNull(c.profileId)
+ INNER JOIN events c ON e.pid = c.pid AND e.profileId = assumeNotNull(c.profileId) AND c.type = '${eventType}'
WHERE
e.pid = {pid:FixedString(12)}
AND e.experimentId = {experimentId:String}
diff --git a/backend/apps/cloud/src/goal/goal.controller.ts b/backend/apps/cloud/src/goal/goal.controller.ts
index 657554968..ed2807bee 100644
--- a/backend/apps/cloud/src/goal/goal.controller.ts
+++ b/backend/apps/cloud/src/goal/goal.controller.ts
@@ -332,38 +332,38 @@ export class GoalController {
})
}
- private buildGoalMatchCondition(
- goal: Goal,
- _table: 'analytics' | 'customEV',
- ): { condition: string; params: Record<string, string> } {
+ private buildGoalMatchCondition(goal: Goal): {
+ condition: string
+ params: Record<string, string>
+ } {
const params: Record<string, string> = {}
if (goal.type === GoalType.CUSTOM_EVENT) {
- // For custom events, match the event name
if (goal.matchType === GoalMatchType.EXACT) {
params.goalValue = goal.value || ''
- return { condition: `ev = {goalValue:String}`, params }
- } else {
- // Contains match
- params.goalValue = goal.value || ''
- return {
- condition: `ev ILIKE concat('%', {goalValue:String}, '%')`,
- params,
- }
+ return { condition: `event_name = {goalValue:String}`, params }
}
- } else {
- // For pageview goals, match the page path
- if (goal.matchType === GoalMatchType.EXACT) {
- params.goalValue = goal.value || ''
- return { condition: `pg = {goalValue:String}`, params }
- } else {
- // Contains match
- params.goalValue = goal.value || ''
- return {
- condition: `pg ILIKE concat('%', {goalValue:String}, '%')`,
- params,
- }
+ if ((goal.value || '').trim() === '') {
+ return { condition: '1=0', params: {} }
}
+ params.goalValue = goal.value || ''
+ return {
+ condition: `event_name ILIKE concat('%', {goalValue:String}, '%')`,
+ params,
+ }
+ }
+
+ if (goal.matchType === GoalMatchType.EXACT) {
+ params.goalValue = goal.value || ''
+ return { condition: `pg = {goalValue:String}`, params }
+ }
+ if ((goal.value || '').trim() === '') {
+ return { condition: '1=0', params: {} }
+ }
+ params.goalValue = goal.value || ''
+ return {
+ condition: `pg ILIKE concat('%', {goalValue:String}, '%')`,
+ params,
}
}
@@ -429,9 +429,10 @@ export class GoalController {
safeTimezone,
)
- const table = goal.type === GoalType.CUSTOM_EVENT ? 'customEV' : 'analytics'
+ const goalType =
+ goal.type === GoalType.CUSTOM_EVENT ? 'custom_event' : 'pageview'
const { condition: matchCondition, params: matchParams } =
- this.buildGoalMatchCondition(goal, table)
+ this.buildGoalMatchCondition(goal)
const { condition: metaCondition, params: metaParams } =
this.buildMetadataCondition(goal)
@@ -440,9 +441,10 @@ export class GoalController {
SELECT
count(*) as conversions,
uniqExact(psid) as uniqueSessions
- FROM ${table}
+ FROM events
WHERE
pid = {pid:FixedString(12)}
+ AND type = {goalType:String}
AND ${matchCondition}
${metaCondition}
AND created BETWEEN {groupFrom:String} AND {groupTo:String}
@@ -450,6 +452,7 @@ export class GoalController {
const queryParams = {
pid: goal.project.id,
+ goalType,
groupFrom: groupFromUTC,
groupTo: groupToUTC,
...matchParams,
@@ -468,9 +471,10 @@ export class GoalController {
// Get total unique sessions for conversion rate
const totalSessionsQuery = `
SELECT uniqExact(psid) as totalSessions
- FROM analytics
+ FROM events
WHERE
pid = {pid:FixedString(12)}
+ AND type = 'pageview'
AND created BETWEEN {groupFrom:String} AND {groupTo:String}
`
@@ -649,7 +653,8 @@ export class GoalController {
safeTimezone,
)
- const table = goal.type === GoalType.CUSTOM_EVENT ? 'customEV' : 'analytics'
+ const goalType =
+ goal.type === GoalType.CUSTOM_EVENT ? 'custom_event' : 'pageview'
const timeBucketFunc = Object.prototype.hasOwnProperty.call(
timeBucketConversion,
resolvedTimeBucket,
@@ -659,7 +664,7 @@ export class GoalController {
const [selector, groupBy] = this.getGroupSubquery(resolvedTimeBucket)
const { condition: matchCondition, params: matchParams } =
- this.buildGoalMatchCondition(goal, table)
+ this.buildGoalMatchCondition(goal)
const { condition: metaCondition, params: metaParams } =
this.buildMetadataCondition(goal)
@@ -671,9 +676,10 @@ export class GoalController {
FROM (
SELECT *,
${timeBucketFunc}(toTimeZone(created, {timezone:String})) as tz_created
- FROM ${table}
+ FROM events
WHERE
pid = {pid:FixedString(12)}
+ AND type = {goalType:String}
AND ${matchCondition}
${metaCondition}
AND created BETWEEN {groupFrom:String} AND {groupTo:String}
@@ -684,6 +690,7 @@ export class GoalController {
const queryParams = {
pid: goal.project.id,
+ goalType,
groupFrom: groupFromUTC,
groupTo: groupToUTC,
timezone: safeTimezone,
diff --git a/backend/apps/cloud/src/project/project.controller.ts b/backend/apps/cloud/src/project/project.controller.ts
index 5e54aab17..13711340a 100644
--- a/backend/apps/cloud/src/project/project.controller.ts
+++ b/backend/apps/cloud/src/project/project.controller.ts
@@ -745,12 +745,8 @@ export class ProjectController {
}
const queries = [
- 'ALTER TABLE analytics DELETE WHERE pid={pid:FixedString(12)}',
- 'ALTER TABLE customEV DELETE WHERE pid={pid:FixedString(12)}',
- 'ALTER TABLE performance DELETE WHERE pid={pid:FixedString(12)}',
- 'ALTER TABLE errors DELETE WHERE pid={pid:FixedString(12)}',
+ 'ALTER TABLE events DELETE WHERE pid={pid:FixedString(12)}',
'ALTER TABLE error_statuses DELETE WHERE pid={pid:FixedString(12)}',
- 'ALTER TABLE captcha DELETE WHERE pid={pid:FixedString(12)}',
]
try {
@@ -1607,12 +1603,8 @@ export class ProjectController {
}
const queries = [
- 'ALTER TABLE analytics DELETE WHERE pid={pid:FixedString(12)}',
- 'ALTER TABLE customEV DELETE WHERE pid={pid:FixedString(12)}',
- 'ALTER TABLE performance DELETE WHERE pid={pid:FixedString(12)}',
- 'ALTER TABLE errors DELETE WHERE pid={pid:FixedString(12)}',
+ 'ALTER TABLE events DELETE WHERE pid={pid:FixedString(12)}',
'ALTER TABLE error_statuses DELETE WHERE pid={pid:FixedString(12)}',
- 'ALTER TABLE captcha DELETE WHERE pid={pid:FixedString(12)}',
]
try {
diff --git a/backend/apps/cloud/src/project/project.service.ts b/backend/apps/cloud/src/project/project.service.ts
index 5e5c29c63..f974e49c8 100644
--- a/backend/apps/cloud/src/project/project.service.ts
+++ b/backend/apps/cloud/src/project/project.service.ts
@@ -489,12 +489,8 @@ export class ProjectService {
const projectIds = _map(projects, 'id')
const queries = [
- 'ALTER TABLE analytics DELETE WHERE pid IN ({pids:Array(FixedString(12))})',
- 'ALTER TABLE customEV DELETE WHERE pid IN ({pids:Array(FixedString(12))})',
- 'ALTER TABLE performance DELETE WHERE pid IN ({pids:Array(FixedString(12))})',
- 'ALTER TABLE errors DELETE WHERE pid IN ({pids:Array(FixedString(12))})',
+ 'ALTER TABLE events DELETE WHERE pid IN ({pids:Array(FixedString(12))})',
'ALTER TABLE error_statuses DELETE WHERE pid IN ({pids:Array(FixedString(12))})',
- 'ALTER TABLE captcha DELETE WHERE pid IN ({pids:Array(FixedString(12))})',
]
await Promise.all(
@@ -654,7 +650,7 @@ export class ProjectService {
)
}
- let query = `ALTER TABLE analytics DELETE WHERE pid={pid:FixedString(12)} AND (`
+ let query = `ALTER TABLE events DELETE WHERE pid={pid:FixedString(12)} AND type = 'pageview' AND (`
const params = {
pid,
@@ -684,10 +680,7 @@ export class ProjectService {
to: string,
): Promise<void> {
const queries = [
- 'ALTER TABLE analytics DELETE WHERE pid = {pid:FixedString(12)} AND created BETWEEN {from:String} AND {to:String}',
- 'ALTER TABLE customEV DELETE WHERE pid = {pid:FixedString(12)} AND created BETWEEN {from:String} AND {to:String}',
- 'ALTER TABLE performance DELETE WHERE pid = {pid:FixedString(12)} AND created BETWEEN {from:String} AND {to:String}',
- 'ALTER TABLE errors DELETE WHERE pid = {pid:FixedString(12)} AND created BETWEEN {from:String} AND {to:String}',
+ "ALTER TABLE events DELETE WHERE pid = {pid:FixedString(12)} AND type IN ('pageview', 'custom_event', 'performance', 'error', 'captcha') AND created BETWEEN {from:String} AND {to:String}",
'ALTER TABLE error_statuses DELETE WHERE pid = {pid:FixedString(12)} AND created BETWEEN {from:String} AND {to:String}',
]
@@ -820,52 +813,37 @@ export class ProjectService {
monthEnd,
}
- const selector = `
+ const query = `
+ SELECT
+ countIf(type = 'pageview') AS pageviews,
+ countIf(type = 'custom_event') AS customEvents,
+ countIf(type = 'captcha') AS captcha,
+ countIf(type = 'error') AS errors
+ FROM events
WHERE created BETWEEN {monthStart:String} AND {monthEnd:String}
- AND pid IN ({pids:Array(FixedString(12))})
+ AND pid IN ({pids:Array(FixedString(12))})
+ AND type IN ('pageview', 'custom_event', 'captcha', 'error')
`
- const countEVQuery = `SELECT count() FROM analytics ${selector}`
- const countCustomEVQuery = `SELECT count() FROM customEV ${selector}`
- const countCaptchaQuery = `SELECT count() FROM captcha ${selector}`
- const countErrorsQuery = `SELECT count() FROM errors ${selector}`
-
- const [
- { data: pageviews },
- { data: customEvents },
- { data: captcha },
- { data: errors },
- ] = await Promise.all([
- clickhouse
- .query({
- query: countEVQuery,
- query_params: params,
- })
- .then((resultSet) => resultSet.json()),
- clickhouse
- .query({
- query: countCustomEVQuery,
- query_params: params,
- })
- .then((resultSet) => resultSet.json()),
- clickhouse
- .query({
- query: countCaptchaQuery,
- query_params: params,
- })
- .then((resultSet) => resultSet.json()),
- clickhouse
- .query({
- query: countErrorsQuery,
- query_params: params,
- })
- .then((resultSet) => resultSet.json()),
- ])
+ const { data: counts } = await clickhouse
+ .query({
+ query,
+ query_params: params,
+ })
+ .then((resultSet) =>
+ resultSet.json<{
+ pageviews: string | number
+ customEvents: string | number
+ captcha: string | number
+ errors: string | number
+ }>(),
+ )
- totalPageviews += pageviews[0]['count()']
- totalCustomEvents += customEvents[0]['count()']
- totalCaptcha += captcha[0]['count()']
- totalErrors += errors[0]['count()']
+ const row = counts[0]
+ totalPageviews += Number(row?.pageviews) || 0
+ totalCustomEvents += Number(row?.customEvents) || 0
+ totalCaptcha += Number(row?.captcha) || 0
+ totalErrors += Number(row?.errors) || 0
}
count = totalPageviews + totalCustomEvents + totalCaptcha + totalErrors
@@ -916,54 +894,43 @@ export class ProjectService {
// Process PIDs in chunks
for (let i = 0; i < pids.length; i += CHUNK_SIZE) {
const pidChunk = pids.slice(i, i + CHUNK_SIZE)
- const params = { pids: pidChunk }
-
- const selector = `
- WHERE pid IN ({pids:Array(FixedString(12))})
- AND created BETWEEN '${monthStart}' AND '${monthEnd}'
- `
-
- const countEVQuery = `SELECT count() FROM analytics ${selector}`
- const countCustomEVQuery = `SELECT count() FROM customEV ${selector}`
- const countCaptchaQuery = `SELECT count() FROM captcha ${selector}`
- const countErrorsQuery = `SELECT count() FROM errors ${selector}`
-
- const [
- { data: rawTraffic },
- { data: rawCustomEvents },
- { data: rawCaptcha },
- { data: rawErrors },
- ] = await Promise.all([
- clickhouse
- .query({
- query: countEVQuery,
- query_params: params,
- })
- .then((resultSet) => resultSet.json()),
- clickhouse
- .query({
- query: countCustomEVQuery,
- query_params: params,
- })
- .then((resultSet) => resultSet.json()),
- clickhouse
- .query({
- query: countCaptchaQuery,
- query_params: params,
- })
- .then((resultSet) => resultSet.json()),
- clickhouse
- .query({
- query: countErrorsQuery,
- query_params: params,
- })
- .then((resultSet) => resultSet.json()),
- ])
+ const params = {
+ pids: pidChunk,
+ monthStart,
+ monthEnd,
+ }
- totalTraffic += rawTraffic[0]['count()']
- totalCustomEvents += rawCustomEvents[0]['count()']
- totalCaptcha += rawCaptcha[0]['count()']
- totalErrors += rawErrors[0]['count()']
+ const query = `
+ SELECT
+ countIf(type = 'pageview') AS traffic,
+ countIf(type = 'custom_event') AS customEvents,
+ countIf(type = 'captcha') AS captcha,
+ countIf(type = 'error') AS errors
+ FROM events
+ WHERE pid IN ({pids:Array(FixedString(12))})
+ AND created BETWEEN {monthStart:String} AND {monthEnd:String}
+ AND type IN ('pageview', 'custom_event', 'captcha', 'error')
+ `
+
+ const { data: counts } = await clickhouse
+ .query({
+ query,
+ query_params: params,
+ })
+ .then((resultSet) =>
+ resultSet.json<{
+ traffic: string | number
+ customEvents: string | number
+ captcha: string | number
+ errors: string | number
+ }>(),
+ )
+
+ const row = counts[0]
+ totalTraffic += Number(row?.traffic) || 0
+ totalCustomEvents += Number(row?.customEvents) || 0
+ totalCaptcha += Number(row?.captcha) || 0
+ totalErrors += Number(row?.errors) || 0
}
const total =
@@ -1043,36 +1010,10 @@ export class ProjectService {
)
const query = `
- SELECT
- pid,
- CASE
- WHEN EXISTS (
- SELECT 1
- FROM analytics
- WHERE pid IN (${pids})
- )
- OR EXISTS (
- SELECT 1
- FROM customEV
- WHERE pid IN (${pids})
- )
- THEN 1
- ELSE 0
- END AS exists
- FROM
- (
- SELECT DISTINCT pid
- FROM
- (
- SELECT pid
- FROM analytics
- WHERE pid IN (${pids})
- UNION ALL
- SELECT pid
- FROM customEV
- WHERE pid IN (${pids})
- ) AS t
- );
+ SELECT DISTINCT pid
+ FROM events
+ WHERE pid IN (${pids})
+ AND type IN ('pageview', 'custom_event')
`
const { data } = await clickhouse
@@ -1105,23 +1046,10 @@ export class ProjectService {
)
const query = `
- SELECT
- pid,
- CASE
- WHEN EXISTS (
- SELECT 1
- FROM errors
- WHERE pid IN (${pids})
- )
- THEN 1
- ELSE 0
- END AS exists
- FROM
- (
- SELECT DISTINCT pid
- FROM errors
- WHERE pid IN (${pids})
- );
+ SELECT DISTINCT pid
+ FROM events
+ WHERE pid IN (${pids})
+ AND type = 'error'
`
const { data } = await clickhouse
@@ -1154,23 +1082,10 @@ export class ProjectService {
)
const query = `
- SELECT
- pid,
- CASE
- WHEN EXISTS (
- SELECT 1
- FROM captcha
- WHERE pid IN (${pids})
- )
- THEN 1
- ELSE 0
- END AS exists
- FROM
- (
- SELECT DISTINCT pid
- FROM captcha
- WHERE pid IN (${pids})
- );
+ SELECT DISTINCT pid
+ FROM events
+ WHERE pid IN (${pids})
+ AND type = 'captcha'
`
const { data } = await clickhouse
diff --git a/backend/apps/cloud/src/task-manager/task-manager.service.ts b/backend/apps/cloud/src/task-manager/task-manager.service.ts
index 83b53f2e3..e4b4cb4a7 100644
--- a/backend/apps/cloud/src/task-manager/task-manager.service.ts
+++ b/backend/apps/cloud/src/task-manager/task-manager.service.ts
@@ -143,13 +143,15 @@ const mapLimit = async <T, R>(
return results
}
-// TODO: Count for other tables like captcha, errors, customEV too
const generatePlanUsageQueryForUser = (): string => {
// NOTE: keep all values parameterized to avoid injection and formatting issues.
+ // Counts every billable event kind (pageview, custom_event, error, captcha)
+ // in a single scan over the unified events table.
return `
SELECT {uid:String} AS id, count(*) AS "count"
- FROM analytics
+ FROM events
WHERE pid IN ({pids:Array(FixedString(12))})
+ AND type IN ('pageview', 'custom_event', 'error', 'captcha')
AND created BETWEEN {from:String} AND {to:String}
`
}
@@ -385,27 +387,53 @@ export class TaskManagerService {
): { condition: string; params: Record<string, string> } {
const goalValue = (goal.value ?? '').toString()
- // If goal value is blank, never match anything (avoid LIKE '%%')
- if (goalValue.trim() === '') {
- return { condition: '1=0', params: {} }
- }
-
const params: Record<string, string> = {}
- const column = goal.type === GoalType.CUSTOM_EVENT ? 'ev' : 'pg'
+ const column = goal.type === GoalType.CUSTOM_EVENT ? 'event_name' : 'pg'
+ const appendMetadataFilters = (condition: string) => {
+ if (!goal.metadataFilters || goal.metadataFilters.length === 0) {
+ return { condition, params }
+ }
+
+ const metaConditions: string[] = []
+
+ goal.metadataFilters.forEach((filter, index) => {
+ const keyParam = `${paramKey}_metaKey${index}`
+ const valueParam = `${paramKey}_metaValue${index}`
+ params[keyParam] = filter.key
+ params[valueParam] = filter.value
+ metaConditions.push(
+ `has(meta.key, {${keyParam}:String}) AND meta.value[indexOf(meta.key, {${keyParam}:String})] = {${valueParam}:String}`,
+ )
+ })
+
+ return {
+ condition: `(${condition}) AND (${metaConditions.join(' AND ')})`,
+ params,
+ }
+ }
if (goal.matchType === GoalMatchType.EXACT) {
params[paramKey] = goalValue
- return { condition: `${column} = {${paramKey}:String}`, params }
+ return appendMetadataFilters(`${column} = {${paramKey}:String}`)
}
if (goal.matchType === GoalMatchType.CONTAINS) {
+ // Avoid wildcard goals matching every row via LIKE '%%'.
+ if (goalValue.trim() === '') {
+ return { condition: '1=0', params: {} }
+ }
+
params[paramKey] = `%${goalValue}%`
- return { condition: `${column} LIKE {${paramKey}:String}`, params }
+ return appendMetadataFilters(`${column} ILIKE {${paramKey}:String}`)
}
// Regex goal
+ if (!goalValue || goalValue === '') {
+ return appendMetadataFilters('false')
+ }
+
params[paramKey] = goalValue
- return { condition: `match(${column}, {${paramKey}:String})`, params }
+ return appendMetadataFilters(`match(${column}, {${paramKey}:String})`)
}
/**
@@ -427,7 +455,7 @@ export class TaskManagerService {
const total = Number(totalSessions) || 0
const buildUnionQuery = (
- table: 'analytics' | 'customEV',
+ eventType: 'pageview' | 'custom_event',
goals: Goal[],
): { query: string; params: Record<string, any> } | null => {
if (_isEmpty(goals)) {
@@ -459,9 +487,10 @@ export class TaskManagerService {
{${goalIdKey}:String} as goalId,
count(*) as conversions,
uniqExact(psid) as uniqueSessions
- FROM ${table}
+ FROM events
WHERE
pid = {pid:FixedString(12)}
+ AND type = '${eventType}'
AND ${condition}
AND created BETWEEN {groupFrom:String} AND {groupTo:String}
`)
@@ -488,8 +517,8 @@ export class TaskManagerService {
(g) => g.type === GoalType.CUSTOM_EVENT,
)
- const analyticsQuery = buildUnionQuery('analytics', analyticsGoals)
- const customEventQuery = buildUnionQuery('customEV', customEventGoals)
+ const analyticsQuery = buildUnionQuery('pageview', analyticsGoals)
+ const customEventQuery = buildUnionQuery('custom_event', customEventGoals)
const [analyticsRes, customEventRes] = await Promise.all([
analyticsQuery
@@ -573,24 +602,10 @@ export class TaskManagerService {
for (let i = 0; i < pids.length; i += CHUNK_SIZE) {
const pidChunk = pids.slice(i, i + CHUNK_SIZE)
const query = `
- SELECT sum(cnt) AS totalEvents
- FROM (
- SELECT count() AS cnt
- FROM analytics
- WHERE pid IN ({pids:Array(FixedString(12))})
- UNION ALL
- SELECT count() AS cnt
- FROM customEV
- WHERE pid IN ({pids:Array(FixedString(12))})
- UNION ALL
- SELECT count() AS cnt
- FROM captcha
- WHERE pid IN ({pids:Array(FixedString(12))})
- UNION ALL
- SELECT count() AS cnt
- FROM errors
- WHERE pid IN ({pids:Array(FixedString(12))})
- )
+ SELECT count() AS totalEvents
+ FROM events
+ WHERE pid IN ({pids:Array(FixedString(12))})
+ AND type IN ('pageview', 'custom_event', 'error', 'captcha', 'performance')
`
const { data } = await clickhouse
@@ -1717,8 +1732,8 @@ export class TaskManagerService {
count()
FROM (
SELECT eid, min(created) as first_seen
- FROM errors
- WHERE pid = {pid:FixedString(12)}
+ FROM events
+ WHERE pid = {pid:FixedString(12)} AND type = 'error'
GROUP BY eid
)
WHERE first_seen >= now() - ${subtractSecondsTimeframe}
@@ -1727,9 +1742,10 @@ export class TaskManagerService {
query = `
SELECT
count()
- FROM errors
+ FROM events
WHERE
pid = {pid:FixedString(12)}
+ AND type = 'error'
AND created >= now() - ${subtractSecondsTimeframe}
`
}
@@ -1737,10 +1753,11 @@ export class TaskManagerService {
query = `
SELECT
count()
- FROM customEV
+ FROM events
WHERE
pid = {pid:FixedString(12)}
- AND ev = {ev:String}
+ AND type = 'custom_event'
+ AND event_name = {ev:String}
AND created >= now() - ${subtractSecondsTimeframe}
`
queryParams.ev = alert.queryCustomEvent
@@ -1749,9 +1766,10 @@ export class TaskManagerService {
query = `
SELECT
count(${isUnique ? 'DISTINCT psid' : '*'})
- FROM analytics
+ FROM events
WHERE
pid = {pid:FixedString(12)}
+ AND type = 'pageview'
AND created >= now() - ${subtractSecondsTimeframe}
`
}
@@ -1798,14 +1816,14 @@ export class TaskManagerService {
if (alert.alertOnNewErrorsOnly) {
detailQuery = `
- SELECT eid, name, message, lineno, colno, filename
- FROM errors
- WHERE pid = {pid:FixedString(12)} AND eid IN (
+ SELECT eid, error_name AS name, error_message AS message, lineno, colno, error_filename AS filename
+ FROM events
+ WHERE pid = {pid:FixedString(12)} AND type = 'error' AND eid IN (
SELECT eid
FROM (
SELECT eid, min(created) AS first_seen_for_eid
- FROM errors
- WHERE pid = {pid:FixedString(12)}
+ FROM events
+ WHERE pid = {pid:FixedString(12)} AND type = 'error'
GROUP BY eid
)
WHERE first_seen_for_eid >= now() - ${subtractSecondsTimeframe}
@@ -1815,10 +1833,11 @@ export class TaskManagerService {
`
} else {
detailQuery = `
- SELECT eid, name, message, lineno, colno, filename
- FROM errors
+ SELECT eid, error_name AS name, error_message AS message, lineno, colno, error_filename AS filename
+ FROM events
WHERE
pid = {pid:FixedString(12)}
+ AND type = 'error'
AND created >= now() - ${subtractSecondsTimeframe}
ORDER BY created DESC
LIMIT 1
@@ -1988,9 +2007,12 @@ export class TaskManagerService {
},
select: ['id'],
})
- const now = dayjs.utc().format('YYYY-MM-DD')
+ const now = dayjs.utc().format('YYYY-MM-DD HH:mm:ss')
// a bit more than 2 months ago
- const nineWeeksAgo = dayjs.utc().subtract(9, 'w').format('YYYY-MM-DD')
+ const nineWeeksAgo = dayjs
+ .utc()
+ .subtract(9, 'w')
+ .format('YYYY-MM-DD HH:mm:ss')
await mapLimit(users, REPORTS_USERS_CONCURRENCY, async (user) => {
const { id } = user
@@ -2002,61 +2024,10 @@ export class TaskManagerService {
return
}
- // No need to check for performance activity because it's not tracked without tracking analytics
- const queryAnalytics = `SELECT count() FROM analytics WHERE pid IN ({pids:Array(FixedString(12))}) AND created BETWEEN {nineWeeksAgo:String} AND {now:String}`
- const queryCaptcha = `SELECT count() FROM captcha WHERE pid IN ({pids:Array(FixedString(12))}) AND created BETWEEN {nineWeeksAgo:String} AND {now:String}`
- const queryCustomEvents = `SELECT count() FROM customEV WHERE pid IN ({pids:Array(FixedString(12))}) AND created BETWEEN {nineWeeksAgo:String} AND {now:String}`
+ const queryEvents = `SELECT 1 FROM events WHERE pid IN ({pids:Array(FixedString(12))}) AND type IN ('pageview', 'captcha', 'custom_event', 'error', 'performance') AND created BETWEEN {nineWeeksAgo:String} AND {now:String} LIMIT 1`
// Process project IDs in chunks to avoid ClickHouse field value limit
- let totalAnalytics = 0
- let totalCaptcha = 0
- let totalCustomEvents = 0
-
- for (let i = 0; i < pids.length; i += CHUNK_SIZE) {
- const pidChunk = pids.slice(i, i + CHUNK_SIZE)
- const queryParams = {
- pids: pidChunk,
- nineWeeksAgo,
- now,
- }
-
- const { data: analyticsResult } = await clickhouse
- .query({
- query: queryAnalytics,
- query_params: queryParams,
- })
- .then((resultSet) => resultSet.json<{ 'count()': number }>())
-
- totalAnalytics += analyticsResult[0]['count()']
-
- // Early return if we found activity
- if (totalAnalytics > 0) {
- return
- }
- }
-
- for (let i = 0; i < pids.length; i += CHUNK_SIZE) {
- const pidChunk = pids.slice(i, i + CHUNK_SIZE)
- const queryParams = {
- pids: pidChunk,
- nineWeeksAgo,
- now,
- }
-
- const { data: captchaResult } = await clickhouse
- .query({
- query: queryCaptcha,
- query_params: queryParams,
- })
- .then((resultSet) => resultSet.json<{ 'count()': number }>())
-
- totalCaptcha += captchaResult[0]['count()']
-
- // Early return if we found activity
- if (totalCaptcha > 0) {
- return
- }
- }
+ let totalEvents = 0
for (let i = 0; i < pids.length; i += CHUNK_SIZE) {
const pidChunk = pids.slice(i, i + CHUNK_SIZE)
@@ -2066,17 +2037,17 @@ export class TaskManagerService {
now,
}
- const { data: customEventsResult } = await clickhouse
+ const { data: eventsResult } = await clickhouse
.query({
- query: queryCustomEvents,
+ query: queryEvents,
query_params: queryParams,
})
- .then((resultSet) => resultSet.json<{ 'count()': number }>())
+ .then((resultSet) => resultSet.json<Record<string, number>>())
- totalCustomEvents += customEventsResult[0]['count()']
+ totalEvents += eventsResult.length
// Early return if we found activity
- if (totalCustomEvents > 0) {
+ if (totalEvents > 0) {
return
}
}
diff --git a/backend/apps/community/src/analytics/analytics.controller.ts b/backend/apps/community/src/analytics/analytics.controller.ts
index c6313fad7..1d5b37746 100644
--- a/backend/apps/community/src/analytics/analytics.controller.ts
+++ b/backend/apps/community/src/analytics/analytics.controller.ts
@@ -73,12 +73,7 @@ import {
GetErrorOverviewOptions,
} from './dto/get-error-overview.dto'
import { PatchStatusDto } from './dto/patch-status.dto'
-import {
- customEventTransformer,
- errorEventTransformer,
- performanceTransformer,
- trafficTransformer,
-} from './utils/transformers'
+import { eventTransformer } from './utils/transformers'
import { enrichTrafficSource } from './utils/clickIdSources'
import { MAX_METRICS_IN_VIEW } from '../project/dto/create-project-view.dto'
import { GetOverallStatsDto } from './dto/get-overall-stats.dto'
@@ -250,7 +245,7 @@ export class AnalyticsController {
if (period === 'all') {
const res = await this.analyticsService.calculateTimeBucketForAllTime(
pid,
- isCaptcha ? 'captcha' : 'analytics',
+ isCaptcha ? 'captcha' : 'pageview',
)
diff = res.diff
@@ -278,13 +273,11 @@ export class AnalyticsController {
diff,
)
- let subQuery = `FROM ${
- isCaptcha ? 'captcha' : 'analytics'
- } WHERE pid = {pid:FixedString(12)} ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String}`
-
- if (customEVFilterApplied && !isCaptcha) {
- subQuery = `FROM customEV WHERE pid = {pid:FixedString(12)} ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String}`
- }
+ const subQuery = this.analyticsService.buildAnalyticsEventsSubQuery(
+ filtersQuery,
+ customEVFilterApplied,
+ isCaptcha,
+ )
const paramsData = {
params: {
@@ -413,12 +406,15 @@ export class AnalyticsController {
let diff
if (period === 'all') {
- const [analyticsRes, customEVRes] = await Promise.all([
- this.analyticsService.calculateTimeBucketForAllTime(pid, 'analytics'),
- this.analyticsService.calculateTimeBucketForAllTime(pid, 'customEV'),
+ const [pageviewRes, customEventRes] = await Promise.all([
+ this.analyticsService.calculateTimeBucketForAllTime(pid, 'pageview'),
+ this.analyticsService.calculateTimeBucketForAllTime(
+ pid,
+ 'custom_event',
+ ),
])
- diff = Math.max(analyticsRes.diff, customEVRes.diff)
+ diff = Math.max(pageviewRes.diff, customEventRes.diff)
}
const safeTimezone = this.analyticsService.getSafeTimezone(timezone)
@@ -526,12 +522,15 @@ export class AnalyticsController {
let diff
if (period === 'all') {
- const [analyticsRes, customEVRes] = await Promise.all([
- this.analyticsService.calculateTimeBucketForAllTime(pid, 'analytics'),
- this.analyticsService.calculateTimeBucketForAllTime(pid, 'customEV'),
+ const [pageviewRes, customEventRes] = await Promise.all([
+ this.analyticsService.calculateTimeBucketForAllTime(pid, 'pageview'),
+ this.analyticsService.calculateTimeBucketForAllTime(
+ pid,
+ 'custom_event',
+ ),
])
- diff = Math.max(analyticsRes.diff, customEVRes.diff)
+ diff = Math.max(pageviewRes.diff, customEventRes.diff)
}
const safeTimezone = this.analyticsService.getSafeTimezone(timezone)
@@ -732,7 +731,7 @@ export class AnalyticsController {
diff,
)
- const subQuery = `FROM performance WHERE pid = {pid:FixedString(12)} ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String}`
+ const subQuery = `FROM events WHERE pid = {pid:FixedString(12)} AND type = 'performance' ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String}`
const paramsData = { params: { pid, groupFrom, groupTo, ...filtersParams } }
@@ -836,7 +835,7 @@ export class AnalyticsController {
if (period === 'all') {
const res = await this.analyticsService.calculateTimeBucketForAllTime(
pid,
- 'analytics',
+ 'pageview',
)
diff = res.diff
@@ -1026,32 +1025,12 @@ export class AnalyticsController {
any(os) AS os,
any(cc) AS cc,
toString(psid) AS psid
- FROM
- (
- SELECT
- psid,
- dv,
- br,
- os,
- cc
- FROM analytics
- WHERE
- pid = {pid:FixedString(12)}
- AND created >= {since:DateTime}
- AND psid IS NOT NULL
- UNION ALL
- SELECT
- psid,
- dv,
- br,
- os,
- cc
- FROM customEV
- WHERE
- pid = {pid:FixedString(12)}
- AND created >= {since:DateTime}
- AND psid IS NOT NULL
- )
+ FROM events
+ WHERE
+ pid = {pid:FixedString(12)}
+ AND type IN ('pageview', 'custom_event')
+ AND created >= {since:DateTime}
+ AND psid IS NOT NULL
GROUP BY psid
`
@@ -1151,7 +1130,8 @@ export class AnalyticsController {
enrichTrafficSource(eventsDTO)
- const transformed = customEventTransformer({
+ const transformed = eventTransformer({
+ type: 'custom_event',
psid,
profileId,
pid: eventsDTO.pid,
@@ -1183,7 +1163,7 @@ export class AnalyticsController {
try {
await clickhouse.insert({
- table: 'customEV',
+ table: 'events',
format: 'JSONEachRow',
values: [transformed],
clickhouse_settings: { async_insert: 1 },
@@ -1321,7 +1301,8 @@ export class AnalyticsController {
enrichTrafficSource(logDTO)
- const transformed = trafficTransformer({
+ const transformed = eventTransformer({
+ type: 'pageview',
psid,
profileId,
pid: logDTO.pid,
@@ -1364,7 +1345,8 @@ export class AnalyticsController {
ttfb,
} = logDTO.perf
- perfTransformed = performanceTransformer({
+ perfTransformed = eventTransformer({
+ type: 'performance',
pid: logDTO.pid,
host: this.analyticsService.getHostFromOrigin(headers.origin),
pg: logDTO.pg,
@@ -1392,7 +1374,7 @@ export class AnalyticsController {
try {
await clickhouse.insert({
- table: 'analytics',
+ table: 'events',
format: 'JSONEachRow',
values: [transformed],
clickhouse_settings: { async_insert: 1 },
@@ -1400,7 +1382,7 @@ export class AnalyticsController {
if (!_isEmpty(perfTransformed)) {
await clickhouse.insert({
- table: 'performance',
+ table: 'events',
format: 'JSONEachRow',
values: [perfTransformed],
clickhouse_settings: { async_insert: 1 },
@@ -1489,7 +1471,8 @@ export class AnalyticsController {
const { deviceType, browserName, browserVersion, osName, osVersion } =
await this.analyticsService.getRequestInformation(headers)
- const transformed = trafficTransformer({
+ const transformed = eventTransformer({
+ type: 'pageview',
psid,
profileId,
pid: logDTO.pid,
@@ -1520,7 +1503,7 @@ export class AnalyticsController {
try {
await clickhouse.insert({
- table: 'analytics',
+ table: 'events',
format: 'JSONEachRow',
values: [transformed],
clickhouse_settings: { async_insert: 1 },
@@ -1571,7 +1554,7 @@ export class AnalyticsController {
if (period === 'all') {
const res = await this.analyticsService.calculateTimeBucketForAllTime(
pid,
- customEVFilterApplied ? 'customEV' : 'analytics',
+ this.analyticsService.getAnalyticsEventType(customEVFilterApplied),
)
timeBucket = res.timeBucket[0]
@@ -1675,7 +1658,7 @@ export class AnalyticsController {
if (period === VALID_PERIODS[VALID_PERIODS.length - 1]) {
const res = await this.analyticsService.calculateTimeBucketForAllTime(
pid,
- 'customEV',
+ 'custom_event',
)
newTimeBucket = _includes(res.timeBucket, timeBucket)
@@ -1869,7 +1852,8 @@ export class AnalyticsController {
const { name, message, lineno, colno, filename, stackTrace, meta } =
errorDTO
- const transformed = errorEventTransformer({
+ const transformed = eventTransformer({
+ type: 'error',
psid,
profileId,
eid: this.analyticsService.getErrorID(errorDTO),
@@ -1901,7 +1885,7 @@ export class AnalyticsController {
try {
await clickhouse.insert({
- table: 'errors',
+ table: 'events',
format: 'JSONEachRow',
values: [transformed],
clickhouse_settings: { async_insert: 1 },
@@ -1986,7 +1970,7 @@ export class AnalyticsController {
if (period === 'all') {
const res = await this.analyticsService.calculateTimeBucketForAllTime(
pid,
- 'errors',
+ 'error',
)
timeBucket = res.timeBucket[0]
@@ -2063,7 +2047,7 @@ export class AnalyticsController {
if (period === 'all') {
const res = await this.analyticsService.calculateTimeBucketForAllTime(
pid,
- 'errors',
+ 'error',
)
newTimeBucket = res.timeBucket[0]
@@ -2139,7 +2123,7 @@ export class AnalyticsController {
if (period === 'all') {
const res = await this.analyticsService.calculateTimeBucketForAllTime(
pid,
- 'errors',
+ 'error',
)
newTimeBucket = res.timeBucket[0]
@@ -2151,6 +2135,8 @@ export class AnalyticsController {
DataType.ERRORS,
true,
)
+ const [sessionFiltersQuery, sessionFiltersParams] =
+ this.analyticsService.getFiltersQuery(filters, DataType.ANALYTICS, true)
const safeTimezone = this.analyticsService.getSafeTimezone(timezone)
const { groupFromUTC, groupToUTC } = this.analyticsService.getGroupFromTo(
@@ -2180,6 +2166,8 @@ export class AnalyticsController {
groupToUTC,
newTimeBucket,
parsedOptions.showResolved || false,
+ sessionFiltersQuery,
+ sessionFiltersParams,
)
}
@@ -2218,7 +2206,7 @@ export class AnalyticsController {
if (period === 'all') {
const res = await this.analyticsService.calculateTimeBucketForAllTime(
pid,
- 'errors',
+ 'error',
)
newTimeBucket = res.timeBucket[0]
@@ -2291,7 +2279,7 @@ export class AnalyticsController {
if (period === 'all') {
const res = await this.analyticsService.calculateTimeBucketForAllTime(
pid,
- customEVFilterApplied ? 'customEV' : 'analytics',
+ this.analyticsService.getAnalyticsEventType(customEVFilterApplied),
)
timeBucket = res.timeBucket[0]
@@ -2368,7 +2356,7 @@ export class AnalyticsController {
if (period === 'all') {
const res = await this.analyticsService.calculateTimeBucketForAllTime(
pid,
- 'analytics',
+ 'pageview',
)
timeBucket = res.timeBucket[0]
diff = res.diff
@@ -2454,7 +2442,7 @@ export class AnalyticsController {
if (period === 'all') {
const res = await this.analyticsService.calculateTimeBucketForAllTime(
pid,
- customEVFilterApplied ? 'customEV' : 'analytics',
+ this.analyticsService.getAnalyticsEventType(customEVFilterApplied),
)
timeBucket = res.timeBucket[0]
diff --git a/backend/apps/community/src/analytics/analytics.service.ts b/backend/apps/community/src/analytics/analytics.service.ts
index f63a48d82..4215d6bce 100644
--- a/backend/apps/community/src/analytics/analytics.service.ts
+++ b/backend/apps/community/src/analytics/analytics.service.ts
@@ -270,6 +270,14 @@ const EXCLUDE_NULL_FOR = [
'ctp',
]
+const ERROR_COLUMN_MAP: Record<string, string> = {
+ name: 'error_name',
+ message: 'error_message',
+ filename: 'error_filename',
+}
+
+const mapErrorColumn = (type: string): string => ERROR_COLUMN_MAP[type] || type
+
const generateParamsQuery = (
col: string,
subQuery: string,
@@ -277,7 +285,8 @@ const generateParamsQuery = (
type: 'traffic' | 'performance' | 'errors' | 'captcha',
measure?: PerfMeasure,
): string => {
- let columns = [`${col} as name`]
+ const sqlCol = type === 'errors' ? mapErrorColumn(col) : col
+ let columns = [`${sqlCol} as name`]
// For regions and cities we'll return an array of objects, that will also include the country code and region code
// We need the conutry code to display the flag next to the region/city name
@@ -301,15 +310,15 @@ const generateParamsQuery = (
const fn = MEASURES_MAP[processedMeasure]
- if (col === 'pg' || col === 'host') {
+ if (sqlCol === 'pg' || sqlCol === 'host') {
return `SELECT ${columnsQuery}, round(divide(${fn}(pageLoad), 1000), 2) as count ${subQuery} GROUP BY ${columnsQuery}`
}
- return `SELECT ${columnsQuery}, round(divide(${fn}(pageLoad), 1000), 2) as count ${subQuery} ${EXCLUDE_NULL_FOR.includes(col) ? `AND ${col} IS NOT NULL` : ''} GROUP BY ${columnsQuery}`
+ return `SELECT ${columnsQuery}, round(divide(${fn}(pageLoad), 1000), 2) as count ${subQuery} ${EXCLUDE_NULL_FOR.includes(sqlCol) ? `AND ${sqlCol} IS NOT NULL` : ''} GROUP BY ${columnsQuery}`
}
if (type === 'errors') {
- return `SELECT ${columnsQuery}, count(*) as count ${subQuery} ${EXCLUDE_NULL_FOR.includes(col) ? `AND ${col} IS NOT NULL` : ''} GROUP BY ${columnsQuery}`
+ return `SELECT ${columnsQuery}, count(*) as count ${subQuery} ${EXCLUDE_NULL_FOR.includes(sqlCol) ? `AND ${sqlCol} IS NOT NULL` : ''} GROUP BY ${columnsQuery}`
}
if (type === 'captcha') {
@@ -330,6 +339,13 @@ export enum DataType {
CAPTCHA = 'captcha',
}
+type AnalyticsEventType = 'pageview' | 'custom_event'
+type EventsAllTimeType =
+ | AnalyticsEventType
+ | 'performance'
+ | 'error'
+ | 'captcha'
+
const isValidOrigin = (origins: string[], origin: string) => {
const escapeRegex = (str: string) =>
str.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')
@@ -665,7 +681,9 @@ export class AnalyticsService {
)
}
- const query = `SELECT ${type} FROM errors WHERE pid={pid:FixedString(12)} AND ${type} IS NOT NULL GROUP BY ${type}`
+ const sqlCol = mapErrorColumn(type)
+
+ const query = `SELECT ${sqlCol} AS ${type} FROM events WHERE pid={pid:FixedString(12)} AND type = 'error' AND ${sqlCol} IS NOT NULL GROUP BY ${sqlCol}`
const { data } = await clickhouse
.query({
@@ -684,13 +702,13 @@ export class AnalyticsService {
type: 'traffic' | 'errors',
column: 'br' | 'os',
): Promise<Array<{ name: string; version: string }>> {
- const safeTable = type === 'errors' ? 'errors' : 'analytics'
+ const safeType = type === 'errors' ? 'error' : 'pageview'
const safeVersionCol = column === 'br' ? 'brv' : 'osv'
const query = `
- SELECT ${column}, ${safeVersionCol}
- FROM ${safeTable}
- WHERE pid={pid:FixedString(12)} AND ${column} IS NOT NULL AND ${safeVersionCol} IS NOT NULL
+ SELECT ${column}, ${safeVersionCol}
+ FROM events
+ WHERE pid={pid:FixedString(12)} AND type = '${safeType}' AND ${column} IS NOT NULL AND ${safeVersionCol} IS NOT NULL
GROUP BY ${column}, ${safeVersionCol}
`
@@ -723,6 +741,22 @@ export class AnalyticsService {
return CAPTCHA_COLUMNS
}
+ getAnalyticsEventType(customEVFilterApplied: boolean): AnalyticsEventType {
+ return customEVFilterApplied ? 'custom_event' : 'pageview'
+ }
+
+ buildAnalyticsEventsSubQuery(
+ filtersQuery: string,
+ customEVFilterApplied: boolean,
+ isCaptcha = false,
+ ): string {
+ const eventType = isCaptcha
+ ? 'captcha'
+ : this.getAnalyticsEventType(customEVFilterApplied)
+
+ return `FROM events WHERE pid = {pid:FixedString(12)} AND type = '${eventType}' ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String}`
+ }
+
getGroupFromTo(
from: string,
to: string,
@@ -871,9 +905,10 @@ export class AnalyticsService {
pg,
created,
lagInFrame(pg) OVER (PARTITION BY psid ORDER BY created) AS prev_page
- FROM analytics
+ FROM events
WHERE
pid = {pid:FixedString(12)}
+ AND type = 'pageview'
AND created BETWEEN {groupFrom:String} AND {groupTo:String}
AND pg IS NOT NULL
${filtersQuery}
@@ -986,15 +1021,17 @@ export class AnalyticsService {
async calculateTimeBucketForAllTime(
pid: string,
- table: 'analytics' | 'customEV' | 'performance' | 'errors' | 'captcha',
+ eventTypes: EventsAllTimeType | readonly EventsAllTimeType[],
): Promise<{
timeBucket: TimeBucketType[]
diff: number
}> {
+ const types = Array.isArray(eventTypes) ? [...eventTypes] : [eventTypes]
+
const { data: fromData } = await clickhouse
.query({
- query: `SELECT min(created) AS firstCreated FROM ${table} WHERE pid = {pid:FixedString(12)}`,
- query_params: { pid },
+ query: `SELECT min(created) AS firstCreated FROM events WHERE pid = {pid:FixedString(12)} AND type IN ({types:Array(String)})`,
+ query_params: { pid, types },
})
.then((res) => res.json<{ firstCreated?: string }>())
@@ -1187,7 +1224,8 @@ export class AnalyticsService {
}
const { filter, isExclusive, isContains } = converted[column][f]
- let sqlColumn = column
+ let sqlColumn =
+ dataType === DataType.ERRORS ? mapErrorColumn(column) : column
let isArrayDataset = false
const param = `qf_${col}_${f}`
@@ -1228,8 +1266,8 @@ export class AnalyticsService {
? 'argMin(pg, created)'
: 'argMax(pg, created)'
const subQueryForPages = isContains
- ? `SELECT psid FROM (SELECT psid, ${pageSelector} as page FROM analytics WHERE pid = {pid:FixedString(12)} GROUP BY psid) WHERE page ILIKE concat('%', {${param}:String}, '%')`
- : `SELECT psid FROM (SELECT psid, ${pageSelector} as page FROM analytics WHERE pid = {pid:FixedString(12)} GROUP BY psid) WHERE page = {${param}:String}`
+ ? `SELECT psid FROM (SELECT psid, ${pageSelector} as page FROM events WHERE pid = {pid:FixedString(12)} AND type = 'pageview' GROUP BY psid) WHERE page ILIKE concat('%', {${param}:String}, '%')`
+ : `SELECT psid FROM (SELECT psid, ${pageSelector} as page FROM events WHERE pid = {pid:FixedString(12)} AND type = 'pageview' GROUP BY psid) WHERE page = {${param}:String}`
// For exclusive filter (isNot) we exclude sessions with matching entry/exit page
query += `psid ${isExclusive ? 'NOT IN' : 'IN'} (${subQueryForPages})`
@@ -1270,6 +1308,9 @@ export class AnalyticsService {
) {
sqlColumn = 'meta.value'
isArrayDataset = true
+ } else if (column === 'ev') {
+ // Public filter contract `ev` maps to the renamed `event_name` column
+ sqlColumn = 'event_name'
}
const isNullFilter =
@@ -1586,21 +1627,11 @@ export class AnalyticsService {
FROM (
SELECT
psid,
- pg AS value,
+ if(type = 'pageview', pg, event_name) AS value,
created
- FROM analytics
- WHERE pid = {pid:FixedString(12)}
- AND psid != 0
- AND created BETWEEN {groupFrom:String} AND {groupTo:String}
-
- UNION ALL
-
- SELECT
- psid,
- ev AS value,
- created
- FROM customEV
+ FROM events
WHERE pid = {pid:FixedString(12)}
+ AND type IN ('pageview', 'custom_event')
AND psid != 0
AND created BETWEEN {groupFrom:String} AND {groupTo:String}
)
@@ -1648,14 +1679,14 @@ export class AnalyticsService {
psid,
CAST(windowFunnel(86400)(created, ${pagesStr}) AS UInt64) AS level
FROM (
- SELECT psid, pg AS value, created
- FROM analytics
- WHERE pid = {pid:FixedString(12)} AND psid != 0
- AND created BETWEEN {groupFrom:String} AND {groupTo:String}
- UNION ALL
- SELECT psid, ev AS value, created
- FROM customEV
- WHERE pid = {pid:FixedString(12)} AND psid != 0
+ SELECT
+ psid,
+ if(type = 'pageview', pg, event_name) AS value,
+ created
+ FROM events
+ WHERE pid = {pid:FixedString(12)}
+ AND type IN ('pageview', 'custom_event')
+ AND psid != 0
AND created BETWEEN {groupFrom:String} AND {groupTo:String}
)
GROUP BY psid
@@ -1670,17 +1701,11 @@ export class AnalyticsService {
psid,
argMin(cc, created) AS cc,
argMin(if(domain(ref) != '', domain(ref), 'Direct / None'), created) AS source
- FROM (
- SELECT psid, cc, ref, created
- FROM analytics
- WHERE pid = {pid:FixedString(12)} AND psid != 0
- AND created BETWEEN {groupFrom:String} AND {groupTo:String}
- UNION ALL
- SELECT psid, cc, ref, created
- FROM customEV
- WHERE pid = {pid:FixedString(12)} AND psid != 0
- AND created BETWEEN {groupFrom:String} AND {groupTo:String}
- )
+ FROM events
+ WHERE pid = {pid:FixedString(12)}
+ AND type IN ('pageview', 'custom_event')
+ AND psid != 0
+ AND created BETWEEN {groupFrom:String} AND {groupTo:String}
GROUP BY psid
)
SELECT step, type, val, cnt FROM (
@@ -1758,14 +1783,14 @@ export class AnalyticsService {
psid,
windowFunnel(86400)(created, ${pagesStr}) AS level
FROM (
- SELECT psid, pg AS value, created
- FROM analytics
- WHERE pid = {pid:FixedString(12)} AND psid != 0
- AND created BETWEEN {groupFrom:String} AND {groupTo:String}
- UNION ALL
- SELECT psid, ev AS value, created
- FROM customEV
- WHERE pid = {pid:FixedString(12)} AND psid != 0
+ SELECT
+ psid,
+ if(type = 'pageview', pg, event_name) AS value,
+ created
+ FROM events
+ WHERE pid = {pid:FixedString(12)}
+ AND type IN ('pageview', 'custom_event')
+ AND psid != 0
AND created BETWEEN {groupFrom:String} AND {groupTo:String}
)
GROUP BY psid
@@ -1781,19 +1806,12 @@ export class AnalyticsService {
any(br) AS br,
min(toTimeZone(created, {timezone:String})) AS sessionStart,
max(toTimeZone(created, {timezone:String})) AS lastActivity
- FROM (
- SELECT psid, pid, cc, os, br, created
- FROM analytics
- WHERE pid = {pid:FixedString(12)} AND psid IS NOT NULL
- AND created BETWEEN {groupFrom:String} AND {groupTo:String}
- AND psid IN (SELECT psid FROM funnel_qualified)
- UNION ALL
- SELECT psid, pid, cc, os, br, created
- FROM customEV
- WHERE pid = {pid:FixedString(12)} AND psid IS NOT NULL
+ FROM events
+ WHERE pid = {pid:FixedString(12)}
+ AND type IN ('pageview', 'custom_event')
+ AND psid IS NOT NULL
AND created BETWEEN {groupFrom:String} AND {groupTo:String}
AND psid IN (SELECT psid FROM funnel_qualified)
- )
GROUP BY psidCasted, pid
),
pageview_counts AS (
@@ -1801,8 +1819,8 @@ export class AnalyticsService {
CAST(psid, 'String') AS psidCasted,
pid,
count() as count
- FROM analytics
- WHERE pid = {pid:FixedString(12)} AND psid IS NOT NULL
+ FROM events
+ WHERE pid = {pid:FixedString(12)} AND type = 'pageview' AND psid IS NOT NULL
AND created BETWEEN {groupFrom:String} AND {groupTo:String}
AND psid IN (SELECT psid FROM funnel_qualified)
GROUP BY psidCasted, pid
@@ -1812,8 +1830,8 @@ export class AnalyticsService {
CAST(psid, 'String') AS psidCasted,
pid,
count() as count
- FROM customEV
- WHERE pid = {pid:FixedString(12)} AND psid IS NOT NULL
+ FROM events
+ WHERE pid = {pid:FixedString(12)} AND type = 'custom_event' AND psid IS NOT NULL
AND created BETWEEN {groupFrom:String} AND {groupTo:String}
AND psid IN (SELECT psid FROM funnel_qualified)
GROUP BY psidCasted, pid
@@ -1823,8 +1841,8 @@ export class AnalyticsService {
CAST(psid, 'String') AS psidCasted,
pid,
count() as count
- FROM errors
- WHERE pid = {pid:FixedString(12)} AND psid IS NOT NULL
+ FROM events
+ WHERE pid = {pid:FixedString(12)} AND type = 'error' AND psid IS NOT NULL
AND created BETWEEN {groupFrom:String} AND {groupTo:String}
AND psid IN (SELECT psid FROM funnel_qualified)
GROUP BY psidCasted, pid
@@ -1860,7 +1878,7 @@ export class AnalyticsService {
COALESCE(errc.count, 0) AS errors,
dsf.sessionStart,
dsf.lastActivity,
- if(dateDiff('second', dsf.lastActivity, now()) < ${LIVE_SESSION_THRESHOLD_SECONDS}, 1, 0) AS isLive,
+ if(dateDiff('second', toTimeZone(dsf.lastActivity, 'UTC'), toTimeZone(now(), 'UTC')) < ${LIVE_SESSION_THRESHOLD_SECONDS}, 1, 0) AS isLive,
sda.avg_duration AS sdur,
sda.profileId AS profileId,
if(startsWith(sda.profileId, '${AnalyticsService.PROFILE_PREFIX_USER}'), 1, 0) AS isIdentified,
@@ -1902,8 +1920,9 @@ export class AnalyticsService {
const query = `
SELECT
count() as c
- FROM analytics
+ FROM events
WHERE pid = {pid:FixedString(12)}
+ AND type = 'pageview'
AND created BETWEEN {groupFrom:String} AND {groupTo:String}
`
const { data } = await clickhouse
@@ -1954,15 +1973,18 @@ export class AnalyticsService {
const promises = pids.map(async (pid) => {
try {
if (period === 'all') {
- let queryAll = `
+ const allTimeType = this.getAnalyticsEventType(customEVFilterApplied)
+
+ const queryAll = `
WITH analytics_counts AS (
SELECT
count(*) AS all,
count(DISTINCT psid) AS unique,
count(DISTINCT profileId) AS users
- FROM analytics
+ FROM events
WHERE
pid = {pid:FixedString(12)}
+ AND type = '${allTimeType}'
${filtersQuery}
),
duration_avg AS (
@@ -1973,6 +1995,14 @@ export class AnalyticsService {
dateDiff('second', min(firstSeen), max(lastSeen)) as duration
FROM sessions
WHERE pid = {pid:FixedString(12)}
+ AND psid IN (
+ SELECT DISTINCT psid
+ FROM events
+ WHERE pid = {pid:FixedString(12)}
+ AND type = '${allTimeType}'
+ AND psid IS NOT NULL
+ ${filtersQuery}
+ )
GROUP BY psid
)
)
@@ -1982,36 +2012,6 @@ export class AnalyticsService {
FROM analytics_counts, duration_avg
`
- if (customEVFilterApplied) {
- queryAll = `
- WITH analytics_counts AS (
- SELECT
- count(*) AS all,
- count(DISTINCT psid) AS unique,
- count(DISTINCT profileId) AS users
- FROM customEV
- WHERE
- pid = {pid:FixedString(12)}
- ${filtersQuery}
- ),
- duration_avg AS (
- SELECT avgOrNull(duration) as sdur
- FROM (
- SELECT
- psid,
- dateDiff('second', min(firstSeen), max(lastSeen)) as duration
- FROM sessions
- WHERE pid = {pid:FixedString(12)}
- GROUP BY psid
- )
- )
- SELECT
- analytics_counts.*,
- duration_avg.sdur
- FROM analytics_counts, duration_avg
- `
- }
-
const { data } = await clickhouse
.query({
query: queryAll,
@@ -2055,7 +2055,7 @@ export class AnalyticsService {
const { diff: allDiff, timeBucket: allowedBuckets } =
await this.calculateTimeBucketForAllTime(
pid,
- customEVFilterApplied ? 'customEV' : 'analytics',
+ this.getAnalyticsEventType(customEVFilterApplied),
)
const allTimeChartBucket = _includes(
allowedBuckets,
@@ -2096,16 +2096,19 @@ export class AnalyticsService {
)
.format('YYYY-MM-DD HH:mm:ss')
- let queryCurrent = `
+ const periodType = this.getAnalyticsEventType(customEVFilterApplied)
+
+ const queryCurrent = `
WITH analytics_counts AS (
SELECT
1 AS sortOrder,
count(*) AS all,
count(DISTINCT psid) AS unique,
count(DISTINCT profileId) AS users
- FROM analytics
+ FROM events
WHERE
pid = {pid:FixedString(12)}
+ AND type = '${periodType}'
AND created BETWEEN {groupFromUTC:String} AND {groupToUTC:String}
${filtersQuery}
),
@@ -2119,8 +2122,9 @@ export class AnalyticsService {
WHERE pid = {pid:FixedString(12)}
AND psid IN (
SELECT DISTINCT psid
- FROM analytics
+ FROM events
WHERE pid = {pid:FixedString(12)}
+ AND type = '${periodType}'
AND psid IS NOT NULL
AND created BETWEEN {groupFromUTC:String} AND {groupToUTC:String}
${filtersQuery}
@@ -2134,16 +2138,17 @@ export class AnalyticsService {
FROM analytics_counts, duration_avg
`
- let queryPrevious = `
+ const queryPrevious = `
WITH analytics_counts AS (
SELECT
2 AS sortOrder,
count(*) AS all,
count(DISTINCT psid) AS unique,
count(DISTINCT profileId) AS users
- FROM analytics
+ FROM events
WHERE
pid = {pid:FixedString(12)}
+ AND type = '${periodType}'
AND created BETWEEN {periodSubtracted:String} AND {groupFromUTC:String}
${filtersQuery}
),
@@ -2157,8 +2162,9 @@ export class AnalyticsService {
WHERE pid = {pid:FixedString(12)}
AND psid IN (
SELECT DISTINCT psid
- FROM analytics
+ FROM events
WHERE pid = {pid:FixedString(12)}
+ AND type = '${periodType}'
AND psid IS NOT NULL
AND created BETWEEN {periodSubtracted:String} AND {groupFromUTC:String}
${filtersQuery}
@@ -2172,83 +2178,6 @@ export class AnalyticsService {
FROM analytics_counts, duration_avg
`
- if (customEVFilterApplied) {
- queryCurrent = `
- WITH analytics_counts AS (
- SELECT
- 1 AS sortOrder,
- count(*) AS all,
- count(DISTINCT psid) AS unique,
- count(DISTINCT profileId) AS users
- FROM customEV
- WHERE
- pid = {pid:FixedString(12)}
- AND created BETWEEN {groupFromUTC:String} AND {groupToUTC:String}
- ${filtersQuery}
- ),
- duration_avg AS (
- SELECT avgOrNull(duration) as sdur
- FROM (
- SELECT
- psid,
- dateDiff('second', min(firstSeen), max(lastSeen)) as duration
- FROM sessions
- WHERE pid = {pid:FixedString(12)}
- AND psid IN (
- SELECT DISTINCT psid
- FROM customEV
- WHERE pid = {pid:FixedString(12)}
- AND psid IS NOT NULL
- AND created BETWEEN {groupFromUTC:String} AND {groupToUTC:String}
- ${filtersQuery}
- )
- GROUP BY psid
- )
- )
- SELECT
- analytics_counts.*,
- duration_avg.sdur
- FROM analytics_counts, duration_avg
- `
- queryPrevious = `
- WITH analytics_counts AS (
- SELECT
- 2 AS sortOrder,
- count(*) AS all,
- count(DISTINCT psid) AS unique,
- count(DISTINCT profileId) AS users
- FROM customEV
- WHERE
- pid = {pid:FixedString(12)}
- AND created BETWEEN {periodSubtracted:String} AND {groupFromUTC:String}
- ${filtersQuery}
- ),
- duration_avg AS (
- SELECT avgOrNull(duration) as sdur
- FROM (
- SELECT
- psid,
- dateDiff('second', min(firstSeen), max(lastSeen)) as duration
- FROM sessions
- WHERE pid = {pid:FixedString(12)}
- AND psid IN (
- SELECT DISTINCT psid
- FROM customEV
- WHERE pid = {pid:FixedString(12)}
- AND psid IS NOT NULL
- AND created BETWEEN {periodSubtracted:String} AND {groupFromUTC:String}
- ${filtersQuery}
- )
- GROUP BY psid
- )
- )
- SELECT
- analytics_counts.*,
- duration_avg.sdur
- FROM analytics_counts, duration_avg
- `
- }
-
const query = `${queryCurrent} UNION ALL ${queryPrevious}`
let { data } = await clickhouse
@@ -2343,6 +2272,63 @@ export class AnalyticsService {
return result
}
+ private generateAnalyticsAggregationQueryForScope(
+ timeBucket: TimeBucketType,
+ filtersQuery: string,
+ mode: ChartRenderMode,
+ customEVFilterApplied: boolean,
+ ): string {
+ return customEVFilterApplied
+ ? this.generateCustomEventsAggregationQuery(
+ timeBucket,
+ filtersQuery,
+ mode,
+ )
+ : this.generateAnalyticsAggregationQuery(timeBucket, filtersQuery, mode)
+ }
+
+ private extractAnalyticsChartDataForScope(
+ result: Array<TrafficCHResponse | TrafficCEFilterCHResponse>,
+ xShifted: string[],
+ customEVFilterApplied: boolean,
+ mode: ChartRenderMode,
+ ): IExtractChartData {
+ if (customEVFilterApplied) {
+ const customEventsData = this.extractCustomEventsChartData(
+ result,
+ xShifted,
+ )
+ const uniques =
+ customEventsData._uniques ||
+ customEventsData._unknown_event ||
+ Array(_size(xShifted)).fill(0)
+ const visits = customEventsData._count || Array(_size(xShifted)).fill(0)
+ const sdur = customEventsData._sdur || Array(_size(xShifted)).fill(0)
+
+ return {
+ visits,
+ uniques,
+ sdur,
+ }
+ }
+
+ const chartData = this.extractChartData(result, xShifted)
+
+ // Propagate the previous cumulative value forward for empty buckets
+ if (mode === ChartRenderMode.CUMULATIVE) {
+ for (let i = 1; i < chartData.visits.length; ++i) {
+ if (chartData.visits[i] === 0) {
+ chartData.visits[i] = chartData.visits[i - 1]
+ }
+ if (chartData.uniques[i] === 0) {
+ chartData.uniques[i] = chartData.uniques[i - 1]
+ }
+ }
+ }
+
+ return chartData
+ }
+
/**
* Get simplified chart data for dashboard cards
* Returns only x (dates) and visits (pageviews) for a lightweight chart
@@ -2373,30 +2359,11 @@ export class AnalyticsService {
},
}
- if (customEVFilterApplied) {
- const query = this.generateCustomEventsAggregationQuery(
- timeBucket,
- filtersQuery,
- ChartRenderMode.PERIODICAL,
- )
-
- const { data } = await clickhouse
- .query({
- query,
- query_params: { ...paramsData.params, timezone: safeTimezone },
- })
- .then((resultSet) => resultSet.json<TrafficCEFilterCHResponse>())
-
- const visits =
- this.extractCustomEventsChartData(data, xShifted)?._unknown_event || []
-
- return { x: xShifted, visits }
- }
-
- const query = this.generateAnalyticsAggregationQuery(
+ const query = this.generateAnalyticsAggregationQueryForScope(
timeBucket,
filtersQuery,
ChartRenderMode.PERIODICAL,
+ customEVFilterApplied,
)
const { data } = await clickhouse
@@ -2404,9 +2371,16 @@ export class AnalyticsService {
query,
query_params: { ...paramsData.params, timezone: safeTimezone },
})
- .then((resultSet) => resultSet.json<TrafficCHResponse>())
+ .then((resultSet) =>
+ resultSet.json<TrafficCHResponse | TrafficCEFilterCHResponse>(),
+ )
- const { visits } = this.extractChartData(data, xShifted)
+ const { visits } = this.extractAnalyticsChartDataForScope(
+ data,
+ xShifted,
+ customEVFilterApplied,
+ ChartRenderMode.PERIODICAL,
+ )
return { x: xShifted, visits }
}
@@ -2456,7 +2430,7 @@ export class AnalyticsService {
const promises = pids.map(async (pid) => {
try {
if (period === 'all') {
- const queryAll = `SELECT ${columnSelectors} FROM performance WHERE pid = {pid:FixedString(12)} ${filtersQuery}`
+ const queryAll = `SELECT ${columnSelectors} FROM events WHERE pid = {pid:FixedString(12)} AND type = 'performance' ${filtersQuery}`
const { data } = await clickhouse
.query({
query: queryAll,
@@ -2518,8 +2492,8 @@ export class AnalyticsService {
.format('YYYY-MM-DD HH:mm:ss')
}
- const queryCurrent = `SELECT 1 AS sortOrder, ${columnSelectors} FROM performance WHERE pid = {pid:FixedString(12)} AND created BETWEEN {periodFormatted:String} AND {now:String} ${filtersQuery}`
- const queryPrevious = `SELECT 2 AS sortOrder, ${columnSelectors} FROM performance WHERE pid = {pid:FixedString(12)} AND created BETWEEN {periodSubtracted:String} AND {periodFormatted:String} ${filtersQuery}`
+ const queryCurrent = `SELECT 1 AS sortOrder, ${columnSelectors} FROM events WHERE pid = {pid:FixedString(12)} AND type = 'performance' AND created BETWEEN {periodFormatted:String} AND {now:String} ${filtersQuery}`
+ const queryPrevious = `SELECT 2 AS sortOrder, ${columnSelectors} FROM events WHERE pid = {pid:FixedString(12)} AND type = 'performance' AND created BETWEEN {periodSubtracted:String} AND {periodFormatted:String} ${filtersQuery}`
const query = `${queryCurrent} UNION ALL ${queryPrevious}`
@@ -2616,8 +2590,8 @@ export class AnalyticsService {
const query = `
WITH session_pages AS (
SELECT psid, ${selector} as page
- FROM analytics
- WHERE pid = {pid:FixedString(12)}
+ FROM events
+ WHERE pid = {pid:FixedString(12)} AND type = 'pageview'
GROUP BY psid
)
SELECT page FROM session_pages WHERE page IS NOT NULL GROUP BY page
@@ -2633,11 +2607,10 @@ export class AnalyticsService {
return _map(data, 'page')
}
- let query = `SELECT ${type} FROM analytics WHERE pid={pid:FixedString(12)} AND ${type} IS NOT NULL GROUP BY ${type}`
+ let query = `SELECT ${type} FROM events WHERE pid={pid:FixedString(12)} AND type = 'pageview' AND ${type} IS NOT NULL GROUP BY ${type}`
if (type === 'ev') {
- query =
- 'SELECT ev FROM customEV WHERE pid={pid:FixedString(12)} AND ev IS NOT NULL GROUP BY ev'
+ query = `SELECT event_name AS ev FROM events WHERE pid={pid:FixedString(12)} AND type = 'custom_event' AND event_name IS NOT NULL GROUP BY event_name`
}
const { data } = await clickhouse
@@ -2695,7 +2668,7 @@ export class AnalyticsService {
const query = `
WITH ${withClauses.join(',\n')}
- SELECT
+ SELECT
column_name,
name,
count,
@@ -2716,7 +2689,7 @@ export class AnalyticsService {
}
return `
- SELECT
+ SELECT
'${col}' as column_name,
name,
count,
@@ -2914,9 +2887,15 @@ export class AnalyticsService {
extractCustomEventsChartData(queryResult, x: string[]): any {
const result = {}
+ const uniques = Array(x.length).fill(0)
+ const sdur = Array(x.length).fill(0)
+ let hasUniques = false
+ let hasSdur = false
for (let row = 0; row < _size(queryResult); ++row) {
- const { ev = '_unknown_event' } = queryResult[row]
+ const rowEventName = queryResult[row].ev
+ const ev =
+ typeof rowEventName === 'undefined' ? '_unknown_event' : rowEventName
const dateString = this.generateDateString(queryResult[row])
@@ -2928,9 +2907,35 @@ export class AnalyticsService {
}
result[ev][index] = queryResult[row].count
+
+ if (typeof rowEventName === 'undefined') {
+ if (!result['_count']) {
+ result['_count'] = Array(x.length).fill(0)
+ }
+
+ result['_count'][index] = queryResult[row].count || 0
+ }
+
+ if (typeof queryResult[row].uniques !== 'undefined') {
+ uniques[index] = queryResult[row].uniques || 0
+ hasUniques = true
+ }
+
+ if (typeof queryResult[row].sdur !== 'undefined') {
+ sdur[index] = queryResult[row].sdur || 0
+ hasSdur = true
+ }
}
}
+ if (hasUniques) {
+ result['_uniques'] = uniques
+ }
+
+ if (hasSdur) {
+ result['_sdur'] = sdur
+ }
+
return result
}
@@ -3005,8 +3010,8 @@ export class AnalyticsService {
pid,
psid,
${timeBucketFunc}(toTimeZone(created, {timezone:String})) as tz_created
- FROM analytics
- PREWHERE pid = {pid:FixedString(12)}
+ FROM events
+ PREWHERE pid = {pid:FixedString(12)} AND type = 'pageview'
WHERE created BETWEEN ${tzFromDate} AND ${tzToDate}
${filtersQuery}
) as subquery
@@ -3054,8 +3059,8 @@ export class AnalyticsService {
FROM (
SELECT
${timeBucketFunc}(toTimeZone(created, {timezone:String})) as tz_created
- FROM analytics
- PREWHERE pid = {pid:FixedString(12)}
+ FROM events
+ PREWHERE pid = {pid:FixedString(12)} AND type = 'pageview'
WHERE created BETWEEN ${tzFromDate} AND ${tzToDate}
${filtersQuery}
) as subquery
@@ -3082,8 +3087,8 @@ export class AnalyticsService {
FROM (
SELECT
${timeBucketFunc}(toTimeZone(created, {timezone:String})) as tz_created
- FROM customEV
- PREWHERE pid = {pid:FixedString(12)}
+ FROM events
+ PREWHERE pid = {pid:FixedString(12)} AND type = 'custom_event'
WHERE created BETWEEN ${tzFromDate} AND ${tzToDate}
${filtersQuery}
) as subquery
@@ -3108,8 +3113,8 @@ export class AnalyticsService {
FROM (
SELECT
${timeBucketFunc}(toTimeZone(created, {timezone:String})) as tz_created
- FROM errors
- PREWHERE pid = {pid:FixedString(12)}
+ FROM events
+ PREWHERE pid = {pid:FixedString(12)} AND type = 'error'
WHERE created BETWEEN ${tzFromDate} AND ${tzToDate}
${filtersQuery}
) as subquery
@@ -3131,15 +3136,31 @@ export class AnalyticsService {
const baseQuery = `
SELECT
${selector},
- count() as count
+ count() as count,
+ countDistinct(subquery.psid) as uniques,
+ avgOrNull(sessions_data.duration) as sdur
FROM (
- SELECT *,
- ${timeBucketFunc}(toTimeZone(created, {timezone:String})) as tz_created
- FROM customEV
+ SELECT
+ pid,
+ psid,
+ ${timeBucketFunc}(toTimeZone(created, {timezone:String})) as tz_created
+ FROM events
WHERE pid = {pid:FixedString(12)}
+ AND type = 'custom_event'
AND created BETWEEN ${tzFromDate} AND ${tzToDate}
${filtersQuery}
) as subquery
+ LEFT JOIN (
+ SELECT
+ pid,
+ psid,
+ dateDiff('second', min(firstSeen), max(lastSeen)) as duration
+ FROM sessions
+ WHERE pid = {pid:FixedString(12)}
+ GROUP BY pid, psid
+ ) as sessions_data
+ ON subquery.pid = sessions_data.pid
+ AND subquery.psid = sessions_data.psid
GROUP BY ${groupBy}
ORDER BY ${groupBy}
`
@@ -3147,8 +3168,10 @@ export class AnalyticsService {
if (mode === ChartRenderMode.CUMULATIVE) {
return `
SELECT
- *,
- sum(count) OVER (ORDER BY ${groupBy}) as count
+ ${groupBy},
+ sum(count) OVER (ORDER BY ${groupBy}) as count,
+ sum(uniques) OVER (ORDER BY ${groupBy}) as uniques,
+ sdur
FROM (${baseQuery})
`
}
@@ -3179,9 +3202,10 @@ export class AnalyticsService {
FROM (
SELECT *,
${timeBucketFunc}(toTimeZone(created, {timezone:String})) as tz_created
- FROM performance
+ FROM events
WHERE
pid = {pid:FixedString(12)}
+ AND type = 'performance'
AND created BETWEEN ${tzFromDate} AND ${tzToDate}
${filtersQuery}
) as subquery
@@ -3214,9 +3238,10 @@ export class AnalyticsService {
FROM (
SELECT *,
${timeBucketFunc}(toTimeZone(created, {timezone:String})) as tz_created
- FROM performance
+ FROM events
WHERE
pid = {pid:FixedString(12)}
+ AND type = 'performance'
AND created BETWEEN ${tzFromDate} AND ${tzToDate}
${filtersQuery}
) as subquery
@@ -3274,9 +3299,10 @@ export class AnalyticsService {
FROM (
SELECT pg, dv, br, os, lc, cc, rg, ct, profileId,
${timeBucketFunc}(created) as tz_created
- FROM errors
+ FROM events
WHERE
pid = {pid:FixedString(12)}
+ AND type = 'error'
AND created BETWEEN {groupFrom:String} AND {groupTo:String}
${filtersQuery}
) as subquery
@@ -3287,10 +3313,11 @@ export class AnalyticsService {
if (mode === ChartRenderMode.CUMULATIVE) {
return `
SELECT
- *,
- sum(pageviews) OVER (ORDER BY ${groupBy}) as pageviews,
- sum(uniques) OVER (ORDER BY ${groupBy}) as uniques
+ ${groupBy},
+ sum(count) OVER (ORDER BY ${groupBy}) as count,
+ sum(affectedUsers) OVER (ORDER BY ${groupBy}) as affectedUsers
FROM (${baseQuery})
+ ORDER BY ${groupBy}
`
}
@@ -3355,7 +3382,7 @@ export class AnalyticsService {
): Promise<{ name: string; count: number }[]> {
const query = `
WITH session_first_pages AS (
- SELECT
+ SELECT
psid,
argMin(pg, created) as entry_page
${subQuery}
@@ -3450,39 +3477,11 @@ export class AnalyticsService {
): Promise<object | void> {
const { xShifted } = this.generateXAxis(timeBucket, from, to, safeTimezone)
- if (customEVFilterApplied) {
- const query = this.generateCustomEventsAggregationQuery(
- timeBucket,
- filtersQuery,
- mode,
- )
-
- const { data } = await clickhouse
- .query({
- query,
- query_params: { ...paramsData.params, timezone: safeTimezone },
- })
- .then((resultSet) => resultSet.json<TrafficCEFilterCHResponse>())
-
- const uniques =
- this.extractCustomEventsChartData(data, xShifted)?._unknown_event || []
-
- const sdur = Array(_size(xShifted)).fill(0)
-
- return Promise.resolve({
- chart: {
- x: xShifted,
- visits: uniques,
- uniques,
- sdur,
- },
- })
- }
-
- const query = this.generateAnalyticsAggregationQuery(
+ const query = this.generateAnalyticsAggregationQueryForScope(
timeBucket,
filtersQuery,
mode,
+ customEVFilterApplied,
)
const { data } = await clickhouse
@@ -3490,21 +3489,16 @@ export class AnalyticsService {
query,
query_params: { ...paramsData.params, timezone: safeTimezone },
})
- .then((resultSet) => resultSet.json<TrafficCHResponse>())
-
- const { visits, uniques, sdur } = this.extractChartData(data, xShifted)
+ .then((resultSet) =>
+ resultSet.json<TrafficCHResponse | TrafficCEFilterCHResponse>(),
+ )
- // Propagate the previous cumulative value forward for empty buckets
- if (mode === ChartRenderMode.CUMULATIVE) {
- for (let i = 1; i < visits.length; ++i) {
- if (visits[i] === 0) {
- visits[i] = visits[i - 1]
- }
- if (uniques[i] === 0) {
- uniques[i] = uniques[i - 1]
- }
- }
- }
+ const { visits, uniques, sdur } = this.extractAnalyticsChartDataForScope(
+ data,
+ xShifted,
+ customEVFilterApplied,
+ mode,
+ )
return Promise.resolve({
chart: {
@@ -3798,9 +3792,10 @@ export class AnalyticsService {
FROM (
SELECT *,
${timeBucketFunc}(toTimeZone(created, {timezone:String})) as tz_created
- FROM captcha
+ FROM events
WHERE
pid = {pid:FixedString(12)}
+ AND type = 'captcha'
AND created BETWEEN ${tzFromDate} AND ${tzToDate}
${filtersQuery}
) as subquery
@@ -4050,7 +4045,7 @@ export class AnalyticsService {
filtersQuery: string,
params: any,
): Promise<ICustomEvent> {
- const query = `SELECT ev, count() FROM customEV WHERE pid = {pid:FixedString(12)} ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String} GROUP BY ev`
+ const query = `SELECT event_name AS ev, count() FROM events WHERE pid = {pid:FixedString(12)} AND type = 'custom_event' ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String} GROUP BY event_name`
const result = {}
const { data } = await clickhouse
@@ -4081,9 +4076,10 @@ export class AnalyticsService {
SELECT
meta.key AS key
FROM
- analytics
+ events
WHERE
pid = {pid:FixedString(12)}
+ AND type = 'pageview'
${filtersQuery}
AND created BETWEEN {groupFrom:String} AND {groupTo:String}
)
@@ -4137,7 +4133,7 @@ export class AnalyticsService {
)
if (period === 'all') {
- const res = await this.calculateTimeBucketForAllTime(pid, 'customEV')
+ const res = await this.calculateTimeBucketForAllTime(pid, 'custom_event')
diff = res.diff
@@ -4165,12 +4161,12 @@ export class AnalyticsService {
SELECT
meta.key,
meta.value
- FROM
- customEV
+ FROM events
WHERE
pid = {pid:FixedString(12)}
+ AND type = 'custom_event'
AND created BETWEEN {groupFrom:String} AND {groupTo:String}
- AND ev = {event:String}
+ AND event_name = {event:String}
${filtersQuery}
)
ARRAY JOIN meta.key, meta.value
@@ -4229,7 +4225,7 @@ export class AnalyticsService {
const [filtersQuery, filtersParams, appliedFilters, customEVFilterApplied] =
this.getFiltersQuery(filters, DataType.ANALYTICS)
- // We cannot make a query to customEV table using analytics table properties
+ // Page properties are only present on pageview-scoped analytics rows.
if (customEVFilterApplied) {
return {
result: [],
@@ -4238,7 +4234,7 @@ export class AnalyticsService {
}
if (period === 'all') {
- const res = await this.calculateTimeBucketForAllTime(pid, 'analytics')
+ const res = await this.calculateTimeBucketForAllTime(pid, 'pageview')
diff = res.diff
@@ -4266,10 +4262,10 @@ export class AnalyticsService {
SELECT
meta.key,
meta.value
- FROM
- analytics
+ FROM events
WHERE
pid = {pid:FixedString(12)}
+ AND type = 'pageview'
AND created BETWEEN {groupFrom:String} AND {groupTo:String}
AND indexOf(meta.key, {property:String}) > 0
${filtersQuery}
@@ -4319,17 +4315,11 @@ export class AnalyticsService {
const query = `
SELECT uniqExact(psid) as count
- FROM (
- SELECT psid FROM analytics
- WHERE pid = {pid:FixedString(12)}
- AND created >= {since:DateTime}
- AND psid IS NOT NULL
- UNION ALL
- SELECT psid FROM customEV
- WHERE pid = {pid:FixedString(12)}
- AND created >= {since:DateTime}
- AND psid IS NOT NULL
- )
+ FROM events
+ WHERE pid = {pid:FixedString(12)}
+ AND type IN ('pageview', 'custom_event', 'error')
+ AND created >= {since:DateTime}
+ AND psid IS NOT NULL
`
try {
@@ -4367,25 +4357,26 @@ export class AnalyticsService {
const tzToDate = `toTimeZone(parseDateTimeBestEffort({groupTo:String}), {timezone:String})`
const customEventsFilter =
customEvents && customEvents.length > 0
- ? 'AND ev IN {customEvents:Array(String)}'
+ ? 'AND event_name IN {customEvents:Array(String)}'
: ''
const query = `
SELECT
${selector},
- ev,
+ event_name AS ev,
count() as count
FROM (
SELECT *,
${timeBucketFunc}(toTimeZone(created, {timezone:String})) as tz_created
- FROM customEV
+ FROM events
WHERE
pid = {pid:FixedString(12)}
+ AND type = 'custom_event'
AND created BETWEEN ${tzFromDate} AND ${tzToDate}
${filtersQuery}
${customEventsFilter}
) as subquery
- GROUP BY ${groupBy}, ev
+ GROUP BY ${groupBy}, event_name
ORDER BY ${groupBy}
`
@@ -4430,16 +4421,21 @@ export class AnalyticsService {
}
return _map(pages, (page: IPageflow) => {
- if (!page.metadata) {
- return page
+ const { created_utc: _createdUtc, ...displayPage } = page
+
+ if (!displayPage.metadata) {
+ return displayPage
}
return {
- ...page,
- metadata: _map(page.metadata, ([key, value]: [string, string]) => ({
- key,
- value,
- })),
+ ...displayPage,
+ metadata: _map(
+ displayPage.metadata,
+ ([key, value]: [string, string]) => ({
+ key,
+ value,
+ }),
+ ),
}
})
}
@@ -4452,70 +4448,69 @@ export class AnalyticsService {
const queryPages = `
WITH events_with_meta AS (
SELECT
- 'pageview' AS type,
- pg AS value,
- toTimeZone(analytics.created, {timezone:String}) AS created,
- pid,
- psid,
- arrayFilter(x -> x.1 != '' AND x.2 != '', arrayZip(meta.key, meta.value)) AS metadata
- FROM analytics
- WHERE
- pid = {pid:FixedString(12)}
- AND analytics.psid IS NOT NULL
- AND CAST(analytics.psid AS String) = {psid:String}
-
- UNION ALL
-
- SELECT
- 'event' AS type,
- ev AS value,
- toTimeZone(customEV.created, {timezone:String}) AS created,
+ multiIf(event_type = 'pageview', 'pageview', event_type = 'custom_event', 'event', 'error') AS type,
+ multiIf(
+ event_type = 'pageview', toString(pg),
+ event_type = 'custom_event', toString(event_name),
+ toString(error_name)
+ ) AS value,
+ created AS created_utc,
+ toTimeZone(created, {timezone:String}) AS created_local,
pid,
- psid,
- arrayFilter(x -> x.1 != '' AND x.2 != '', arrayZip(meta.key, meta.value)) AS metadata
- FROM customEV
- WHERE
- pid = {pid:FixedString(12)}
- AND customEV.psid IS NOT NULL
- AND CAST(customEV.psid AS String) = {psid:String}
-
- UNION ALL
-
- SELECT
- 'error' AS type,
- errors.name AS value,
- toTimeZone(errors.created, {timezone:String}) AS created,
- pid,
- psid,
- [
- tuple('message', COALESCE(errors.message, '')),
- tuple('lineno', CAST(COALESCE(errors.lineno, 0), 'String')),
- tuple('colno', CAST(COALESCE(errors.colno, 0), 'String')),
- tuple('filename', COALESCE(errors.filename, ''))
- ] AS metadata
- FROM errors
- WHERE
- pid = {pid:FixedString(12)}
- AND errors.psid IS NOT NULL
- AND CAST(errors.psid AS String) = {psid:String}
+ toString(psid) AS psid,
+ if(
+ event_type = 'error',
+ [
+ tuple('message', COALESCE(error_message, '')),
+ tuple('lineno', toString(COALESCE(lineno, 0))),
+ tuple('colno', toString(COALESCE(colno, 0))),
+ tuple('filename', COALESCE(error_filename, ''))
+ ],
+ arrayFilter(x -> x.1 != '' AND x.2 != '', arrayZip(meta_key, meta_value))
+ ) AS metadata
+ FROM (
+ SELECT
+ type AS event_type,
+ pg,
+ event_name,
+ error_name,
+ error_message,
+ lineno,
+ colno,
+ error_filename,
+ created,
+ pid,
+ psid,
+ meta.key AS meta_key,
+ meta.value AS meta_value
+ FROM events
+ WHERE
+ pid = {pid:FixedString(12)}
+ AND type IN ('pageview', 'custom_event', 'error')
+ AND psid IS NOT NULL
+ AND toString(psid) = {psid:String}
+ )
)
SELECT
type,
value,
- created,
+ created_local AS created,
+ created_utc,
metadata
FROM events_with_meta
- ORDER BY created ASC
+ ORDER BY created_utc ASC
`
const querySessionDetails = `
SELECT
dv, br, brv, os, osv, lc, ref, so, me, ca, te, co, cc, rg, ct, profileId
- FROM analytics
+ FROM events
WHERE
pid = {pid:FixedString(12)}
- AND CAST(psid, 'String') = {psid:String}
+ AND type IN ('pageview', 'custom_event', 'error')
+ AND psid IS NOT NULL
+ AND toString(psid) = {psid:String}
ORDER BY created ASC
LIMIT 1;
`
@@ -4581,7 +4576,7 @@ export class AnalyticsService {
}
if (!duration && lastSeen && Array.isArray(pages) && pages.length >= 1) {
- const first = dayjs(pages[0].created)
+ const first = dayjs(pages[0].created_utc)
const diffSeconds = dayjs(lastSeen).diff(first, 'second')
if (diffSeconds > 0) {
duration = diffSeconds
@@ -4589,8 +4584,8 @@ export class AnalyticsService {
}
if (!duration && Array.isArray(pages) && pages.length >= 2) {
- const first = dayjs(pages[0].created)
- const last = dayjs(pages[pages.length - 1].created)
+ const first = dayjs(pages[0].created_utc)
+ const last = dayjs(pages[pages.length - 1].created_utc)
const diffSeconds = last.diff(first, 'second')
if (diffSeconds > 0) {
duration = diffSeconds
@@ -4601,11 +4596,12 @@ export class AnalyticsService {
const querySessionDetailsFromCustomEV = `
SELECT
dv, br, brv, os, osv, lc, ref, so, me, ca, te, co, cc, rg, ct, profileId
- FROM customEV
+ FROM events
WHERE
pid = {pid:FixedString(12)}
+ AND type IN ('custom_event', 'error')
AND psid IS NOT NULL
- AND CAST(psid, 'String') = {psid:String}
+ AND toString(psid) = {psid:String}
ORDER BY created ASC
LIMIT 1;
`
@@ -4626,12 +4622,12 @@ export class AnalyticsService {
if (!_isEmpty(pages)) {
const from = dayjs
- .utc(pages[0].created)
+ .utc(pages[0].created_utc)
.tz(safeTimezone)
.startOf('minute')
.format('YYYY-MM-DD HH:mm:ss')
const to = dayjs
- .utc(pages[_size(pages) - 1].created)
+ .utc(pages[_size(pages) - 1].created_utc)
.tz(safeTimezone)
.endOf('minute')
.format('YYYY-MM-DD HH:mm:ss')
@@ -4649,8 +4645,8 @@ export class AnalyticsService {
{
params: {
...paramsData.params,
- groupFrom: pages[0].created,
- groupTo: pages[_size(pages) - 1].created,
+ groupFrom: from,
+ groupTo: to,
},
},
safeTimezone,
@@ -4663,7 +4659,7 @@ export class AnalyticsService {
let isLive = false
if (!_isEmpty(pages)) {
- const lastActivityTime = dayjs(pages[pages.length - 1].created)
+ const lastActivityTime = dayjs(pages[pages.length - 1].created_utc)
const liveThresholdTime = dayjs().subtract(
LIVE_SESSION_THRESHOLD_SECONDS,
@@ -4693,61 +4689,16 @@ export class AnalyticsService {
skip = 0,
customEVFilterApplied = false,
): Promise<object | void> {
- let primaryEventsSubquery: string
-
- if (customEVFilterApplied) {
- primaryEventsSubquery = `
- SELECT
- CAST(customEV.psid, 'String') AS psidCasted,
- customEV.pid,
- customEV.cc,
- customEV.os,
- customEV.br,
- toTimeZone(customEV.created, {timezone:String}) AS created_for_grouping
- FROM customEV
- WHERE
- customEV.pid = {pid:FixedString(12)}
- AND customEV.psid IS NOT NULL
- AND customEV.created BETWEEN {groupFrom:String} AND {groupTo:String}
- ${filtersQuery}
- `
- } else {
- primaryEventsSubquery = `
- SELECT
- CAST(analytics.psid, 'String') AS psidCasted,
- analytics.pid,
- analytics.cc,
- analytics.os,
- analytics.br,
- toTimeZone(analytics.created, {timezone:String}) AS created_for_grouping
- FROM analytics
- WHERE
- analytics.pid = {pid:FixedString(12)}
- AND analytics.psid IS NOT NULL
- AND analytics.created BETWEEN {groupFrom:String} AND {groupTo:String}
- ${filtersQuery}
- UNION ALL
- SELECT
- CAST(customEV.psid, 'String') AS psidCasted,
- customEV.pid,
- customEV.cc,
- customEV.os,
- customEV.br,
- toTimeZone(customEV.created, {timezone:String}) AS created_for_grouping
- FROM customEV
- WHERE
- customEV.pid = {pid:FixedString(12)}
- AND customEV.psid IS NOT NULL
- AND customEV.created BETWEEN {groupFrom:String} AND {groupTo:String}
- ${filtersQuery}
- `
- }
+ const primaryEventsSubquery = this.buildSessionsListPrimaryEventsSubquery(
+ filtersQuery,
+ customEVFilterApplied,
+ )
const query = `
WITH distinct_sessions_filtered AS (
SELECT
psidCasted,
- pid,
+ pid,
any(cc) AS cc,
any(os) AS os,
any(br) AS br,
@@ -4757,39 +4708,39 @@ export class AnalyticsService {
GROUP BY psidCasted, pid
),
pageview_counts AS (
- SELECT
+ SELECT
CAST(psid, 'String') AS psidCasted,
- pid,
- count() as count
- FROM analytics
- WHERE pid = {pid:FixedString(12)} AND psid IS NOT NULL
+ pid,
+ count() as count
+ FROM events
+ WHERE pid = {pid:FixedString(12)} AND type = 'pageview' AND psid IS NOT NULL
AND created BETWEEN {groupFrom:String} AND {groupTo:String}
GROUP BY psidCasted, pid
),
event_counts AS (
- SELECT
+ SELECT
CAST(psid, 'String') AS psidCasted,
- pid,
- count() as count
- FROM customEV
- WHERE pid = {pid:FixedString(12)} AND psid IS NOT NULL
+ pid,
+ count() as count
+ FROM events
+ WHERE pid = {pid:FixedString(12)} AND type = 'custom_event' AND psid IS NOT NULL
AND created BETWEEN {groupFrom:String} AND {groupTo:String}
GROUP BY psidCasted, pid
),
error_counts AS (
- SELECT
+ SELECT
CAST(psid, 'String') AS psidCasted,
- pid,
- count() as count
- FROM errors
- WHERE pid = {pid:FixedString(12)} AND psid IS NOT NULL
+ pid,
+ count() as count
+ FROM events
+ WHERE pid = {pid:FixedString(12)} AND type = 'error' AND psid IS NOT NULL
AND created BETWEEN {groupFrom:String} AND {groupTo:String}
GROUP BY psidCasted, pid
),
session_duration_agg AS (
- SELECT
- CAST(psid, 'String') AS psidCasted,
- pid,
+ SELECT
+ CAST(psid, 'String') AS psidCasted,
+ pid,
dateDiff('second', min(firstSeen), max(lastSeen)) as avg_duration,
argMax(profileId, lastSeen) as profileId
FROM sessions
@@ -4816,7 +4767,7 @@ export class AnalyticsService {
COALESCE(errc.count, 0) AS errors,
dsf.sessionStart,
dsf.lastActivity,
- if(dateDiff('second', dsf.lastActivity, now()) < ${LIVE_SESSION_THRESHOLD_SECONDS}, 1, 0) AS isLive,
+ if(dateDiff('second', toTimeZone(dsf.lastActivity, 'UTC'), toTimeZone(now(), 'UTC')) < ${LIVE_SESSION_THRESHOLD_SECONDS}, 1, 0) AS isLive,
sda.avg_duration AS sdur,
sda.profileId AS profileId,
if(startsWith(sda.profileId, '${AnalyticsService.PROFILE_PREFIX_USER}'), 1, 0) AS isIdentified,
@@ -4848,6 +4799,43 @@ export class AnalyticsService {
return data
}
+ private buildSessionsListPrimaryEventsSubquery(
+ filtersQuery: string,
+ customEVFilterApplied: boolean,
+ ): string {
+ const scopedSessionFilter = customEVFilterApplied
+ ? `
+ AND CAST(psid, 'String') IN (
+ SELECT DISTINCT CAST(psid, 'String')
+ FROM events
+ WHERE
+ pid = {pid:FixedString(12)}
+ AND type = 'custom_event'
+ AND psid IS NOT NULL
+ AND created BETWEEN {groupFrom:String} AND {groupTo:String}
+ ${filtersQuery}
+ )
+ `
+ : filtersQuery
+
+ return `
+ SELECT
+ CAST(psid, 'String') AS psidCasted,
+ pid,
+ cc,
+ os,
+ br,
+ toTimeZone(created, {timezone:String}) AS created_for_grouping
+ FROM events
+ WHERE
+ pid = {pid:FixedString(12)}
+ AND type IN ('pageview', 'custom_event', 'error')
+ AND psid IS NOT NULL
+ AND created BETWEEN {groupFrom:String} AND {groupTo:String}
+ ${scopedSessionFilter}
+ `
+ }
+
async getErrorsList(
options: string,
filtersQuery: string,
@@ -4878,10 +4866,18 @@ export class AnalyticsService {
count(DISTINCT psid) as sessions,
status.status
FROM (
- SELECT eid, name, message, filename, psid, profileId, toTimeZone(errors.created, {timezone:String}) AS created
- FROM errors
+ SELECT
+ eid,
+ error_name AS name,
+ error_message AS message,
+ error_filename AS filename,
+ psid,
+ profileId,
+ toTimeZone(created, {timezone:String}) AS created
+ FROM events
WHERE pid = {pid:FixedString(12)}
- AND errors.created BETWEEN {groupFrom:String} AND {groupTo:String}
+ AND type = 'error'
+ AND created BETWEEN {groupFrom:String} AND {groupTo:String}
${filtersQuery}
) AS errors
LEFT JOIN (
@@ -4895,7 +4891,7 @@ export class AnalyticsService {
${
parsedOptions?.showResolved
? ''
- : "WHERE status.status = 'active' OR status.status = 'regressed'"
+ : "WHERE status.status = 'active' OR status.status = 'regressed' OR status.status IS NULL"
}
GROUP BY errors.eid, status.status
ORDER BY last_seen DESC
@@ -4940,14 +4936,15 @@ export class AnalyticsService {
FROM (
SELECT
eid,
- name,
- message,
- filename,
+ error_name AS name,
+ error_message AS message,
+ error_filename AS filename,
colno,
lineno,
stackTrace
- FROM errors
+ FROM events
WHERE pid = {pid:FixedString(12)}
+ AND type = 'error'
AND eid = {eid:FixedString(32)}
AND created BETWEEN {groupFrom:String} AND {groupTo:String}
ORDER BY created DESC
@@ -4970,9 +4967,10 @@ export class AnalyticsService {
max(created) AS last_seen,
min(created) AS first_seen,
count() AS count
- FROM errors
+ FROM events
WHERE
pid = {pid:FixedString(12)}
+ AND type = 'error'
AND eid = {eid:FixedString(32)};
`
@@ -4985,9 +4983,10 @@ export class AnalyticsService {
SELECT
meta.key,
meta.value
- FROM errors
+ FROM events
WHERE
pid = {pid:FixedString(12)}
+ AND type = 'error'
AND eid = {eid:FixedString(32)}
AND created BETWEEN {groupFrom:String} AND {groupTo:String}
)
@@ -5034,8 +5033,8 @@ export class AnalyticsService {
timeBucket,
groupFrom,
groupTo,
- `FROM errors WHERE eid = {eid:FixedString(32)} AND created BETWEEN {groupFrom:String} AND {groupTo:String}`,
- 'AND eid = {eid:FixedString(32)}',
+ `FROM events WHERE type = 'error' AND pid = {pid:FixedString(12)} AND eid = {eid:FixedString(32)} AND created BETWEEN {groupFrom:String} AND {groupTo:String}`,
+ 'AND pid = {pid:FixedString(12)} AND eid = {eid:FixedString(32)}',
paramsData,
safeTimezone,
ChartRenderMode.PERIODICAL,
@@ -5061,17 +5060,22 @@ export class AnalyticsService {
groupTo: string,
timeBucket: string,
showResolved: boolean,
+ sessionFiltersQuery = '',
+ sessionFiltersParams: Record<string, unknown> = {},
): Promise<any> {
const resolvedFilter = showResolved
? ''
: "AND (status.status = 'active' OR status.status = 'regressed' OR status.status IS NULL)"
- // Get total sessions from analytics table for the time range
+ // Get total sessions from all matching events for the time range
const queryTotalSessions = `
SELECT count(DISTINCT psid) as totalSessions
- FROM analytics
+ FROM events
WHERE pid = {pid:FixedString(12)}
+ AND type IN ('pageview', 'custom_event', 'error')
+ AND psid IS NOT NULL
AND created BETWEEN {groupFrom:String} AND {groupTo:String}
+ ${sessionFiltersQuery}
`
// Get error stats: total errors, unique errors, affected sessions, affected users
@@ -5081,7 +5085,7 @@ export class AnalyticsService {
count(DISTINCT eid) as uniqueErrors,
count(DISTINCT psid) as affectedSessions,
count(DISTINCT profileId) as affectedUsers
- FROM errors
+ FROM events AS errors
LEFT JOIN (
SELECT eid, argMax(status, updated) AS status
FROM error_statuses
@@ -5089,6 +5093,8 @@ export class AnalyticsService {
GROUP BY eid
) AS status ON errors.eid = status.eid
WHERE pid = {pid:FixedString(12)}
+ AND type = 'error'
+ AND psid IS NOT NULL
AND created BETWEEN {groupFrom:String} AND {groupTo:String}
${filtersQuery}
${resolvedFilter}
@@ -5098,12 +5104,12 @@ export class AnalyticsService {
const queryMostFrequentError = `
SELECT
errors.eid as eid,
- any(errors.name) as name,
- any(errors.message) as message,
+ any(errors.error_name) as name,
+ any(errors.error_message) as message,
count(*) as count,
count(DISTINCT errors.profileId) as usersAffected,
max(errors.created) as lastSeen
- FROM errors
+ FROM events AS errors
LEFT JOIN (
SELECT eid, argMax(status, updated) AS status
FROM error_statuses
@@ -5111,6 +5117,7 @@ export class AnalyticsService {
GROUP BY eid
) AS status ON errors.eid = status.eid
WHERE errors.pid = {pid:FixedString(12)}
+ AND errors.type = 'error'
AND errors.created BETWEEN {groupFrom:String} AND {groupTo:String}
${filtersQuery}
${resolvedFilter}
@@ -5135,7 +5142,7 @@ export class AnalyticsService {
SELECT
profileId,
${timeBucketFunc}(toTimeZone(created, {timezone:String})) as tz_created
- FROM errors
+ FROM events AS errors
LEFT JOIN (
SELECT eid, argMax(status, updated) AS status
FROM error_statuses
@@ -5143,6 +5150,7 @@ export class AnalyticsService {
GROUP BY eid
) AS status ON errors.eid = status.eid
WHERE pid = {pid:FixedString(12)}
+ AND type = 'error'
AND created BETWEEN {groupFrom:String} AND {groupTo:String}
${filtersQuery}
${resolvedFilter}
@@ -5165,7 +5173,10 @@ export class AnalyticsService {
clickhouse
.query({
query: queryTotalSessions,
- query_params: paramsData.params,
+ query_params: {
+ ...paramsData.params,
+ ...sessionFiltersParams,
+ },
})
.then((resultSet) => resultSet.json<any>())
.then(({ data }) => data[0]),
@@ -5254,9 +5265,11 @@ export class AnalyticsService {
): Promise<{ sessions: any[]; total: number }> {
const queryCount = `
SELECT count(DISTINCT psid) as total
- FROM errors
+ FROM events
WHERE pid = {pid:FixedString(12)}
+ AND type = 'error'
AND eid = {eid:FixedString(32)}
+ AND psid IS NOT NULL
AND created BETWEEN {groupFrom:String} AND {groupTo:String}
`
@@ -5264,16 +5277,17 @@ export class AnalyticsService {
SELECT DISTINCT
CAST(errors.psid, 'String') as psid,
any(errors.profileId) as profileId,
- any(analytics.cc) as cc,
- any(analytics.br) as br,
- any(analytics.os) as os,
+ any(errors.cc) as cc,
+ any(errors.br) as br,
+ any(errors.os) as os,
min(errors.created) as firstErrorAt,
max(errors.created) as lastErrorAt,
count(*) as errorCount
- FROM errors
- LEFT JOIN analytics ON errors.psid = analytics.psid AND errors.pid = analytics.pid
+ FROM events AS errors
WHERE errors.pid = {pid:FixedString(12)}
+ AND errors.type = 'error'
AND errors.eid = {eid:FixedString(32)}
+ AND errors.psid IS NOT NULL
AND errors.created BETWEEN {groupFrom:String} AND {groupTo:String}
GROUP BY errors.psid
ORDER BY lastErrorAt DESC
@@ -5319,7 +5333,7 @@ export class AnalyticsService {
{},
)
- const query = `SELECT count(DISTINCT eid) as count FROM errors WHERE pid = {pid:FixedString(12)} AND eid IN (${_map(
+ const query = `SELECT count(DISTINCT eid) as count FROM events WHERE pid = {pid:FixedString(12)} AND type = 'error' AND eid IN (${_map(
params,
(val, key) => `{${key}:FixedString(32)}`,
).join(', ')})`
@@ -5482,12 +5496,13 @@ export class AnalyticsService {
key,
round(sum(${aggFunction}), 2) AS sum,
round(avg(${aggFunction}), 2) AS avg
- FROM customEV
+ FROM events
ARRAY JOIN meta.key AS key, meta.value AS value
WHERE
pid = {pid:FixedString(12)}
+ AND type = 'custom_event'
AND key = {metricKey:String}
- AND ev = {customEventName:String}
+ AND event_name = {customEventName:String}
AND created BETWEEN {periodFormatted:String} AND {now:String}
${kvQuery}
${filtersQuery}
@@ -5500,12 +5515,13 @@ export class AnalyticsService {
key,
round(sum(${aggFunction}), 2) AS sum,
round(avg(${aggFunction}), 2) AS avg
- FROM customEV
+ FROM events
ARRAY JOIN meta.key AS key, meta.value AS value
WHERE
pid = {pid:FixedString(12)}
+ AND type = 'custom_event'
AND key = {metricKey:String}
- AND ev = {customEventName:String}
+ AND event_name = {customEventName:String}
AND created BETWEEN {periodSubtracted:String} AND {periodFormatted:String}
${kvQuery}
${filtersQuery}
@@ -5599,75 +5615,28 @@ export class AnalyticsService {
}
}
- async getProfilesList(
- pid: string,
+ private buildProfilesListDataCTE(
filtersQuery: string,
- paramsData: any,
- safeTimezone: string,
- take = 30,
- skip = 0,
- profileType: 'all' | 'anonymous' | 'identified' = 'all',
- customEVFilterApplied = false,
- ): Promise<object[]> {
- let profileTypeFilter = ''
- if (profileType === 'anonymous') {
- profileTypeFilter = `AND profileId LIKE '${AnalyticsService.PROFILE_PREFIX_ANON}%'`
- } else if (profileType === 'identified') {
- profileTypeFilter = `AND profileId LIKE '${AnalyticsService.PROFILE_PREFIX_USER}%'`
- }
-
- let allProfileDataCTE: string
-
- if (customEVFilterApplied) {
- allProfileDataCTE = `
- all_profile_data AS (
- SELECT
- profileId,
- psid,
- cc,
- os,
- br,
- dv,
- created,
- 1 AS isPageview,
- 0 AS isEvent
- FROM analytics
- WHERE pid = {pid:FixedString(12)}
- AND created BETWEEN {groupFrom:String} AND {groupTo:String}
- AND profileId IS NOT NULL
- AND profileId != ''
- ${profileTypeFilter}
+ profileTypeFilter: string,
+ customEVFilterApplied: boolean,
+ ): string {
+ const scopedProfileFilter = customEVFilterApplied
+ ? `
AND profileId IN (
SELECT DISTINCT profileId
- FROM customEV
+ FROM events
WHERE pid = {pid:FixedString(12)}
+ AND type = 'custom_event'
AND created BETWEEN {groupFrom:String} AND {groupTo:String}
AND profileId IS NOT NULL
AND profileId != ''
${profileTypeFilter}
${filtersQuery}
)
- UNION ALL
- SELECT
- profileId,
- psid,
- cc,
- os,
- br,
- dv,
- created,
- 0 AS isPageview,
- 1 AS isEvent
- FROM customEV
- WHERE pid = {pid:FixedString(12)}
- AND created BETWEEN {groupFrom:String} AND {groupTo:String}
- AND profileId IS NOT NULL
- AND profileId != ''
- ${profileTypeFilter}
- ${filtersQuery}
- )`
- } else {
- allProfileDataCTE = `
+ `
+ : filtersQuery
+
+ return `
all_profile_data AS (
SELECT
profileId,
@@ -5677,36 +5646,43 @@ export class AnalyticsService {
br,
dv,
created,
- 1 AS isPageview,
- 0 AS isEvent
- FROM analytics
+ if(type = 'pageview', 1, 0) AS isPageview,
+ if(type = 'custom_event', 1, 0) AS isEvent,
+ if(type = 'error', 1, 0) AS isError
+ FROM events
WHERE pid = {pid:FixedString(12)}
+ AND type IN ('pageview', 'custom_event', 'error')
AND created BETWEEN {groupFrom:String} AND {groupTo:String}
AND profileId IS NOT NULL
AND profileId != ''
${profileTypeFilter}
- ${filtersQuery}
- UNION ALL
- SELECT
- profileId,
- psid,
- cc,
- os,
- br,
- dv,
- created,
- 0 AS isPageview,
- 1 AS isEvent
- FROM customEV
- WHERE pid = {pid:FixedString(12)}
- AND created BETWEEN {groupFrom:String} AND {groupTo:String}
- AND profileId IS NOT NULL
- AND profileId != ''
- ${profileTypeFilter}
- ${filtersQuery}
+ ${scopedProfileFilter}
)`
+ }
+
+ async getProfilesList(
+ pid: string,
+ filtersQuery: string,
+ paramsData: any,
+ safeTimezone: string,
+ take = 30,
+ skip = 0,
+ profileType: 'all' | 'anonymous' | 'identified' = 'all',
+ customEVFilterApplied = false,
+ ): Promise<object[]> {
+ let profileTypeFilter = ''
+ if (profileType === 'anonymous') {
+ profileTypeFilter = `AND profileId LIKE '${AnalyticsService.PROFILE_PREFIX_ANON}%'`
+ } else if (profileType === 'identified') {
+ profileTypeFilter = `AND profileId LIKE '${AnalyticsService.PROFILE_PREFIX_USER}%'`
}
+ const allProfileDataCTE = this.buildProfilesListDataCTE(
+ filtersQuery,
+ profileTypeFilter,
+ customEVFilterApplied,
+ )
+
const query = `
WITH ${allProfileDataCTE},
profile_aggregated AS (
@@ -5715,6 +5691,7 @@ export class AnalyticsService {
sum(isPageview) AS pageviewsCount,
countDistinct(psid) AS sessionsCount,
sum(isEvent) AS eventsCount,
+ sum(isError) AS errorsCount,
min(created) AS firstSeen,
max(created) AS lastSeen,
any(cc) AS cc_agg,
@@ -5723,24 +5700,13 @@ export class AnalyticsService {
any(dv) AS dv_agg
FROM all_profile_data
GROUP BY profileId
- ),
- profile_errors AS (
- SELECT
- profileId,
- count() AS errorsCount
- FROM errors
- WHERE pid = {pid:FixedString(12)}
- AND profileId IS NOT NULL
- AND profileId != ''
- ${profileTypeFilter}
- GROUP BY profileId
)
SELECT
pa.profileId AS profileId,
pa.sessionsCount AS sessionsCount,
pa.pageviewsCount AS pageviewsCount,
pa.eventsCount AS eventsCount,
- COALESCE(perr.errorsCount, 0) AS errorsCount,
+ pa.errorsCount AS errorsCount,
pa.firstSeen AS firstSeen,
pa.lastSeen AS lastSeen,
pa.cc_agg AS cc,
@@ -5748,7 +5714,6 @@ export class AnalyticsService {
pa.br_agg AS br,
pa.dv_agg AS dv
FROM profile_aggregated AS pa
- LEFT JOIN profile_errors AS perr ON pa.profileId = perr.profileId
ORDER BY pa.lastSeen DESC
LIMIT {take:UInt32}
OFFSET {skip:UInt32}
@@ -5785,54 +5750,83 @@ export class AnalyticsService {
AND profileId = {profileId:String}
`
- // Query avg duration from analytics table (more accurate than sessions table)
+ // Prefer pageview timings, but fall back to sessions for event/error-only profiles.
const queryAvgDuration = `
- SELECT
- avg(session_duration) AS avgDuration
- FROM (
+ WITH pageview_avg AS (
SELECT
- psid,
- dateDiff('second', min(created), max(created)) AS session_duration
- FROM analytics
- WHERE pid = {pid:FixedString(12)}
- AND profileId = {profileId:String}
- AND psid IS NOT NULL
- GROUP BY psid
- HAVING session_duration > 0
+ avgOrNull(session_duration) AS avgDuration
+ FROM (
+ SELECT
+ psid,
+ dateDiff('second', min(created), max(created)) AS session_duration
+ FROM events
+ WHERE pid = {pid:FixedString(12)}
+ AND type = 'pageview'
+ AND profileId = {profileId:String}
+ AND psid IS NOT NULL
+ GROUP BY psid
+ HAVING session_duration > 0
+ )
+ ),
+ sessions_avg AS (
+ SELECT
+ avgOrNull(session_duration) AS avgDuration
+ FROM (
+ SELECT
+ psid,
+ dateDiff('second', min(firstSeen), max(lastSeen)) AS session_duration
+ FROM sessions FINAL
+ WHERE pid = {pid:FixedString(12)}
+ AND profileId = {profileId:String}
+ GROUP BY psid
+ HAVING session_duration > 0
+ )
)
+ SELECT
+ coalesce(nullIf(pageview_avg.avgDuration, 0), sessions_avg.avgDuration, 0) AS avgDuration
+ FROM pageview_avg, sessions_avg
`
- // Query analytics for accurate pageview count
const queryPageviews = `
SELECT count() AS pageviewsCount
- FROM analytics
+ FROM events
WHERE pid = {pid:FixedString(12)}
+ AND type = 'pageview'
AND profileId = {profileId:String}
`
- // Query customEV for accurate events count
const queryEvents = `
SELECT count() AS eventsCount
- FROM customEV
+ FROM events
+ WHERE pid = {pid:FixedString(12)}
+ AND type = 'custom_event'
+ AND profileId = {profileId:String}
+ `
+
+ const queryErrors = `
+ SELECT count() AS errorsCount
+ FROM events
WHERE pid = {pid:FixedString(12)}
+ AND type = 'error'
AND profileId = {profileId:String}
`
// Query for device/location details
const queryDetails = `
SELECT
- any(cc) AS cc,
- any(rg) AS rg,
- any(ct) AS ct,
- any(os) AS os,
- any(osv) AS osv,
- any(br) AS br,
- any(brv) AS brv,
- any(dv) AS dv,
- any(lc) AS lc
- FROM analytics
+ argMax(cc, created) AS cc,
+ argMax(rg, created) AS rg,
+ argMax(ct, created) AS ct,
+ argMax(os, created) AS os,
+ argMax(osv, created) AS osv,
+ argMax(br, created) AS br,
+ argMax(brv, created) AS brv,
+ argMax(dv, created) AS dv,
+ argMax(lc, created) AS lc
+ FROM events
WHERE pid = {pid:FixedString(12)}
AND profileId = {profileId:String}
+ AND type IN ('pageview', 'custom_event', 'error')
`
const params = { pid, profileId }
@@ -5842,6 +5836,7 @@ export class AnalyticsService {
avgDurationResult,
pageviewsResult,
eventsResult,
+ errorsResult,
detailsResult,
] = await Promise.all([
clickhouse
@@ -5856,6 +5851,9 @@ export class AnalyticsService {
clickhouse
.query({ query: queryEvents, query_params: params })
.then((resultSet) => resultSet.json()),
+ clickhouse
+ .query({ query: queryErrors, query_params: params })
+ .then((resultSet) => resultSet.json()),
clickhouse
.query({ query: queryDetails, query_params: params })
.then((resultSet) => resultSet.json()),
@@ -5868,6 +5866,7 @@ export class AnalyticsService {
const avgDuration = (avgDurationResult.data[0] || {}) as Record<string, any>
const pageviews = (pageviewsResult.data[0] || {}) as Record<string, any>
const events = (eventsResult.data[0] || {}) as Record<string, any>
+ const errors = (errorsResult.data[0] || {}) as Record<string, any>
const details = (detailsResult.data[0] || {}) as Record<string, any>
return {
@@ -5876,6 +5875,7 @@ export class AnalyticsService {
sessionsCount: sessionCount.sessionsCount || 0,
pageviewsCount: pageviews.pageviewsCount || 0,
eventsCount: events.eventsCount || 0,
+ errorsCount: errors.errorsCount || 0,
firstSeen: sessionCount.firstSeen,
lastSeen: sessionCount.lastSeen,
avgDuration: avgDuration.avgDuration || 0,
@@ -5892,8 +5892,9 @@ export class AnalyticsService {
SELECT
pg AS page,
count() AS count
- FROM analytics
+ FROM events
WHERE pid = {pid:FixedString(12)}
+ AND type = 'pageview'
AND profileId = {profileId:String}
GROUP BY pg
ORDER BY count DESC
@@ -5922,30 +5923,15 @@ export class AnalyticsService {
.format('YYYY-MM-DD')
const query = `
- WITH all_activity AS (
- SELECT
- toDate(created) AS date,
- 1 AS isPageview,
- 0 AS isEvent
- FROM analytics
- WHERE pid = {pid:FixedString(12)}
- AND profileId = {profileId:String}
- AND created >= {startDate:Date}
- UNION ALL
- SELECT
- toDate(created) AS date,
- 0 AS isPageview,
- 1 AS isEvent
- FROM customEV
- WHERE pid = {pid:FixedString(12)}
- AND profileId = {profileId:String}
- AND created >= {startDate:Date}
- )
SELECT
- date,
- sum(isPageview) AS pageviews,
- sum(isEvent) AS events
- FROM all_activity
+ toDate(created) AS date,
+ countIf(type = 'pageview') AS pageviews,
+ countIf(type = 'custom_event') AS events
+ FROM events
+ WHERE pid = {pid:FixedString(12)}
+ AND type IN ('pageview', 'custom_event')
+ AND profileId = {profileId:String}
+ AND created >= {startDate:Date}
GROUP BY date
ORDER BY date ASC
`
@@ -5980,8 +5966,9 @@ export class AnalyticsService {
SELECT
${timeBucketFunc}(toTimeZone(created, {timezone:String})) AS time,
count() AS count
- FROM analytics
+ FROM events
WHERE pid = {pid:FixedString(12)}
+ AND type = 'pageview'
AND profileId = {profileId:String}
AND created BETWEEN {from:String} AND {to:String}
GROUP BY time
@@ -5992,8 +5979,9 @@ export class AnalyticsService {
SELECT
${timeBucketFunc}(toTimeZone(created, {timezone:String})) AS time,
count() AS count
- FROM customEV
+ FROM events
WHERE pid = {pid:FixedString(12)}
+ AND type = 'custom_event'
AND profileId = {profileId:String}
AND created BETWEEN {from:String} AND {to:String}
GROUP BY time
@@ -6004,8 +5992,9 @@ export class AnalyticsService {
SELECT
${timeBucketFunc}(toTimeZone(created, {timezone:String})) AS time,
count() AS count
- FROM errors
+ FROM events
WHERE pid = {pid:FixedString(12)}
+ AND type = 'error'
AND profileId = {profileId:String}
AND created BETWEEN {from:String} AND {to:String}
GROUP BY time
@@ -6066,70 +6055,26 @@ export class AnalyticsService {
}
}
- async getProfileSessionsList(
- pid: string,
- profileId: string,
+ private buildProfileSessionsEventsCTE(
filtersQuery: string,
- paramsData: any,
- safeTimezone: string,
- take = 30,
- skip = 0,
- customEVFilterApplied = false,
- ): Promise<object[]> {
- let allProfileEventsCTE: string
+ customEVFilterApplied: boolean,
+ ): string {
+ const scopedSessionFilter = customEVFilterApplied
+ ? `
+ AND CAST(psid, 'String') IN (
+ SELECT DISTINCT CAST(psid, 'String')
+ FROM events
+ WHERE pid = {pid:FixedString(12)}
+ AND type = 'custom_event'
+ AND profileId = {profileId:String}
+ AND psid IS NOT NULL
+ AND created BETWEEN {groupFrom:String} AND {groupTo:String}
+ ${filtersQuery}
+ )
+ `
+ : filtersQuery
- if (customEVFilterApplied) {
- allProfileEventsCTE = `
- all_profile_events AS (
- SELECT
- all_events.psidCasted,
- all_events.pid,
- all_events.profileId,
- all_events.cc,
- all_events.os,
- all_events.br,
- all_events.created_tz
- FROM (
- SELECT
- CAST(psid, 'String') AS psidCasted,
- pid,
- profileId,
- cc,
- os,
- br,
- toTimeZone(created, {timezone:String}) AS created_tz
- FROM analytics
- WHERE pid = {pid:FixedString(12)}
- AND profileId = {profileId:String}
- AND psid IS NOT NULL
- AND created BETWEEN {groupFrom:String} AND {groupTo:String}
- UNION ALL
- SELECT
- CAST(psid, 'String') AS psidCasted,
- pid,
- profileId,
- cc,
- os,
- br,
- toTimeZone(created, {timezone:String}) AS created_tz
- FROM customEV
- WHERE pid = {pid:FixedString(12)}
- AND profileId = {profileId:String}
- AND psid IS NOT NULL
- AND created BETWEEN {groupFrom:String} AND {groupTo:String}
- ) AS all_events
- WHERE all_events.psidCasted IN (
- SELECT DISTINCT CAST(psid, 'String')
- FROM customEV
- WHERE pid = {pid:FixedString(12)}
- AND profileId = {profileId:String}
- AND psid IS NOT NULL
- AND created BETWEEN {groupFrom:String} AND {groupTo:String}
- ${filtersQuery}
- )
- )`
- } else {
- allProfileEventsCTE = `
+ return `
all_profile_events AS (
SELECT
CAST(psid, 'String') AS psidCasted,
@@ -6139,29 +6084,30 @@ export class AnalyticsService {
os,
br,
toTimeZone(created, {timezone:String}) AS created_tz
- FROM analytics
- WHERE pid = {pid:FixedString(12)}
- AND profileId = {profileId:String}
- AND psid IS NOT NULL
- AND created BETWEEN {groupFrom:String} AND {groupTo:String}
- ${filtersQuery}
- UNION ALL
- SELECT
- CAST(psid, 'String') AS psidCasted,
- pid,
- profileId,
- cc,
- os,
- br,
- toTimeZone(created, {timezone:String}) AS created_tz
- FROM customEV
+ FROM events
WHERE pid = {pid:FixedString(12)}
+ AND type IN ('pageview', 'custom_event', 'error')
AND profileId = {profileId:String}
AND psid IS NOT NULL
AND created BETWEEN {groupFrom:String} AND {groupTo:String}
- ${filtersQuery}
+ ${scopedSessionFilter}
)`
- }
+ }
+
+ async getProfileSessionsList(
+ pid: string,
+ profileId: string,
+ filtersQuery: string,
+ paramsData: any,
+ safeTimezone: string,
+ take = 30,
+ skip = 0,
+ customEVFilterApplied = false,
+ ): Promise<object[]> {
+ const allProfileEventsCTE = this.buildProfileSessionsEventsCTE(
+ filtersQuery,
+ customEVFilterApplied,
+ )
const query = `
WITH ${allProfileEventsCTE},
@@ -6183,8 +6129,9 @@ export class AnalyticsService {
CAST(psid, 'String') AS psidCasted,
pid,
count() as count
- FROM analytics
+ FROM events
WHERE pid = {pid:FixedString(12)}
+ AND type = 'pageview'
AND profileId = {profileId:String}
AND psid IS NOT NULL
AND created BETWEEN {groupFrom:String} AND {groupTo:String}
@@ -6195,8 +6142,9 @@ export class AnalyticsService {
CAST(psid, 'String') AS psidCasted,
pid,
count() as count
- FROM customEV
+ FROM events
WHERE pid = {pid:FixedString(12)}
+ AND type = 'custom_event'
AND profileId = {profileId:String}
AND psid IS NOT NULL
AND created BETWEEN {groupFrom:String} AND {groupTo:String}
@@ -6207,8 +6155,9 @@ export class AnalyticsService {
CAST(psid, 'String') AS psidCasted,
pid,
count() as count
- FROM errors
+ FROM events
WHERE pid = {pid:FixedString(12)}
+ AND type = 'error'
AND profileId = {profileId:String}
AND psid IS NOT NULL
AND created BETWEEN {groupFrom:String} AND {groupTo:String}
@@ -6234,7 +6183,7 @@ export class AnalyticsService {
COALESCE(errc.count, 0) AS errors,
ps.sessionStart,
ps.lastActivity,
- if(dateDiff('second', ps.lastActivity, now()) < ${LIVE_SESSION_THRESHOLD_SECONDS}, 1, 0) AS isLive,
+ if(dateDiff('second', toTimeZone(ps.lastActivity, 'UTC'), toTimeZone(now(), 'UTC')) < ${LIVE_SESSION_THRESHOLD_SECONDS}, 1, 0) AS isLive,
sda.avg_duration AS sdur
FROM profile_sessions ps
LEFT JOIN pageview_counts pc ON ps.psidCasted = pc.psidCasted AND ps.pid = pc.pid
diff --git a/backend/apps/community/src/analytics/interfaces/index.ts b/backend/apps/community/src/analytics/interfaces/index.ts
index 84da70732..2dace48be 100644
--- a/backend/apps/community/src/analytics/interfaces/index.ts
+++ b/backend/apps/community/src/analytics/interfaces/index.ts
@@ -183,9 +183,10 @@ export interface IOverallPerformance {
}
export interface IPageflow {
- type: 'pageview' | 'event'
+ type: 'pageview' | 'event' | 'error'
value: string
created: string
+ created_utc?: string
metadata?: [string, string][]
}
diff --git a/backend/apps/community/src/analytics/utils/transformers.ts b/backend/apps/community/src/analytics/utils/transformers.ts
index b420cdac8..7ec1ed491 100644
--- a/backend/apps/community/src/analytics/utils/transformers.ts
+++ b/backend/apps/community/src/analytics/utils/transformers.ts
@@ -8,7 +8,7 @@ import utc from 'dayjs/plugin/utc'
dayjs.extend(utc)
const processMetaKV = (
- rawMeta: Record<string, string>,
+ rawMeta: Record<string, string> | null | undefined,
): { 'meta.key': string[] | null; 'meta.value': string[] | null } => {
if (!rawMeta || _isEmpty(rawMeta)) {
return {
@@ -23,10 +23,10 @@ const processMetaKV = (
}
}
-interface TrafficTransformerOptions {
- psid: string
- profileId: string
+interface CommonOptions {
pid: string
+ psid?: string | null
+ profileId?: string | null
host?: string | null
pg?: string | null
dv?: string | null
@@ -52,262 +52,42 @@ interface TrafficTransformerOptions {
meta?: Record<string, string> | null
}
-export const trafficTransformer = ({
- psid,
- profileId,
- pid,
- host,
- pg,
- dv,
- br,
- brv,
- os,
- osv,
- lc,
- ref,
- so,
- me,
- ca,
- te,
- co,
- cc,
- rg,
- rgc,
- ct,
- isp,
- og,
- ut,
- ctp,
- meta,
-}: TrafficTransformerOptions) => {
- return {
- psid,
- profileId,
- pid,
- host: host || null,
- pg: pg || null,
- dv: dv || null,
- br: br || null,
- brv: brv || null,
- os: os || null,
- osv: osv || null,
- lc: lc || null,
- ref: ref || null,
- so: so || null,
- me: me || null,
- ca: ca || null,
- te: te || null,
- co: co || null,
- cc: cc || null,
- rg: rg || null,
- rgc: rgc || null,
- ct: ct || null,
- isp: isp || null,
- og: og || null,
- ut: ut || null,
- ctp: ctp || null,
- ...processMetaKV(meta),
- created: dayjs.utc().format('YYYY-MM-DD HH:mm:ss'),
- }
+interface PageviewOptions extends CommonOptions {
+ type: 'pageview'
}
-interface CustomEventTransformerOptions {
- psid: string
- profileId: string
- pid: string
- host?: string | null
+interface CustomEventOptions extends CommonOptions {
+ type: 'custom_event'
ev: string
- pg?: string | null
- dv?: string | null
- br?: string | null
- brv?: string | null
- os?: string | null
- osv?: string | null
- lc?: string | null
- ref?: string | null
- so?: string | null
- me?: string | null
- ca?: string | null
- te?: string | null
- co?: string | null
- cc?: string | null
- rg?: string | null
- rgc?: string | null
- ct?: string | null
- isp?: string | null
- og?: string | null
- ut?: string | null
- ctp?: string | null
- meta?: Record<string, string> | null
-}
-
-export const customEventTransformer = ({
- psid,
- profileId,
- pid,
- host,
- ev,
- pg,
- dv,
- br,
- brv,
- os,
- osv,
- lc,
- ref,
- so,
- me,
- ca,
- te,
- co,
- cc,
- rg,
- rgc,
- ct,
- isp,
- og,
- ut,
- ctp,
- meta,
-}: CustomEventTransformerOptions) => {
- return {
- psid,
- profileId,
- pid,
- host: host || null,
- ev,
- pg: pg || null,
- dv: dv || null,
- br: br || null,
- brv: brv || null,
- os: os || null,
- osv: osv || null,
- lc: lc || null,
- ref: ref || null,
- so: so || null,
- me: me || null,
- ca: ca || null,
- te: te || null,
- co: co || null,
- cc: cc || null,
- rg: rg || null,
- rgc: rgc || null,
- ct: ct || null,
- isp: isp || null,
- og: og || null,
- ut: ut || null,
- ctp: ctp || null,
- ...processMetaKV(meta),
- created: dayjs.utc().format('YYYY-MM-DD HH:mm:ss'),
- }
}
-interface ErrorEventTransformerOptions {
- psid: string
- profileId: string
+interface ErrorEventOptions extends CommonOptions {
+ type: 'error'
eid: string
- pid: string
- host?: string | null
- pg?: string | null
- dv?: string | null
- br?: string | null
- brv?: string | null
- os?: string | null
- osv?: string | null
- lc?: string | null
- cc?: string | null
- rg?: string | null
- rgc?: string | null
- ct?: string | null
- isp?: string | null
- og?: string | null
- ut?: string | null
- ctp?: string | null
name?: string | null
message?: string | null
lineno?: number | null
colno?: number | null
filename?: string | null
stackTrace?: string | null
- meta?: Record<string, string> | null
-}
-
-export const errorEventTransformer = ({
- psid,
- profileId,
- eid,
- pid,
- host,
- pg,
- dv,
- br,
- brv,
- os,
- osv,
- lc,
- cc,
- rg,
- rgc,
- ct,
- isp,
- og,
- ut,
- ctp,
- name,
- message,
- lineno,
- colno,
- filename,
- stackTrace,
- meta,
-}: ErrorEventTransformerOptions) => {
- return {
- psid,
- profileId,
- eid,
- pid,
- host: host || null,
- pg: pg || null,
- dv: dv || null,
- br: br || null,
- brv: brv || null,
- os: os || null,
- osv: osv || null,
- lc: lc || null,
- cc: cc || null,
- rg: rg || null,
- rgc: rgc || null,
- ct: ct || null,
- isp: isp || null,
- og: og || null,
- ut: ut || null,
- ctp: ctp || null,
- name: name || null,
- message: message || null,
- lineno: lineno || null,
- colno: colno || null,
- filename: filename || null,
- stackTrace: stackTrace || null,
- ...processMetaKV(meta),
- created: dayjs.utc().format('YYYY-MM-DD HH:mm:ss'),
- }
}
-interface PerformanceTransformerOptions {
- pid: string
- host?: string | null
- pg?: string | null
- dv?: string | null
- br?: string | null
- brv?: string | null
- cc?: string | null
- rg?: string | null
- rgc?: string | null
- ct?: string | null
- isp?: string | null
- og?: string | null
- ut?: string | null
- ctp?: string | null
+interface PerformanceOptions extends Omit<
+ CommonOptions,
+ | 'psid'
+ | 'profileId'
+ | 'os'
+ | 'osv'
+ | 'lc'
+ | 'ref'
+ | 'so'
+ | 'me'
+ | 'ca'
+ | 'te'
+ | 'co'
+ | 'meta'
+> {
+ type: 'performance'
dns: number
tls: number
conn: number
@@ -318,53 +98,131 @@ interface PerformanceTransformerOptions {
ttfb: number
}
-export const performanceTransformer = ({
- pid,
- host,
- pg,
- dv,
- br,
- brv,
- cc,
- rg,
- rgc,
- ct,
- isp,
- og,
- ut,
- ctp,
- dns,
- tls,
- conn,
- response,
- render,
- domLoad,
- pageLoad,
- ttfb,
-}: PerformanceTransformerOptions) => {
+interface CaptchaOptions {
+ type: 'captcha'
+ pid: string
+ dv?: string | null
+ br?: string | null
+ os?: string | null
+ cc?: string | null
+ timestamp?: number
+}
+
+type EventTransformerOptions =
+ | PageviewOptions
+ | CustomEventOptions
+ | ErrorEventOptions
+ | PerformanceOptions
+ | CaptchaOptions
+
+const buildCommon = (opts: CommonOptions) => ({
+ pid: opts.pid,
+ psid: opts.psid ?? null,
+ profileId: opts.profileId ?? null,
+ host: opts.host || null,
+ pg: opts.pg || null,
+ dv: opts.dv || null,
+ br: opts.br || null,
+ brv: opts.brv || null,
+ os: opts.os || null,
+ osv: opts.osv || null,
+ lc: opts.lc || null,
+ ref: opts.ref || null,
+ so: opts.so || null,
+ me: opts.me || null,
+ ca: opts.ca || null,
+ te: opts.te || null,
+ co: opts.co || null,
+ cc: opts.cc || null,
+ rg: opts.rg || null,
+ rgc: opts.rgc || null,
+ ct: opts.ct || null,
+ isp: opts.isp || null,
+ og: opts.og || null,
+ ut: opts.ut || null,
+ ctp: opts.ctp || null,
+ ...processMetaKV(opts.meta),
+})
+
+// Single transformer for all event kinds; the discriminator is `type`.
+// Renames at the storage boundary: ev -> event_name, name -> error_name,
+// message -> error_message, filename -> error_filename. DTO and API field
+// names stay as the legacy short forms.
+export const eventTransformer = (opts: EventTransformerOptions) => {
+ const created = dayjs.utc().format('YYYY-MM-DD HH:mm:ss')
+
+ if (opts.type === 'pageview') {
+ return {
+ type: 'pageview' as const,
+ ...buildCommon(opts),
+ created,
+ }
+ }
+
+ if (opts.type === 'custom_event') {
+ return {
+ type: 'custom_event' as const,
+ ...buildCommon(opts),
+ event_name: opts.ev,
+ created,
+ }
+ }
+
+ if (opts.type === 'error') {
+ return {
+ type: 'error' as const,
+ ...buildCommon(opts),
+ eid: opts.eid,
+ error_name: opts.name || null,
+ error_message: opts.message || null,
+ stackTrace: opts.stackTrace || null,
+ lineno: opts.lineno ?? null,
+ colno: opts.colno ?? null,
+ error_filename: opts.filename || null,
+ created,
+ }
+ }
+
+ if (opts.type === 'performance') {
+ return {
+ type: 'performance' as const,
+ pid: opts.pid,
+ host: opts.host || null,
+ pg: opts.pg || null,
+ dv: opts.dv || null,
+ br: opts.br || null,
+ brv: opts.brv || null,
+ cc: opts.cc || null,
+ rg: opts.rg || null,
+ rgc: opts.rgc || null,
+ ct: opts.ct || null,
+ isp: opts.isp || null,
+ og: opts.og || null,
+ ut: opts.ut || null,
+ ctp: opts.ctp || null,
+ dns: _round(opts.dns),
+ tls: _round(opts.tls),
+ conn: _round(opts.conn),
+ response: _round(opts.response),
+ render: _round(opts.render),
+ domLoad: _round(opts.domLoad),
+ pageLoad: _round(opts.pageLoad),
+ ttfb: _round(opts.ttfb),
+ created,
+ }
+ }
+
+ // captcha
return {
- pid,
- host: host || null,
- pg: pg || null,
- dv: dv || null,
- br: br || null,
- brv: brv || null,
- cc: cc || null,
- rg: rg || null,
- rgc: rgc || null,
- ct: ct || null,
- isp: isp || null,
- og: og || null,
- ut: ut || null,
- ctp: ctp || null,
- dns: _round(dns),
- tls: _round(tls),
- conn: _round(conn),
- response: _round(response),
- render: _round(render),
- domLoad: _round(domLoad),
- pageLoad: _round(pageLoad),
- ttfb: _round(ttfb),
- created: dayjs.utc().format('YYYY-MM-DD HH:mm:ss'),
+ type: 'captcha' as const,
+ pid: opts.pid,
+ dv: opts.dv || null,
+ br: opts.br || null,
+ os: opts.os || null,
+ cc: opts.cc || null,
+ created:
+ typeof opts.timestamp === 'number'
+ ? dayjs.utc(opts.timestamp).format('YYYY-MM-DD HH:mm:ss')
+ : created,
}
}
diff --git a/backend/apps/community/src/captcha/captcha.service.ts b/backend/apps/community/src/captcha/captcha.service.ts
index 921e6d7b7..99ac3516c 100644
--- a/backend/apps/community/src/captcha/captcha.service.ts
+++ b/backend/apps/community/src/captcha/captcha.service.ts
@@ -17,7 +17,7 @@ import {
} from '../common/constants'
import { getIPDetails } from '../common/utils'
import { GeneratedChallenge } from './interfaces/generated-captcha'
-import { captchaTransformer } from './utils/transformers'
+import { eventTransformer } from '../analytics/utils/transformers'
import { clickhouse } from '../common/integrations/clickhouse'
dayjs.extend(utc)
@@ -152,18 +152,19 @@ export class CaptchaService {
const osName = ua.os.name
const { country } = getIPDetails(ip)
- const transformed = captchaTransformer(
+ const transformed = eventTransformer({
+ type: 'captcha',
pid,
- deviceType,
- browserName,
- osName,
- country,
+ dv: deviceType,
+ br: browserName,
+ os: osName,
+ cc: country,
timestamp,
- )
+ })
try {
await clickhouse.insert({
- table: 'captcha',
+ table: 'events',
format: 'JSONEachRow',
values: [transformed],
clickhouse_settings: {
diff --git a/backend/apps/community/src/captcha/utils/transformers.ts b/backend/apps/community/src/captcha/utils/transformers.ts
deleted file mode 100644
index 3a0d21d43..000000000
--- a/backend/apps/community/src/captcha/utils/transformers.ts
+++ /dev/null
@@ -1,22 +0,0 @@
-import dayjs from 'dayjs'
-import utc from 'dayjs/plugin/utc'
-
-dayjs.extend(utc)
-
-export const captchaTransformer = (
- pid: string,
- dv: string | null,
- br: string | null,
- os: string | null,
- cc: string | null,
- timestamp: number,
-) => {
- return {
- pid,
- dv: dv || null,
- br: br || null,
- os: os || null,
- cc: cc || null,
- created: dayjs.utc(timestamp).format('YYYY-MM-DD HH:mm:ss'),
- }
-}
diff --git a/backend/apps/community/src/data-import/data-import.processor.ts b/backend/apps/community/src/data-import/data-import.processor.ts
index 0d0646d1b..5f3ebfe97 100644
--- a/backend/apps/community/src/data-import/data-import.processor.ts
+++ b/backend/apps/community/src/data-import/data-import.processor.ts
@@ -8,6 +8,7 @@ import { Job } from 'bullmq'
import { DataImportService } from './data-import.service'
import { DataImportStatus } from './entity/data-import.entity'
import { getMapper } from './mappers'
+import { ImportError } from './mappers/mapper.interface'
import { clickhouse } from '../common/integrations/clickhouse'
const CLICKHOUSE_DB = process.env.CLICKHOUSE_DATABASE || 'analytics'
@@ -85,8 +86,7 @@ export class DataImportProcessor extends WorkerHost {
let minDate: string | null = null
let maxDate: string | null = null
- const analyticsBatch: Record<string, unknown>[] = []
- const customEVBatch: Record<string, unknown>[] = []
+ const eventsBatch: Record<string, unknown>[] = []
try {
for await (const row of mapper.createRowStream(
@@ -102,28 +102,17 @@ export class DataImportProcessor extends WorkerHost {
if (!maxDate || created > maxDate) maxDate = created
}
- if (row.table === 'analytics') {
- analyticsBatch.push(row.data)
- } else {
- customEVBatch.push(row.data)
- }
-
- if (analyticsBatch.length >= BATCH_SIZE) {
- await this.flushBatch('analytics', analyticsBatch)
- importedRows += analyticsBatch.length
- analyticsBatch.length = 0
- }
+ eventsBatch.push({ type: row.type, ...row.data })
- if (customEVBatch.length >= BATCH_SIZE) {
- await this.flushBatch('customEV', customEVBatch)
- importedRows += customEVBatch.length
- customEVBatch.length = 0
+ if (eventsBatch.length >= BATCH_SIZE) {
+ await this.flushBatch(eventsBatch)
+ importedRows += eventsBatch.length
+ eventsBatch.length = 0
}
if (totalRows % 10000 === 0) {
const progress = {
- importedRows:
- importedRows + analyticsBatch.length + customEVBatch.length,
+ importedRows: importedRows + eventsBatch.length,
totalRows,
}
@@ -141,14 +130,9 @@ export class DataImportProcessor extends WorkerHost {
}
}
- if (analyticsBatch.length > 0) {
- await this.flushBatch('analytics', analyticsBatch)
- importedRows += analyticsBatch.length
- }
-
- if (customEVBatch.length > 0) {
- await this.flushBatch('customEV', customEVBatch)
- importedRows += customEVBatch.length
+ if (eventsBatch.length > 0) {
+ await this.flushBatch(eventsBatch)
+ importedRows += eventsBatch.length
}
await this.dataImportService.markCompleted(importId, projectId, {
@@ -176,10 +160,15 @@ export class DataImportProcessor extends WorkerHost {
)
}
+ const userMessage =
+ error instanceof ImportError
+ ? error.message
+ : 'An unexpected error occurred while processing the import. Please try again or contact support.'
+
await this.dataImportService.markFailed(
importId,
projectId,
- error.message,
+ userMessage,
)
}
} finally {
@@ -187,12 +176,9 @@ export class DataImportProcessor extends WorkerHost {
}
}
- private async flushBatch(
- table: 'analytics' | 'customEV',
- batch: Record<string, unknown>[],
- ): Promise<void> {
+ private async flushBatch(batch: Record<string, unknown>[]): Promise<void> {
await clickhouse.insert({
- table: `${CLICKHOUSE_DB}.${table}`,
+ table: `${CLICKHOUSE_DB}.events`,
values: batch,
format: 'JSONEachRow',
})
diff --git a/backend/apps/community/src/data-import/data-import.service.ts b/backend/apps/community/src/data-import/data-import.service.ts
index ac9be0c50..e1453fbe0 100644
--- a/backend/apps/community/src/data-import/data-import.service.ts
+++ b/backend/apps/community/src/data-import/data-import.service.ts
@@ -281,11 +281,7 @@ export class DataImportService {
importId: number,
): Promise<void> {
await clickhouse.command({
- query: `ALTER TABLE ${CLICKHOUSE_DB}.analytics DELETE WHERE pid = {pid:FixedString(12)} AND importID = {importID:UInt8}`,
- query_params: { pid: projectId, importID: importId },
- })
- await clickhouse.command({
- query: `ALTER TABLE ${CLICKHOUSE_DB}.customEV DELETE WHERE pid = {pid:FixedString(12)} AND importID = {importID:UInt8}`,
+ query: `ALTER TABLE ${CLICKHOUSE_DB}.events DELETE WHERE pid = {pid:FixedString(12)} AND importID = {importID:UInt8} AND type IN ('pageview', 'custom_event')`,
query_params: { pid: projectId, importID: importId },
})
}
@@ -330,7 +326,7 @@ export class DataImportService {
): Promise<boolean> {
const result = await clickhouse
.query({
- query: `SELECT count() as cnt FROM ${CLICKHOUSE_DB}.analytics WHERE pid = {pid:FixedString(12)} AND importID IS NOT NULL AND created >= {from:DateTime} AND created <= {to:DateTime} LIMIT 1`,
+ query: `SELECT count() as cnt FROM ${CLICKHOUSE_DB}.events WHERE pid = {pid:FixedString(12)} AND type = 'pageview' AND importID IS NOT NULL AND created >= {from:DateTime} AND created <= {to:DateTime} LIMIT 1`,
query_params: { pid: projectId, from, to },
})
.then((rs) => rs.json<{ cnt: string }>())
diff --git a/backend/apps/community/src/data-import/mappers/fathom.mapper.ts b/backend/apps/community/src/data-import/mappers/fathom.mapper.ts
index 55c578f7c..1ae5397ca 100644
--- a/backend/apps/community/src/data-import/mappers/fathom.mapper.ts
+++ b/backend/apps/community/src/data-import/mappers/fathom.mapper.ts
@@ -1,7 +1,11 @@
import * as fs from 'fs'
import { parse } from 'csv-parse'
-import { ImportMapper, AnalyticsImportRow } from './mapper.interface'
+import {
+ ImportMapper,
+ AnalyticsImportRow,
+ ImportError,
+} from './mapper.interface'
import {
normalizeNull,
truncate,
@@ -41,7 +45,7 @@ function detectFileType(headers: string[]): {
const missing = COMMON_REQUIRED.filter((c) => !normalized.includes(c))
if (missing.length > 0) {
- throw new Error(
+ throw new ImportError(
`CSV does not appear to be a Fathom Analytics export. Missing required columns: ${missing.join(', ')}.`,
)
}
@@ -54,7 +58,7 @@ function detectFileType(headers: string[]): {
return { type: 'events', normalized }
}
- throw new Error(
+ throw new ImportError(
'CSV does not appear to be a Fathom Analytics export. Expected a pageviews export (with "pageviews" column) or an events export (with "event_name" column).',
)
}
@@ -202,19 +206,19 @@ export class FathomMapper implements ImportMapper {
if (fileType === 'pageviews') {
for (let i = 0; i < count; i++) {
- yield { table: 'analytics', data: baseData }
+ yield { type: 'pageview', data: baseData }
}
} else {
const eventName = truncate(normalizeNull(row.event_name), 256) || ''
- const evData = { ...baseData, ev: eventName }
+ const evData = { ...baseData, event_name: eventName }
for (let i = 0; i < count; i++) {
- yield { table: 'customEV', data: evData }
+ yield { type: 'custom_event', data: evData }
}
}
}
if (!headerChecked) {
- throw new Error(
+ throw new ImportError(
'CSV appears empty or is missing a header row. Please upload a CSV export from Fathom Analytics.',
)
}
diff --git a/backend/apps/community/src/data-import/mappers/mapper.interface.ts b/backend/apps/community/src/data-import/mappers/mapper.interface.ts
index b7bdc9bf0..73ea8538b 100644
--- a/backend/apps/community/src/data-import/mappers/mapper.interface.ts
+++ b/backend/apps/community/src/data-import/mappers/mapper.interface.ts
@@ -6,7 +6,7 @@ export class ImportError extends Error {
}
export interface AnalyticsImportRow {
- table: 'analytics' | 'customEV'
+ type: 'pageview' | 'custom_event'
data: Record<string, unknown>
}
diff --git a/backend/apps/community/src/data-import/mappers/plausible.mapper.ts b/backend/apps/community/src/data-import/mappers/plausible.mapper.ts
index 99141bb58..f0c5926d3 100644
--- a/backend/apps/community/src/data-import/mappers/plausible.mapper.ts
+++ b/backend/apps/community/src/data-import/mappers/plausible.mapper.ts
@@ -859,7 +859,7 @@ export class PlausibleMapper implements ImportMapper {
created,
})
- yield { table: 'analytics', data }
+ yield { type: 'pageview', data }
}
}
@@ -889,8 +889,8 @@ export class PlausibleMapper implements ImportMapper {
page: null,
created,
})
- data.ev = truncate(ev.name, 256) || ''
- yield { table: 'customEV', data }
+ data.event_name = truncate(ev.name, 256) || ''
+ yield { type: 'custom_event', data }
}
}
}
diff --git a/backend/apps/community/src/data-import/mappers/simple-analytics.mapper.ts b/backend/apps/community/src/data-import/mappers/simple-analytics.mapper.ts
index e95c04049..b95e37380 100644
--- a/backend/apps/community/src/data-import/mappers/simple-analytics.mapper.ts
+++ b/backend/apps/community/src/data-import/mappers/simple-analytics.mapper.ts
@@ -1,7 +1,11 @@
import * as fs from 'fs'
import { parse } from 'csv-parse'
-import { ImportMapper, AnalyticsImportRow } from './mapper.interface'
+import {
+ ImportMapper,
+ ImportError,
+ AnalyticsImportRow,
+} from './mapper.interface'
import {
normalizeNull,
truncate,
@@ -20,7 +24,7 @@ function validateHeaders(headers: string[]): string[] {
)
if (missingColumns.length > 0) {
- throw new Error(
+ throw new ImportError(
`CSV does not appear to be a Simple Analytics export. Missing required columns: ${missingColumns.join(', ')}.`,
)
}
@@ -133,7 +137,7 @@ export class SimpleAnalyticsMapper implements ImportMapper {
hostname,
)
- const cc = normalizeNull(row.country_code)
+ const cc = normalizeNull(row.country_code)?.toUpperCase() || null
const validCC = cc && /^[A-Z]{2}$/.test(cc) ? cc : null
const locale = buildLocale(
@@ -172,17 +176,17 @@ export class SimpleAnalyticsMapper implements ImportMapper {
const datapoint = normalizeNull(row.datapoint)
if (!datapoint || datapoint === 'pageview') {
- yield { table: 'analytics', data: baseData }
+ yield { type: 'pageview', data: baseData }
} else {
yield {
- table: 'customEV',
- data: { ...baseData, ev: truncate(datapoint, 256) || '' },
+ type: 'custom_event',
+ data: { ...baseData, event_name: truncate(datapoint, 256) || '' },
}
}
}
if (!headerChecked) {
- throw new Error(
+ throw new ImportError(
'CSV appears empty or is missing a header row. Please upload the raw CSV export from Simple Analytics.',
)
}
diff --git a/backend/apps/community/src/data-import/mappers/umami.mapper.ts b/backend/apps/community/src/data-import/mappers/umami.mapper.ts
index a1514ae46..3d8c9d9b8 100644
--- a/backend/apps/community/src/data-import/mappers/umami.mapper.ts
+++ b/backend/apps/community/src/data-import/mappers/umami.mapper.ts
@@ -4,7 +4,11 @@ import * as os from 'os'
import { Unzip, UnzipInflate } from 'fflate'
import { parse } from 'csv-parse'
-import { ImportMapper, AnalyticsImportRow } from './mapper.interface'
+import {
+ ImportMapper,
+ ImportError,
+ AnalyticsImportRow,
+} from './mapper.interface'
import {
normalizeNull,
truncate,
@@ -188,7 +192,7 @@ export class UmamiMapper implements ImportMapper {
file.originalSize > MAX_UMAMI_CSV_BYTES
) {
fail(
- new Error(
+ new ImportError(
`${WEBSITE_EVENT_CSV} exceeds the ${MAX_UMAMI_CSV_BYTES} byte limit.`,
),
)
@@ -213,7 +217,7 @@ export class UmamiMapper implements ImportMapper {
extractedBytes += chunk.length
if (extractedBytes > MAX_UMAMI_CSV_BYTES) {
fail(
- new Error(
+ new ImportError(
`${WEBSITE_EVENT_CSV} exceeds the ${MAX_UMAMI_CSV_BYTES} byte limit.`,
),
)
@@ -263,7 +267,7 @@ export class UmamiMapper implements ImportMapper {
if (!foundEntry) {
fail(
- new Error(
+ new ImportError(
`ZIP does not contain ${WEBSITE_EVENT_CSV}. Please upload the export ZIP from Umami.`,
),
)
@@ -368,12 +372,12 @@ export class UmamiMapper implements ImportMapper {
}
if (eventType === '1') {
- yield { table: 'analytics', data: baseData }
+ yield { type: 'pageview', data: baseData }
} else {
const eventName = truncate(normalizeNull(row.event_name), 256) || ''
yield {
- table: 'customEV',
- data: { ...baseData, ev: eventName },
+ type: 'custom_event',
+ data: { ...baseData, event_name: eventName },
}
}
}
diff --git a/backend/apps/community/src/goal/goal.controller.ts b/backend/apps/community/src/goal/goal.controller.ts
index 0d9e12b71..37f85f30a 100644
--- a/backend/apps/community/src/goal/goal.controller.ts
+++ b/backend/apps/community/src/goal/goal.controller.ts
@@ -252,38 +252,38 @@ export class GoalController {
await this.goalService.delete(id)
}
- private buildGoalMatchCondition(
- goal: Goal,
- _table: 'analytics' | 'customEV',
- ): { condition: string; params: Record<string, string> } {
+ private buildGoalMatchCondition(goal: Goal): {
+ condition: string
+ params: Record<string, string>
+ } {
const params: Record<string, string> = {}
if (goal.type === GoalType.CUSTOM_EVENT) {
- // For custom events, match the event name
if (goal.matchType === GoalMatchType.EXACT) {
params.goalValue = goal.value || ''
- return { condition: `ev = {goalValue:String}`, params }
- } else {
- // Contains match
- params.goalValue = goal.value || ''
- return {
- condition: `ev ILIKE concat('%', {goalValue:String}, '%')`,
- params,
- }
+ return { condition: `event_name = {goalValue:String}`, params }
}
- } else {
- // For pageview goals, match the page path
- if (goal.matchType === GoalMatchType.EXACT) {
- params.goalValue = goal.value || ''
- return { condition: `pg = {goalValue:String}`, params }
- } else {
- // Contains match
- params.goalValue = goal.value || ''
- return {
- condition: `pg ILIKE concat('%', {goalValue:String}, '%')`,
- params,
- }
+ if ((goal.value || '').trim() === '') {
+ return { condition: '1=0', params: {} }
}
+ params.goalValue = goal.value || ''
+ return {
+ condition: `event_name ILIKE concat('%', {goalValue:String}, '%')`,
+ params,
+ }
+ }
+
+ if (goal.matchType === GoalMatchType.EXACT) {
+ params.goalValue = goal.value || ''
+ return { condition: `pg = {goalValue:String}`, params }
+ }
+ if ((goal.value || '').trim() === '') {
+ return { condition: '1=0', params: {} }
+ }
+ params.goalValue = goal.value || ''
+ return {
+ condition: `pg ILIKE concat('%', {goalValue:String}, '%')`,
+ params,
}
}
@@ -347,9 +347,10 @@ export class GoalController {
safeTimezone,
)
- const table = goal.type === GoalType.CUSTOM_EVENT ? 'customEV' : 'analytics'
+ const goalType =
+ goal.type === GoalType.CUSTOM_EVENT ? 'custom_event' : 'pageview'
const { condition: matchCondition, params: matchParams } =
- this.buildGoalMatchCondition(goal, table)
+ this.buildGoalMatchCondition(goal)
const { condition: metaCondition, params: metaParams } =
this.buildMetadataCondition(goal)
@@ -358,9 +359,10 @@ export class GoalController {
SELECT
count(*) as conversions,
uniqExact(psid) as uniqueSessions
- FROM ${table}
+ FROM events
WHERE
pid = {pid:FixedString(12)}
+ AND type = {goalType:String}
AND ${matchCondition}
${metaCondition}
AND created BETWEEN {groupFrom:String} AND {groupTo:String}
@@ -368,6 +370,7 @@ export class GoalController {
const queryParams = {
pid: goal.projectId,
+ goalType,
groupFrom: groupFromUTC,
groupTo: groupToUTC,
...matchParams,
@@ -386,9 +389,10 @@ export class GoalController {
// Get total unique sessions for conversion rate
const totalSessionsQuery = `
SELECT uniqExact(psid) as totalSessions
- FROM analytics
+ FROM events
WHERE
pid = {pid:FixedString(12)}
+ AND type = 'pageview'
AND created BETWEEN {groupFrom:String} AND {groupTo:String}
`
@@ -566,7 +570,8 @@ export class GoalController {
safeTimezone,
)
- const table = goal.type === GoalType.CUSTOM_EVENT ? 'customEV' : 'analytics'
+ const goalType =
+ goal.type === GoalType.CUSTOM_EVENT ? 'custom_event' : 'pageview'
const timeBucketFunc = Object.prototype.hasOwnProperty.call(
timeBucketConversion,
resolvedTimeBucket,
@@ -576,7 +581,7 @@ export class GoalController {
const [selector, groupBy] = this.getGroupSubquery(resolvedTimeBucket)
const { condition: matchCondition, params: matchParams } =
- this.buildGoalMatchCondition(goal, table)
+ this.buildGoalMatchCondition(goal)
const { condition: metaCondition, params: metaParams } =
this.buildMetadataCondition(goal)
@@ -588,9 +593,10 @@ export class GoalController {
FROM (
SELECT *,
${timeBucketFunc}(toTimeZone(created, {timezone:String})) as tz_created
- FROM ${table}
+ FROM events
WHERE
pid = {pid:FixedString(12)}
+ AND type = {goalType:String}
AND ${matchCondition}
${metaCondition}
AND created BETWEEN {groupFrom:String} AND {groupTo:String}
@@ -601,6 +607,7 @@ export class GoalController {
const queryParams = {
pid: goal.projectId,
+ goalType,
groupFrom: groupFromUTC,
groupTo: groupToUTC,
timezone: safeTimezone,
diff --git a/backend/apps/community/src/project/project.controller.ts b/backend/apps/community/src/project/project.controller.ts
index 757444527..1c98b7939 100644
--- a/backend/apps/community/src/project/project.controller.ts
+++ b/backend/apps/community/src/project/project.controller.ts
@@ -576,11 +576,7 @@ export class ProjectController {
try {
await clickhouse.command({
- query: `ALTER TABLE analytics DELETE WHERE pid={pid:FixedString(12)}`,
- query_params: { pid: id },
- })
- await clickhouse.command({
- query: `ALTER TABLE customEV DELETE WHERE pid={pid:FixedString(12)}`,
+ query: `ALTER TABLE events DELETE WHERE pid={pid:FixedString(12)} AND type IN ('pageview', 'custom_event')`,
query_params: { pid: id },
})
return 'Project reset successfully'
@@ -1021,11 +1017,11 @@ export class ProjectController {
try {
await deleteProjectSharesByProjectClickhouse(id)
await clickhouse.command({
- query: `ALTER TABLE analytics DELETE WHERE pid={pid:FixedString(12)}`,
+ query: `ALTER TABLE events DELETE WHERE pid={pid:FixedString(12)} AND type IN ('pageview', 'custom_event', 'error', 'performance', 'captcha')`,
query_params: { pid: id },
})
await clickhouse.command({
- query: `ALTER TABLE customEV DELETE WHERE pid={pid:FixedString(12)}`,
+ query: `ALTER TABLE error_statuses DELETE WHERE pid={pid:FixedString(12)}`,
query_params: { pid: id },
})
await deleteProjectRedis(id)
diff --git a/backend/apps/community/src/project/project.service.ts b/backend/apps/community/src/project/project.service.ts
index 0d6147406..160c46ac3 100644
--- a/backend/apps/community/src/project/project.service.ts
+++ b/backend/apps/community/src/project/project.service.ts
@@ -248,36 +248,10 @@ export class ProjectService {
)
const query = `
- SELECT
- pid,
- CASE
- WHEN EXISTS (
- SELECT 1
- FROM analytics
- WHERE pid IN (${pids})
- )
- OR EXISTS (
- SELECT 1
- FROM customEV
- WHERE pid IN (${pids})
- )
- THEN 1
- ELSE 0
- END AS exists
- FROM
- (
- SELECT DISTINCT pid
- FROM
- (
- SELECT pid
- FROM analytics
- WHERE pid IN (${pids})
- UNION ALL
- SELECT pid
- FROM customEV
- WHERE pid IN (${pids})
- ) AS t
- );
+ SELECT DISTINCT pid
+ FROM events
+ WHERE pid IN (${pids})
+ AND type IN ('pageview', 'custom_event')
`
const { data } = await clickhouse
@@ -310,23 +284,10 @@ export class ProjectService {
)
const query = `
- SELECT
- pid,
- CASE
- WHEN EXISTS (
- SELECT 1
- FROM errors
- WHERE pid IN (${pids})
- )
- THEN 1
- ELSE 0
- END AS exists
- FROM
- (
- SELECT DISTINCT pid
- FROM errors
- WHERE pid IN (${pids})
- );
+ SELECT DISTINCT pid
+ FROM events
+ WHERE pid IN (${pids})
+ AND type = 'error'
`
const { data } = await clickhouse
@@ -359,23 +320,10 @@ export class ProjectService {
)
const query = `
- SELECT
- pid,
- CASE
- WHEN EXISTS (
- SELECT 1
- FROM captcha
- WHERE pid IN (${pids})
- )
- THEN 1
- ELSE 0
- END AS exists
- FROM
- (
- SELECT DISTINCT pid
- FROM captcha
- WHERE pid IN (${pids})
- );
+ SELECT DISTINCT pid
+ FROM events
+ WHERE pid IN (${pids})
+ AND type = 'captcha'
`
const { data } = await clickhouse
@@ -394,10 +342,7 @@ export class ProjectService {
to: string,
): Promise<void> {
const queries = [
- 'ALTER TABLE analytics DELETE WHERE pid = {pid:FixedString(12)} AND created BETWEEN {from:String} AND {to:String}',
- 'ALTER TABLE customEV DELETE WHERE pid = {pid:FixedString(12)} AND created BETWEEN {from:String} AND {to:String}',
- 'ALTER TABLE performance DELETE WHERE pid = {pid:FixedString(12)} AND created BETWEEN {from:String} AND {to:String}',
- 'ALTER TABLE errors DELETE WHERE pid = {pid:FixedString(12)} AND created BETWEEN {from:String} AND {to:String}',
+ "ALTER TABLE events DELETE WHERE pid = {pid:FixedString(12)} AND type IN ('pageview', 'custom_event', 'performance', 'error', 'captcha') AND created BETWEEN {from:String} AND {to:String}",
'ALTER TABLE error_statuses DELETE WHERE pid = {pid:FixedString(12)} AND created BETWEEN {from:String} AND {to:String}',
]
diff --git a/backend/apps/community/src/user/user.controller.ts b/backend/apps/community/src/user/user.controller.ts
index 64dd758fd..cb3e4fe8b 100644
--- a/backend/apps/community/src/user/user.controller.ts
+++ b/backend/apps/community/src/user/user.controller.ts
@@ -181,10 +181,7 @@ export class UserController {
if (!_isEmpty(projects)) {
const pidArray = projects.map((el) => el.id)
const queries = [
- 'ALTER TABLE analytics DELETE WHERE pid IN ({pids:Array(FixedString(12))})',
- 'ALTER TABLE customEV DELETE WHERE pid IN ({pids:Array(FixedString(12))})',
- 'ALTER TABLE performance DELETE WHERE pid IN ({pids:Array(FixedString(12))})',
- 'ALTER TABLE errors DELETE WHERE pid IN ({pids:Array(FixedString(12))})',
+ 'ALTER TABLE events DELETE WHERE pid IN ({pids:Array(FixedString(12))})',
'ALTER TABLE error_statuses DELETE WHERE pid IN ({pids:Array(FixedString(12))})',
]
await deleteProjectsByUserIdClickhouse(id)
diff --git a/backend/meta/clickhouse/generate-dummy-data.js b/backend/meta/clickhouse/generate-dummy-data.js
index 1b5f9b8fa..004d81d45 100644
--- a/backend/meta/clickhouse/generate-dummy-data.js
+++ b/backend/meta/clickhouse/generate-dummy-data.js
@@ -115,10 +115,10 @@ const chalk = {
cyan: text => `\x1b[36m${text}\x1b[0m`,
}
-const insertData = async (pid, rowCount, processedRecords, table) => {
+const insertData = async (pid, rowCount, processedRecords) => {
try {
await clickhouse.insert({
- table,
+ table: 'events',
format: 'JSONEachRow',
values: processedRecords,
clickhouse_settings: {
@@ -154,6 +154,7 @@ const generateAnalyticsData = async (pid, rowCount, from, to) => {
for (let i = 0; i < rowCount; ++i) {
records.push({
+ type: 'pageview',
psid:
Math.random() < 0.05
? faker.helpers.arrayElement(PSIDS)
@@ -193,7 +194,7 @@ const generateAnalyticsData = async (pid, rowCount, from, to) => {
),
)
- insertData(pid, rowCount, records, 'analytics')
+ insertData(pid, rowCount, records)
}
const generateCaptchaData = async (pid, rowCount, from, to) => {
@@ -201,6 +202,7 @@ const generateCaptchaData = async (pid, rowCount, from, to) => {
for (let i = 0; i < rowCount; ++i) {
records.push({
+ type: 'captcha',
pid,
dv: faker.helpers.arrayElement(DEVICES),
br: faker.helpers.arrayElement(BROWSERS),
@@ -223,7 +225,7 @@ const generateCaptchaData = async (pid, rowCount, from, to) => {
),
)
- insertData(pid, rowCount, records, 'captcha')
+ insertData(pid, rowCount, records)
}
const generateCustomEventsData = async (pid, rowCount, from, to) => {
@@ -231,8 +233,9 @@ const generateCustomEventsData = async (pid, rowCount, from, to) => {
for (let i = 0; i < rowCount; ++i) {
records.push({
+ type: 'custom_event',
pid,
- ev: faker.helpers.arrayElement(CUSTOM_EVENTS),
+ event_name: faker.helpers.arrayElement(CUSTOM_EVENTS),
pg: faker.helpers.arrayElement(PAGES),
dv: faker.helpers.arrayElement(DEVICES),
br: faker.helpers.arrayElement(BROWSERS),
@@ -267,7 +270,7 @@ const generateCustomEventsData = async (pid, rowCount, from, to) => {
),
)
- insertData(pid, rowCount, records, 'customEV')
+ insertData(pid, rowCount, records)
}
const generatePerformanceData = async (pid, rowCount, from, to) => {
@@ -275,6 +278,7 @@ const generatePerformanceData = async (pid, rowCount, from, to) => {
for (let i = 0; i < rowCount; ++i) {
records.push({
+ type: 'performance',
pid,
pg: faker.helpers.arrayElement(PAGES),
dv: faker.helpers.arrayElement(DEVICES),
@@ -308,7 +312,7 @@ const generatePerformanceData = async (pid, rowCount, from, to) => {
),
)
- insertData(pid, rowCount, records, 'performance')
+ insertData(pid, rowCount, records)
}
const main = () => {
diff --git a/backend/migrations/clickhouse/2026_05_01_unify_events.js b/backend/migrations/clickhouse/2026_05_01_unify_events.js
new file mode 100644
index 000000000..164c012af
--- /dev/null
+++ b/backend/migrations/clickhouse/2026_05_01_unify_events.js
@@ -0,0 +1,97 @@
+const { queriesRunner, dbName } = require('./setup')
+
+const queries = [
+ `DROP TABLE IF EXISTS ${dbName}.events_tmp;`,
+ `DROP TABLE IF EXISTS ${dbName}.events_backup;`,
+
+ `CREATE TABLE IF NOT EXISTS ${dbName}.events
+ (
+ type LowCardinality(String),
+ pid FixedString(12),
+ psid Nullable(UInt64),
+ profileId Nullable(String) CODEC(ZSTD(3)),
+ host Nullable(String) CODEC(ZSTD(3)),
+ pg Nullable(String) CODEC(ZSTD(3)),
+ dv LowCardinality(Nullable(String)),
+ br LowCardinality(Nullable(String)),
+ brv Nullable(String) CODEC(ZSTD(3)),
+ os LowCardinality(Nullable(String)),
+ osv Nullable(String) CODEC(ZSTD(3)),
+ lc LowCardinality(Nullable(String)),
+ ref Nullable(String) CODEC(ZSTD(3)),
+ so Nullable(String) CODEC(ZSTD(3)),
+ me Nullable(String) CODEC(ZSTD(3)),
+ ca Nullable(String) CODEC(ZSTD(3)),
+ te Nullable(String) CODEC(ZSTD(3)),
+ co Nullable(String) CODEC(ZSTD(3)),
+ cc Nullable(FixedString(2)),
+ rg LowCardinality(Nullable(String)),
+ rgc LowCardinality(Nullable(String)),
+ ct Nullable(String) CODEC(ZSTD(3)),
+ isp LowCardinality(Nullable(String)),
+ og Nullable(String) CODEC(ZSTD(3)),
+ ut LowCardinality(Nullable(String)),
+ ctp LowCardinality(Nullable(String)),
+ \`meta.key\` Array(String) CODEC(ZSTD(3)),
+ \`meta.value\` Array(String) CODEC(ZSTD(3)),
+ importID Nullable(UInt8),
+ event_name Nullable(String) CODEC(ZSTD(3)),
+ eid Nullable(FixedString(32)),
+ error_name Nullable(String) CODEC(ZSTD(3)),
+ error_message Nullable(String) CODEC(ZSTD(3)),
+ stackTrace Nullable(String) CODEC(ZSTD(3)),
+ lineno Nullable(UInt32),
+ colno Nullable(UInt32),
+ error_filename Nullable(String) CODEC(ZSTD(3)),
+ dns Nullable(UInt32),
+ tls Nullable(UInt32),
+ conn Nullable(UInt32),
+ response Nullable(UInt32),
+ render Nullable(UInt32),
+ domLoad Nullable(UInt32),
+ pageLoad Nullable(UInt32),
+ ttfb Nullable(UInt32),
+ created DateTime('UTC') CODEC(Delta(4), LZ4)
+ )
+ ENGINE = MergeTree()
+ PARTITION BY toYYYYMM(created)
+ ORDER BY (pid, type, created);`,
+
+ `CREATE TABLE ${dbName}.events_tmp AS ${dbName}.events;`,
+
+ `INSERT INTO ${dbName}.events_tmp
+ (type, pid, psid, profileId, host, pg, dv, br, brv, os, osv, lc, ref, so, me, ca, te, co, cc, rg, rgc, ct, isp, og, ut, ctp, \`meta.key\`, \`meta.value\`, importID, created)
+ SELECT
+ 'pageview', pid, psid, profileId, host, pg, dv, br, brv, os, osv, lc, ref, so, me, ca, te, co, cc, rg, rgc, ct, isp, og, ut, ctp, \`meta.key\`, \`meta.value\`, importID, created
+ FROM ${dbName}.analytics;`,
+
+ `INSERT INTO ${dbName}.events_tmp
+ (type, pid, psid, profileId, host, pg, dv, br, brv, os, osv, lc, ref, so, me, ca, te, co, cc, rg, rgc, ct, isp, og, ut, ctp, \`meta.key\`, \`meta.value\`, importID, event_name, created)
+ SELECT
+ 'custom_event', pid, psid, profileId, host, pg, dv, br, brv, os, osv, lc, ref, so, me, ca, te, co, cc, rg, rgc, ct, isp, og, ut, ctp, \`meta.key\`, \`meta.value\`, importID, ev, created
+ FROM ${dbName}.customEV;`,
+
+ `INSERT INTO ${dbName}.events_tmp
+ (type, pid, psid, profileId, host, pg, dv, br, brv, os, osv, lc, cc, rg, rgc, ct, isp, og, ut, ctp, \`meta.key\`, \`meta.value\`, eid, error_name, error_message, stackTrace, lineno, colno, error_filename, created)
+ SELECT
+ 'error', pid, psid, profileId, host, pg, dv, br, brv, os, osv, lc, cc, rg, rgc, ct, isp, og, ut, ctp, \`meta.key\`, \`meta.value\`, eid, name, message, stackTrace, lineno, colno, filename, created
+ FROM ${dbName}.errors;`,
+
+ `INSERT INTO ${dbName}.events_tmp
+ (type, pid, host, pg, dv, br, brv, cc, rg, rgc, ct, isp, og, ut, ctp, dns, tls, conn, response, render, domLoad, pageLoad, ttfb, created)
+ SELECT
+ 'performance', pid, host, pg, dv, br, brv, cc, rg, rgc, ct, isp, og, ut, ctp, dns, tls, conn, response, render, domLoad, pageLoad, ttfb, created
+ FROM ${dbName}.performance;`,
+
+ `INSERT INTO ${dbName}.events_tmp
+ (type, pid, dv, br, os, cc, created)
+ SELECT
+ 'captcha', pid, dv, br, os, cc, created
+ FROM ${dbName}.captcha;`,
+
+ `RENAME TABLE ${dbName}.events TO ${dbName}.events_backup, ${dbName}.events_tmp TO ${dbName}.events;`,
+
+ `DROP TABLE IF EXISTS ${dbName}.events_backup;`,
+]
+
+queriesRunner(queries)
diff --git a/backend/migrations/clickhouse/2026_05_04_drop_legacy_tables.js b/backend/migrations/clickhouse/2026_05_04_drop_legacy_tables.js
new file mode 100644
index 000000000..4964e865c
--- /dev/null
+++ b/backend/migrations/clickhouse/2026_05_04_drop_legacy_tables.js
@@ -0,0 +1,18 @@
+const { queriesRunner, dbName } = require('./setup')
+
+const queries = [
+ `SELECT throwIf(
+ count() = 0,
+ 'The ${dbName}.events table does not exist. Run 2026_05_01_unify_events.js before dropping legacy tables.'
+ )
+ FROM system.tables
+ WHERE database = '${dbName}' AND name = 'events';`,
+
+ `DROP TABLE IF EXISTS ${dbName}.analytics;`,
+ `DROP TABLE IF EXISTS ${dbName}.customEV;`,
+ `DROP TABLE IF EXISTS ${dbName}.errors;`,
+ `DROP TABLE IF EXISTS ${dbName}.performance;`,
+ `DROP TABLE IF EXISTS ${dbName}.captcha;`,
+]
+
+queriesRunner(queries)
diff --git a/backend/migrations/clickhouse/initialise_database.js b/backend/migrations/clickhouse/initialise_database.js
index 5018ca040..beb10a635 100644
--- a/backend/migrations/clickhouse/initialise_database.js
+++ b/backend/migrations/clickhouse/initialise_database.js
@@ -4,51 +4,14 @@ const { queriesRunner, dbName, databaselessQueriesRunner } = require('./setup')
const CLICKHOUSE_DB_INIT_QUERIES = [`CREATE DATABASE IF NOT EXISTS ${dbName}`]
const CLICKHOUSE_INIT_QUERIES = [
- // The traffic data table
- `CREATE TABLE IF NOT EXISTS ${dbName}.analytics
- (
- psid Nullable(UInt64),
- profileId Nullable(String) CODEC(ZSTD(3)),
- pid FixedString(12),
- host Nullable(String) CODEC(ZSTD(3)),
- pg Nullable(String) CODEC(ZSTD(3)),
- dv LowCardinality(Nullable(String)),
- br LowCardinality(Nullable(String)),
- brv Nullable(String) CODEC(ZSTD(3)),
- os LowCardinality(Nullable(String)),
- osv Nullable(String) CODEC(ZSTD(3)),
- lc LowCardinality(Nullable(String)),
- ref Nullable(String) CODEC(ZSTD(3)),
- so Nullable(String) CODEC(ZSTD(3)),
- me Nullable(String) CODEC(ZSTD(3)),
- ca Nullable(String) CODEC(ZSTD(3)),
- te Nullable(String) CODEC(ZSTD(3)),
- co Nullable(String) CODEC(ZSTD(3)),
- cc Nullable(FixedString(2)),
- rg LowCardinality(Nullable(String)),
- rgc LowCardinality(Nullable(String)),
- ct Nullable(String) CODEC(ZSTD(3)),
- isp LowCardinality(Nullable(String)),
- og Nullable(String) CODEC(ZSTD(3)),
- ut LowCardinality(Nullable(String)),
- ctp LowCardinality(Nullable(String)),
- \`meta.key\` Array(String) CODEC(ZSTD(3)),
- \`meta.value\` Array(String) CODEC(ZSTD(3)),
- importID Nullable(UInt8),
- created DateTime('UTC') CODEC(Delta(4), LZ4)
- )
- ENGINE = MergeTree()
- PARTITION BY toYYYYMM(created)
- ORDER BY (pid, created);`,
- // Custom events table
- `CREATE TABLE IF NOT EXISTS ${dbName}.customEV
+ `CREATE TABLE IF NOT EXISTS ${dbName}.events
(
+ type LowCardinality(String),
+ pid FixedString(12),
psid Nullable(UInt64),
profileId Nullable(String) CODEC(ZSTD(3)),
- pid FixedString(12),
host Nullable(String) CODEC(ZSTD(3)),
- ev String CODEC(ZSTD(3)),
pg Nullable(String) CODEC(ZSTD(3)),
dv LowCardinality(Nullable(String)),
br LowCardinality(Nullable(String)),
@@ -73,29 +36,14 @@ const CLICKHOUSE_INIT_QUERIES = [
\`meta.key\` Array(String) CODEC(ZSTD(3)),
\`meta.value\` Array(String) CODEC(ZSTD(3)),
importID Nullable(UInt8),
- created DateTime('UTC') CODEC(Delta(4), LZ4)
- )
- ENGINE = MergeTree()
- PARTITION BY toYYYYMM(created)
- ORDER BY (pid, created);`,
-
- // The performance data table
- `CREATE TABLE IF NOT EXISTS ${dbName}.performance
- (
- pid FixedString(12),
- host Nullable(String) CODEC(ZSTD(3)),
- pg Nullable(String) CODEC(ZSTD(3)),
- dv LowCardinality(Nullable(String)),
- br LowCardinality(Nullable(String)),
- brv Nullable(String) CODEC(ZSTD(3)),
- cc Nullable(FixedString(2)),
- rg LowCardinality(Nullable(String)),
- rgc LowCardinality(Nullable(String)),
- ct Nullable(String) CODEC(ZSTD(3)),
- isp LowCardinality(Nullable(String)),
- og Nullable(String) CODEC(ZSTD(3)),
- ut LowCardinality(Nullable(String)),
- ctp LowCardinality(Nullable(String)),
+ event_name Nullable(String) CODEC(ZSTD(3)),
+ eid Nullable(FixedString(32)),
+ error_name Nullable(String) CODEC(ZSTD(3)),
+ error_message Nullable(String) CODEC(ZSTD(3)),
+ stackTrace Nullable(String) CODEC(ZSTD(3)),
+ lineno Nullable(UInt32),
+ colno Nullable(UInt32),
+ error_filename Nullable(String) CODEC(ZSTD(3)),
dns Nullable(UInt32),
tls Nullable(UInt32),
conn Nullable(UInt32),
@@ -108,44 +56,7 @@ const CLICKHOUSE_INIT_QUERIES = [
)
ENGINE = MergeTree()
PARTITION BY toYYYYMM(created)
- ORDER BY (pid, created);`,
-
- // Error events table
- `CREATE TABLE IF NOT EXISTS ${dbName}.errors
- (
- psid Nullable(UInt64),
- profileId Nullable(String) CODEC(ZSTD(3)),
- eid FixedString(32),
- pid FixedString(12),
- host Nullable(String) CODEC(ZSTD(3)),
- pg Nullable(String) CODEC(ZSTD(3)),
- dv LowCardinality(Nullable(String)),
- br LowCardinality(Nullable(String)),
- brv Nullable(String) CODEC(ZSTD(3)),
- os LowCardinality(Nullable(String)),
- osv Nullable(String) CODEC(ZSTD(3)),
- lc LowCardinality(Nullable(String)),
- cc LowCardinality(Nullable(FixedString(2))),
- rg LowCardinality(Nullable(String)),
- rgc LowCardinality(Nullable(String)),
- ct Nullable(String) CODEC(ZSTD(3)),
- isp LowCardinality(Nullable(String)),
- og Nullable(String) CODEC(ZSTD(3)),
- ut LowCardinality(Nullable(String)),
- ctp LowCardinality(Nullable(String)),
- name String CODEC(ZSTD(3)),
- message Nullable(String) CODEC(ZSTD(3)),
- stackTrace Nullable(String) CODEC(ZSTD(3)),
- \`meta.key\` Array(String) CODEC(ZSTD(3)),
- \`meta.value\` Array(String) CODEC(ZSTD(3)),
- lineno Nullable(UInt32),
- colno Nullable(UInt32),
- filename Nullable(String) CODEC(ZSTD(3)),
- created DateTime('UTC') CODEC(Delta(4), LZ4)
- )
- ENGINE = MergeTree()
- PARTITION BY toYYYYMM(created)
- ORDER BY (pid, created);`,
+ ORDER BY (pid, type, created);`,
// Error events status table
`CREATE TABLE IF NOT EXISTS ${dbName}.error_statuses (
@@ -201,20 +112,6 @@ const CLICKHOUSE_INIT_QUERIES = [
ORDER BY (pid, experimentId, created)
TTL created + INTERVAL 1 YEAR;`,
- // The CAPTCHA data table
- `CREATE TABLE IF NOT EXISTS ${dbName}.captcha
- (
- pid FixedString(12),
- dv LowCardinality(Nullable(String)),
- br LowCardinality(Nullable(String)),
- os Nullable(String) CODEC(ZSTD(3)),
- cc Nullable(FixedString(2)),
- created DateTime('UTC') CODEC(Delta(4), LZ4)
- )
- ENGINE = MergeTree()
- PARTITION BY toYYYYMM(created)
- ORDER BY (pid, created);`,
-
// Bot block events table for advanced bot protection reporting
`CREATE TABLE IF NOT EXISTS ${dbName}.bot_blocks
(
diff --git a/web/app/api/api.server.ts b/web/app/api/api.server.ts
index 3eb66d8d5..052d09680 100644
--- a/web/app/api/api.server.ts
+++ b/web/app/api/api.server.ts
@@ -1036,10 +1036,12 @@ export async function getFunnelSessionsServer(
}
interface PageflowItem {
- type: 'pageview' | 'event' | 'error'
+ type: 'pageview' | 'event' | 'error' | 'sale' | 'refund'
value: string
created: string
metadata?: { key: string; value: string }[]
+ amount?: number
+ currency?: string
}
export interface SessionDetailsResponse {
@@ -1067,6 +1069,7 @@ export interface SessionDetailsResponse {
refunds?: number
created: string
sdur?: number
+ isLive?: boolean
profileId: string | null
isIdentified: 1 | 0
isFirstSession: 1 | 0
diff --git a/web/app/pages/Project/tabs/Sessions/SessionDetailView.tsx b/web/app/pages/Project/tabs/Sessions/SessionDetailView.tsx
index 0dd99425a..7a3573cc7 100644
--- a/web/app/pages/Project/tabs/Sessions/SessionDetailView.tsx
+++ b/web/app/pages/Project/tabs/Sessions/SessionDetailView.tsx
@@ -42,10 +42,12 @@ import { Pageflow } from './Pageflow'
import { SessionChart } from './SessionChart'
interface PageflowItem {
- type: 'pageview' | 'event' | 'error'
+ type: 'pageview' | 'event' | 'error' | 'sale' | 'refund'
value: string
created: string
metadata?: { key: string; value: string }[]
+ amount?: number
+ currency?: string
}
interface SessionDetailViewProps {
diff --git a/web/app/pages/Project/tabs/Sessions/SessionsView.tsx b/web/app/pages/Project/tabs/Sessions/SessionsView.tsx
index c9631ed51..e43ad3018 100644
--- a/web/app/pages/Project/tabs/Sessions/SessionsView.tsx
+++ b/web/app/pages/Project/tabs/Sessions/SessionsView.tsx
@@ -85,10 +85,12 @@ class SessionsErrorBoundary extends Component<
const SESSIONS_TAKE = 30
interface PageflowItem {
- type: 'pageview' | 'event' | 'error'
+ type: 'pageview' | 'event' | 'error' | 'sale' | 'refund'
value: string
created: string
metadata?: { key: string; value: string }[]
+ amount?: number
+ currency?: string
}
interface ActiveSession {
@@ -201,7 +203,9 @@ const SessionsViewInner = ({
dv: apiDetails.dv,
profileId: apiDetails.profileId,
sdur: apiDetails.sdur,
- isLive: matchingSession?.isLive === 1,
+ isLive: matchingSession
+ ? matchingSession.isLive === 1
+ : apiDetails.isLive,
}
return {
details,