Skip to content

📈 Open source, privacy-first web analytics — a cookie-less Google Analytics alternative with built-in error tracking and performance monitoring. Self-host it or use our cloud. Get started for free!

Stop relying on unique column for unique session calculation

Blue Mouse3e9924d5e52c
diff --git a/backend/apps/production/src/analytics/analytics.service.ts b/backend/apps/production/src/analytics/analytics.service.ts
index d66c20899..a19ed5558 100644
--- a/backend/apps/production/src/analytics/analytics.service.ts
+++ b/backend/apps/production/src/analytics/analytics.service.ts
@@ -301,7 +301,7 @@ const generateParamsQuery = (
return `SELECT ${columnsQuery}, count(*) as count ${subQuery} AND ${col} IS NOT NULL GROUP BY ${columnsQuery}`
}
- return `SELECT ${columnsQuery}, count(*) as count ${subQuery} AND ${col} IS NOT NULL AND unique='1' GROUP BY ${columnsQuery}`
+ return `SELECT ${columnsQuery}, count(DISTINCT psid) as count ${subQuery} AND ${col} IS NOT NULL GROUP BY ${columnsQuery}`
}
export enum DataType {
@@ -1621,7 +1621,7 @@ export class AnalyticsService {
WITH analytics_counts AS (
SELECT
count(*) AS all,
- countIf(unique=1) AS unique
+ count(DISTINCT psid) AS unique
FROM analytics
WHERE
pid = {pid:FixedString(12)}
@@ -1717,7 +1717,7 @@ export class AnalyticsService {
SELECT
1 AS sortOrder,
count(*) AS all,
- countIf(unique=1) AS unique
+ count(DISTINCT psid) AS unique
FROM analytics
WHERE
pid = {pid:FixedString(12)}
@@ -1729,11 +1729,10 @@ export class AnalyticsService {
FROM session_durations
WHERE pid = {pid:FixedString(12)}
AND psid IN (
- SELECT psid
+ SELECT DISTINCT psid
FROM analytics
WHERE pid = {pid:FixedString(12)}
AND created BETWEEN {periodFormatted:String} AND {now:String}
- AND unique = 1
${filtersQuery}
)
)
@@ -1748,7 +1747,7 @@ export class AnalyticsService {
SELECT
2 AS sortOrder,
count(*) AS all,
- countIf(unique=1) AS unique
+ count(DISTINCT psid) AS unique
FROM analytics
WHERE
pid = {pid:FixedString(12)}
@@ -1760,11 +1759,10 @@ export class AnalyticsService {
FROM session_durations
WHERE pid = {pid:FixedString(12)}
AND psid IN (
- SELECT psid
+ SELECT DISTINCT psid
FROM analytics
WHERE pid = {pid:FixedString(12)}
AND created BETWEEN {periodSubtracted:String} AND {periodFormatted:String}
- AND unique = 1
${filtersQuery}
)
)
@@ -2102,6 +2100,99 @@ export class AnalyticsService {
return _map(data, type)
}
+ async generateParams2(
+ parsedFilters: Array<{ [key: string]: string }>,
+ subQuery: string,
+ customEVFilterApplied: boolean,
+ paramsData: any,
+ type: 'traffic' | 'performance' | 'captcha' | 'errors' | 'uptime',
+ measure?: PerfMeasure,
+ ) {
+ // We need this to display all the pageview related data (e.g. country, browser) when user applies an inclusive filter on the Page column
+ const isPageInclusiveFilterSet = ['captcha', 'performance'].includes(type)
+ ? false
+ : !_isEmpty(
+ _find(
+ parsedFilters,
+ filter => filter.column === 'pg' && !filter.isExclusive,
+ ),
+ )
+
+ let columns = TRAFFIC_COLUMNS
+
+ if (type === 'captcha') {
+ columns = CAPTCHA_COLUMNS
+ }
+
+ if (type === 'errors') {
+ columns = ERROR_COLUMNS
+ }
+
+ if (type === 'performance') {
+ columns = PERFORMANCE_COLUMNS
+ }
+
+ if (type === 'uptime') {
+ columns = UPTIME_COLUMNS
+ }
+
+ // Build a single query combining all metrics
+ const withClauses = columns.map(col => {
+ const baseQuery = generateParamsQuery(
+ col,
+ subQuery,
+ customEVFilterApplied,
+ isPageInclusiveFilterSet,
+ type,
+ measure,
+ )
+ return `metrics_${col} AS (${baseQuery})`
+ })
+
+ const query = `
+ WITH ${withClauses.join(',\n')}
+ SELECT
+ column_name,
+ name,
+ count,
+ cc
+ FROM (
+ ${columns
+ .map(col => {
+ const extraFields =
+ col === 'rg' || col === 'ct' ? ', cc' : ', NULL as cc'
+ return `
+ SELECT
+ '${col}' as column_name,
+ name,
+ count
+ ${extraFields}
+ FROM metrics_${col}
+ `
+ })
+ .join('\nUNION ALL\n')}
+ )
+ ORDER BY column_name, count DESC
+ `
+
+ console.log(query)
+
+ const { data } = await clickhouse
+ .query({
+ query,
+ query_params: paramsData.params,
+ })
+ .then(resultSet => resultSet.json<any>())
+
+ // Transform results back into the expected format
+ const params = {}
+ columns.forEach(col => {
+ params[col] = data.filter(row => row.column_name === col)
+ })
+
+ return params
+ }
+
async generateParams(
parsedFilters: Array<{ [key: string]: string }>,
subQuery: string,
@@ -2379,12 +2470,11 @@ export class AnalyticsService {
${selector},
avgOrNull(session_durations.duration) as sdur,
count() as pageviews,
- sum(unique) as uniques
+ count(DISTINCT psid) as uniques
FROM (
SELECT
pid,
psid,
- unique,
${timeBucketFunc}(toTimeZone(created, '${safeTimezone}')) as tz_created
FROM analytics
PREWHERE pid = {pid:FixedString(12)}
@@ -2392,7 +2482,7 @@ export class AnalyticsService {
${filtersQuery}
) as subquery
LEFT JOIN (
- SELECT
+ SELECT
pid,
psid,
duration
@@ -2692,6 +2782,9 @@ export class AnalyticsService {
const promises = [
// Getting params
(async () => {
+ const id = Math.random().toString(16).substring(5, 16)
+ console.time(`${id} - groupParamsByTimeBucket`)
+
params = await this.groupParamsByTimeBucket(
subQuery,
paramsData,
@@ -2699,6 +2792,8 @@ export class AnalyticsService {
parsedFilters,
)
+ console.timeLog(`${id} - groupParamsByTimeBucket`)
+
if (!_some(_values(params), val => !_isEmpty(val))) {
throw new BadRequestException(
'There are no parameters for the specified time frames',
@@ -2708,6 +2803,9 @@ export class AnalyticsService {
// Getting chart & average session duration data
(async () => {
+ const id = Math.random().toString(16).substring(5, 16)
+ console.time(`${id} - groupChartByTimeBucket`)
+
const groupedData = await this.groupChartByTimeBucket(
timeBucket,
from,
@@ -2719,6 +2817,8 @@ export class AnalyticsService {
mode,
)
+ console.timeLog(`${id} - groupChartByTimeBucket`)
+
// @ts-ignore
chart = groupedData.chart
})(),
@@ -2738,7 +2838,7 @@ export class AnalyticsService {
customEVFilterApplied: boolean,
parsedFilters: Array<{ [key: string]: string }>,
): Promise<object | void> {
- return this.generateParams(
+ return this.generateParams2(
parsedFilters,
subQuery,
customEVFilterApplied,
@@ -2968,7 +3068,7 @@ export class AnalyticsService {
const promises = [
// Getting params
(async () => {
- params = await this.generateParams(
+ params = await this.generateParams2(
null,
subQuery,
false,
@@ -3038,7 +3138,7 @@ export class AnalyticsService {
const promises = [
// Params
(async () => {
- params = await this.generateParams(
+ params = await this.generateParams2(
null,
subQuery,
false,
@@ -3281,7 +3381,7 @@ export class AnalyticsService {
const promises = [
// Getting params
(async () => {
- params = await this.generateParams(
+ params = await this.generateParams2(
null,
subQuery,
false,
@@ -3731,7 +3831,7 @@ export class AnalyticsService {
WHERE
pid = {pid:FixedString(12)}
AND psid = {psid:String}
- AND unique = 1
+ ORDER BY created ASC
LIMIT 1;
`
diff --git a/backend/apps/production/src/analytics/decorators/validate-period.decorator.ts b/backend/apps/production/src/analytics/decorators/validate-period.decorator.ts
index 917104e61..4e2950c6f 100644
--- a/backend/apps/production/src/analytics/decorators/validate-period.decorator.ts
+++ b/backend/apps/production/src/analytics/decorators/validate-period.decorator.ts
@@ -24,7 +24,7 @@ export function ValidatePeriod(validationOptions?: ValidationOptions) {
options: validationOptions,
validator: {
validate(value: any) {
- if (value === undefined || value === '') {
+ if (value === undefined || value === '' || value === 'custom') {
return true // Allow empty or undefined values
}
if (!_includes(VALID_PERIODS, value)) {
diff --git a/backend/apps/production/src/project/project.service.ts b/backend/apps/production/src/project/project.service.ts
index 56ef46a63..64a4a84f0 100644
--- a/backend/apps/production/src/project/project.service.ts
+++ b/backend/apps/production/src/project/project.service.ts
@@ -1320,8 +1320,14 @@ export class ProjectService {
from: string,
to: string,
): Promise<number> {
- const query =
- "SELECT count() FROM analytics WHERE pid = {pid:FixedString(12)} AND created BETWEEN {from:String} AND {to:String} AND unique='1'"
+ const query = `
+ SELECT
+ count(DISTINCT psid)
+ FROM analytics
+ WHERE
+ pid = {pid:FixedString(12)}
+ AND created BETWEEN {from:String} AND {to:String}
+ `
const params = { pid, from, to }
let count = 0
diff --git a/backend/apps/selfhosted/src/analytics/decorators/validate-period.decorator.ts b/backend/apps/selfhosted/src/analytics/decorators/validate-period.decorator.ts
index 917104e61..4e2950c6f 100644
--- a/backend/apps/selfhosted/src/analytics/decorators/validate-period.decorator.ts
+++ b/backend/apps/selfhosted/src/analytics/decorators/validate-period.decorator.ts
@@ -24,7 +24,7 @@ export function ValidatePeriod(validationOptions?: ValidationOptions) {
options: validationOptions,
validator: {
validate(value: any) {
- if (value === undefined || value === '') {
+ if (value === undefined || value === '' || value === 'custom') {
return true // Allow empty or undefined values
}
if (!_includes(VALID_PERIODS, value)) {
diff --git a/backend/meta/clickhouse/generate-dummy-data.js b/backend/meta/clickhouse/generate-dummy-data.js
index f864f886e..a779e3a8b 100644
--- a/backend/meta/clickhouse/generate-dummy-data.js
+++ b/backend/meta/clickhouse/generate-dummy-data.js
@@ -3,7 +3,7 @@ const { faker } = require('@faker-js/faker')
const _ = require('lodash')
const chalk = require('chalk')
-const { queriesRunner, dbName } = require('../../migrations/clickhouse/setup')
+const { clickhouse, dbName } = require('../../migrations/clickhouse/setup')
const PID_SIZE = 12
const DEFAULT_ROW_COUNT = 10
@@ -111,86 +111,76 @@ const CITIES = [
'Kobe', // Japan
]
-const chunkArray = (arr, chunkSize = 1000) => {
- const result = []
- for (let i = 0; i < arr.length; i += chunkSize) {
- result.push(arr.slice(i, i + chunkSize))
- }
- return result
-}
-
const insertData = async (pid, rowCount, processedRecords, table) => {
- const insertQueries = []
-
- const recordChunks = chunkArray(processedRecords, 1000)
-
- for (const chunk of recordChunks) {
- const insertQuery = `INSERT INTO ${dbName}.${table} (*) VALUES ${chunk.join(
- ',',
- )}`
- insertQueries.push(insertQuery)
- }
-
- if (recordChunks.length > 10) {
- console.warn(
- chalk.yellow(
- '[WARNING] The number of records is big, so it might take longer to insert the records',
- ),
- )
- }
-
try {
- const result = await queriesRunner(insertQueries, false)
-
- if (!result) {
- throw new Error()
- }
+ await clickhouse.insert({
+ table,
+ format: 'JSONEachRow',
+ values: processedRecords,
+ clickhouse_settings: {
+ async_insert: 1,
+ },
+ })
console.log(
chalk.green(
`[SUCCESS] Successfully inserted ${rowCount} records for ${pid}!`,
),
)
- } catch {
+ } catch (reason) {
console.error(
chalk.red('[ERROR] Error occured whilst inserting the records'),
)
+ console.error(reason)
}
}
+const PSIDS = [
+ '123456789012',
+ '123456789013',
+ '123456789014',
+ '123456789015',
+ '123456789016',
+ '123456789017',
+ '123456789018',
+]
+
const generateAnalyticsData = async (pid, rowCount, from, to) => {
const records = []
for (let i = 0; i < rowCount; ++i) {
- const record = [
- 0, // psid
- null, // sid
+ records.push({
+ psid:
+ Math.random() < 0.05
+ ? faker.helpers.arrayElement(PSIDS)
+ : Math.floor(Math.random() * 1000000000000000000),
pid,
- faker.helpers.arrayElement(PAGES), // pg
- null, // prev
- faker.helpers.arrayElement(DEVICES), // dv
- faker.helpers.arrayElement(BROWSERS), // br
- faker.helpers.arrayElement(OS), // os
- _.toLower(faker.location.countryCode()), // lc
- faker.helpers.arrayElement(URLS), // ref
- faker.helpers.arrayElement(UTM_SOURCES), // so
- faker.helpers.arrayElement(UTM_MEDIUMS), // me
- faker.helpers.arrayElement(UTM_CAMPAIGNS), // ca
- faker.location.countryCode(), // cc
- faker.helpers.arrayElement(REGIONS), // rg
- faker.helpers.arrayElement(CITIES), // ct
- faker.number.int({ min: 0, max: 10000 }), // sdur
- faker.number.int({ min: 0, max: 9 }) === 0 ? 1 : 0, // unique; 10% chance of unique record
- _.split(
+ pg: faker.helpers.arrayElement(PAGES),
+ dv: faker.helpers.arrayElement(DEVICES),
+ br: faker.helpers.arrayElement(BROWSERS),
+ brv: null,
+ os: faker.helpers.arrayElement(OS),
+ osv: null,
+ lc: _.toLower(faker.location.countryCode()),
+ ref: faker.helpers.arrayElement(URLS),
+ so: faker.helpers.arrayElement(UTM_SOURCES),
+ me: faker.helpers.arrayElement(UTM_MEDIUMS),
+ ca: faker.helpers.arrayElement(UTM_CAMPAIGNS),
+ te: null,
+ co: null,
+ cc: faker.location.countryCode(),
+ rg: faker.helpers.arrayElement(REGIONS),
+ ct: faker.helpers.arrayElement(CITIES),
+ unique: faker.number.int({ min: 0, max: 9 }) === 0 ? 1 : 0,
+ created: _.split(
faker.date
.between({ from, to })
.toISOString()
.replace('T', ' ')
.replace('Z', ''),
'.',
- )[0], // created; format the date string
- ]
- records.push(record)
+ )[0],
+ })
}
console.info(
@@ -199,32 +189,28 @@ const generateAnalyticsData = async (pid, rowCount, from, to) => {
),
)
- const processedRecords = records.map(
- record => `(${record.map(item => `'${item}'`).join(',')})`,
- )
- insertData(pid, rowCount, processedRecords, 'analytics')
+ insertData(pid, rowCount, records, 'analytics')
}
const generateCaptchaData = async (pid, rowCount, from, to) => {
const records = []
for (let i = 0; i < rowCount; ++i) {
- const record = [
+ records.push({
pid,
- faker.helpers.arrayElement(DEVICES), // dv
- faker.helpers.arrayElement(BROWSERS), // br
- faker.helpers.arrayElement(OS), // os
- faker.location.countryCode(), // cc
- _.split(
+ dv: faker.helpers.arrayElement(DEVICES),
+ br: faker.helpers.arrayElement(BROWSERS),
+ os: faker.helpers.arrayElement(OS),
+ cc: faker.location.countryCode(),
+ created: _.split(
faker.date
.between({ from, to })
.toISOString()
.replace('T', ' ')
.replace('Z', ''),
'.',
- )[0], // created; format the date string
- ]
- records.push(record)
+ )[0],
+ })
}
console.info(
@@ -233,41 +219,42 @@ const generateCaptchaData = async (pid, rowCount, from, to) => {
),
)
- const processedRecords = records.map(
- record => `(${record.map(item => `'${item}'`).join(',')})`,
- )
- insertData(pid, rowCount, processedRecords, 'captcha')
+ insertData(pid, rowCount, records, 'captcha')
}
const generateCustomEventsData = async (pid, rowCount, from, to) => {
const records = []
for (let i = 0; i < rowCount; ++i) {
- const record = [
+ records.push({
pid,
- faker.helpers.arrayElement(CUSTOM_EVENTS), // ev
- faker.helpers.arrayElement(PAGES), // pg
- faker.helpers.arrayElement(DEVICES), // dv
- faker.helpers.arrayElement(BROWSERS), // br
- faker.helpers.arrayElement(OS), // os
- _.toLower(faker.location.countryCode()), // lc
- faker.helpers.arrayElement(URLS), // ref
- faker.helpers.arrayElement(UTM_SOURCES), // so
- faker.helpers.arrayElement(UTM_MEDIUMS), // me
- faker.helpers.arrayElement(UTM_CAMPAIGNS), // ca
- faker.location.countryCode(), // cc
- faker.helpers.arrayElement(REGIONS), // rg
- faker.helpers.arrayElement(CITIES), // ct
- _.split(
+ ev: faker.helpers.arrayElement(CUSTOM_EVENTS),
+ pg: faker.helpers.arrayElement(PAGES),
+ dv: faker.helpers.arrayElement(DEVICES),
+ br: faker.helpers.arrayElement(BROWSERS),
+ brv: null,
+ os: faker.helpers.arrayElement(OS),
+ osv: null,
+ lc: _.toLower(faker.location.countryCode()),
+ ref: faker.helpers.arrayElement(URLS),
+ so: faker.helpers.arrayElement(UTM_SOURCES),
+ me: faker.helpers.arrayElement(UTM_MEDIUMS),
+ ca: faker.helpers.arrayElement(UTM_CAMPAIGNS),
+ te: null,
+ co: null,
+ cc: faker.location.countryCode(),
+ rg: faker.helpers.arrayElement(REGIONS),
+ ct: faker.helpers.arrayElement(CITIES),
+ unique: faker.number.int({ min: 0, max: 9 }) === 0 ? 1 : 0,
+ created: _.split(
faker.date
.between({ from, to })
.toISOString()
.replace('T', ' ')
.replace('Z', ''),
'.',
- )[0], // created; format the date string
- ]
- records.push(record)
+ )[0],
+ })
}
console.info(
@@ -276,42 +263,39 @@ const generateCustomEventsData = async (pid, rowCount, from, to) => {
),
)
- const processedRecords = records.map(
- record => `(${record.map(item => `'${item}'`).join(',')})`,
- )
- insertData(pid, rowCount, processedRecords, 'customEV')
+ insertData(pid, rowCount, records, 'customEV')
}
const generatePerformanceData = async (pid, rowCount, from, to) => {
const records = []
for (let i = 0; i < rowCount; ++i) {
- const record = [
+ records.push({
pid,
- faker.helpers.arrayElement(PAGES), // pg
- faker.helpers.arrayElement(DEVICES), // dv
- faker.helpers.arrayElement(BROWSERS), // br
- faker.location.countryCode(), // cc
- faker.helpers.arrayElement(REGIONS), // rg
- faker.helpers.arrayElement(CITIES), // ct
- faker.number.int({ min: 0, max: 500 }), // dns
- faker.number.int({ min: 0, max: 1000 }), // tls
- faker.number.int({ min: 0, max: 100 }), // conn
- faker.number.int({ min: 0, max: 500 }), // response
- faker.number.int({ min: 0, max: 1500 }), // render
- faker.number.int({ min: 300, max: 2000 }), // domLoad
- faker.number.int({ min: 300, max: 2000 }), // pageLoad
- faker.number.int({ min: 0, max: 500 }), // ttfb
- _.split(
+ pg: faker.helpers.arrayElement(PAGES),
+ dv: faker.helpers.arrayElement(DEVICES),
+ br: faker.helpers.arrayElement(BROWSERS),
+ brv: null,
+ cc: faker.location.countryCode(),
+ rg: faker.helpers.arrayElement(REGIONS),
+ ct: faker.helpers.arrayElement(CITIES),
+ dns: faker.number.int({ min: 0, max: 500 }),
+ tls: faker.number.int({ min: 0, max: 1000 }),
+ conn: faker.number.int({ min: 0, max: 100 }),
+ response: faker.number.int({ min: 0, max: 500 }),
+ render: faker.number.int({ min: 0, max: 1500 }),
+ domLoad: faker.number.int({ min: 300, max: 2000 }),
+ pageLoad: faker.number.int({ min: 300, max: 2000 }),
+ ttfb: faker.number.int({ min: 0, max: 500 }),
+ created: _.split(
faker.date
.between({ from, to })
.toISOString()
.replace('T', ' ')
.replace('Z', ''),
'.',
- )[0], // created; format the date string
- ]
- records.push(record)
+ )[0],
+ })
}
console.info(
@@ -320,10 +304,7 @@ const generatePerformanceData = async (pid, rowCount, from, to) => {
),
)
- const processedRecords = records.map(
- record => `(${record.map(item => `'${item}'`).join(',')})`,
- )
- insertData(pid, rowCount, processedRecords, 'performance')
+ insertData(pid, rowCount, records, 'performance')
}
const main = () => {
diff --git a/backend/migrations/clickhouse/2024_10_28_broken.js b/backend/migrations/clickhouse/2024_10_28_broken.js
new file mode 100644
index 000000000..5f0e82c61
--- /dev/null
+++ b/backend/migrations/clickhouse/2024_10_28_broken.js
@@ -0,0 +1,60 @@
+const { queriesRunner, dbName } = require('./setup')
+
+const queries = [
+ // Step 1: Create Temporary Table
+ `
+ CREATE TABLE IF NOT EXISTS ${dbName}.analytics_temp AS ${dbName}.analytics
+ ENGINE = MergeTree()
+ PARTITION BY toYYYYMM(created)
+ ORDER BY (pid, created);
+ `,
+
+ // Step 2: Insert Data with Backfilled psid
+ // TODO: Refactor it. It will fail if there are more than 50 million rows.
+ `INSERT INTO ${dbName}.analytics_temp
+ SELECT
+ IF(psid IS NOT NULL, psid,
+ dense_rank() OVER (
+ PARTITION BY toDate(created), br, os, lc, cc, rg, ct
+ ORDER BY created
+ )
+ ) AS psid,
+ pid,
+ pg,
+ NULL as prev,
+ dv,
+ br,
+ brv,
+ os,
+ osv,
+ lc,
+ ref,
+ so,
+ me,
+ ca,
+ te,
+ co,
+ cc,
+ rg,
+ ct,
+ meta.key,
+ meta.value,
+ unique,
+ created
+ FROM ${dbName}.analytics`,
+
+ // Step 3: Swap Tables
+ `
+ RENAME TABLE ${dbName}.analytics TO ${dbName}.analytics_backup, ${dbName}.analytics_temp TO ${dbName}.analytics;
+ `,
+
+ // Step 4: Drop Backup Table (Optional)
+ // `
+ // DROP TABLE ${dbName}.analytics_backup;
+ // `,
+
+ // Step 5: Drop the Obsolete `unique` Column
+ // `ALTER TABLE ${dbName}.analytics DROP COLUMN IF EXISTS unique;`,
+]
+
+queriesRunner(queries, true)
diff --git a/backend/migrations/clickhouse/2024_10_28_sort_of_working.js b/backend/migrations/clickhouse/2024_10_28_sort_of_working.js
new file mode 100644
index 000000000..00dd7ed70
--- /dev/null
+++ b/backend/migrations/clickhouse/2024_10_28_sort_of_working.js
@@ -0,0 +1,107 @@
+const { queriesRunner, dbName } = require('./setup')
+
+// Generates insert queries from 2022-01-01 to 2023-10-17 AND where psid is NULL.
+const generateInsertQueries = () => {
+ const queries = []
+ const currentDate = new Date('2021-01-01')
+ const endDate = new Date('2023-10-18')
+
+ while (currentDate <= endDate) {
+ // Calculate the start of next month
+ const nextMonth = new Date(currentDate)
+ nextMonth.setMonth(nextMonth.getMonth() + 1)
+
+ // Ensure we don't go past the end date
+ const queryEndDate = nextMonth > endDate ? endDate : nextMonth
+
+ queries.push(`INSERT INTO ${dbName}.analytics_temp
+ WITH
+ SessionStarts AS (
+ SELECT
+ *,
+ cityHash64(
+ toString(toDate(created)),
+ toString(br),
+ toString(os),
+ toString(lc),
+ toString(cc),
+ toString(rg),
+ toString(ct)
+ ) AS params_hash,
+ 1000000 + dense_rank() OVER (
+ PARTITION BY toDate(created)
+ ORDER BY br, os, lc, cc, rg, ct, created
+ ) AS session_id
+ FROM ${dbName}.analytics
+ WHERE isNull(psid) AND created >= parseDateTimeBestEffort('${currentDate.toISOString()}') AND created < parseDateTimeBestEffort('${queryEndDate.toISOString()}')
+ ),
+ SessionsAssigned AS (
+ SELECT
+ a.*,
+ LAST_VALUE(IF(a.unique = 1, a.session_id, NULL)) OVER (
+ PARTITION BY toDate(a.created), a.br, a.os, a.lc, a.cc, a.rg, a.ct
+ ORDER BY a.created
+ RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW
+ ) AS new_psid
+ FROM SessionStarts a
+ )
+ SELECT
+ CASE
+ WHEN isNotNull(psid) OR created >= '2023-10-18' THEN psid
+ ELSE new_psid
+ END AS psid,
+ pid,
+ pg,
+ dv,
+ br,
+ brv,
+ os,
+ osv,
+ lc,
+ ref,
+ so,
+ me,
+ ca,
+ te,
+ co,
+ cc,
+ rg,
+ ct,
+ meta.key,
+ meta.value,
+ unique,
+ created
+ FROM SessionsAssigned
+ WHERE new_psid IS NOT NULL OR created >= '2023-10-18';`)
+
+ // Move to the next month
+ currentDate.setMonth(currentDate.getMonth() + 1)
+ }
+
+ return queries
+}
+
+const queries = [
+ `DROP TABLE IF EXISTS ${dbName}.analytics_temp;`,
+
+ // Step 1: Create Temporary Table
+ `CREATE TABLE ${dbName}.analytics_temp AS ${dbName}.analytics
+ ENGINE = MergeTree()
+ PARTITION BY toYYYYMM(created)
+ ORDER BY (pid, created);`,
+
+ ...generateInsertQueries(),
+ `INSERT INTO analytics_temp SELECT * FROM analytics WHERE created >= '2023-10-18';`,
+
+ // Step 3: Swap Tables
+ `RENAME TABLE ${dbName}.analytics TO ${dbName}.analytics_backup,
+ ${dbName}.analytics_temp TO ${dbName}.analytics;`,
+
+ // Step 4: Drop Backup Table (Optional)
+ // `DROP TABLE ${dbName}.analytics_backup;`,
+
+ // Step 5: Drop the Obsolete 'unique' Column
+ // `ALTER TABLE ${dbName}.analytics DROP COLUMN IF EXISTS unique;`
+]
+
+queriesRunner(queries, true)
diff --git a/backend/migrations/clickhouse/setup.js b/backend/migrations/clickhouse/setup.js
index 7d259754d..fc7fc94eb 100644
--- a/backend/migrations/clickhouse/setup.js
+++ b/backend/migrations/clickhouse/setup.js
@@ -62,7 +62,15 @@ const databaselessQueriesRunner = async queries => {
}
}
-const queriesRunner = async (queries, log = true) => {
+/**
+ * @typedef {import('@clickhouse/client').CommandParams} CommandParams
+ *
+ * @param {string[]} queries
+ * @param {boolean} log
+ * @param {Omit<CommandParams, 'query'>} params
+ * @returns
+ */
+const queriesRunner = async (queries, log = true, params = {}) => {
let failed = false
for (const query of queries) {
@@ -74,6 +82,7 @@ const queriesRunner = async (queries, log = true) => {
try {
// eslint-disable-next-line no-await-in-loop
await clickhouse.command({
+ ...params,
query,
})