📈 Open source, privacy-first web analytics — a cookie-less Google Analytics alternative with built-in error tracking and performance monitoring. Self-host it or use our cloud. Get started for free!
TypeScript chartsclickhousedockergdprgoogle-analyticsmetricsmonitoringmysqlnestjsprivacyredisself-hostedselfhostedstatisticstailwindcssweb-analyticswebanalyticswebsite
Stop relying on unique column for unique session calculation
Blue Mouse3e9924d5e52c
diff --git a/backend/apps/production/src/analytics/analytics.service.ts b/backend/apps/production/src/analytics/analytics.service.tsindex d66c20899..a19ed5558 100644--- a/backend/apps/production/src/analytics/analytics.service.ts+++ b/backend/apps/production/src/analytics/analytics.service.ts@@ -301,7 +301,7 @@ const generateParamsQuery = (return `SELECT ${columnsQuery}, count(*) as count ${subQuery} AND ${col} IS NOT NULL GROUP BY ${columnsQuery}`}- return `SELECT ${columnsQuery}, count(*) as count ${subQuery} AND ${col} IS NOT NULL AND unique='1' GROUP BY ${columnsQuery}`+ return `SELECT ${columnsQuery}, count(DISTINCT psid) as count ${subQuery} AND ${col} IS NOT NULL GROUP BY ${columnsQuery}`}export enum DataType {@@ -1621,7 +1621,7 @@ export class AnalyticsService {WITH analytics_counts AS (SELECTcount(*) AS all,- countIf(unique=1) AS unique+ count(DISTINCT psid) AS uniqueFROM analyticsWHEREpid = {pid:FixedString(12)}@@ -1717,7 +1717,7 @@ export class AnalyticsService {SELECT1 AS sortOrder,count(*) AS all,- countIf(unique=1) AS unique+ count(DISTINCT psid) AS uniqueFROM analyticsWHEREpid = {pid:FixedString(12)}@@ -1729,11 +1729,10 @@ export class AnalyticsService {FROM session_durationsWHERE pid = {pid:FixedString(12)}AND psid IN (- SELECT psid+ SELECT DISTINCT psidFROM analyticsWHERE pid = {pid:FixedString(12)}AND created BETWEEN {periodFormatted:String} AND {now:String}- AND unique = 1${filtersQuery}))@@ -1748,7 +1747,7 @@ export class AnalyticsService {SELECT2 AS sortOrder,count(*) AS all,- countIf(unique=1) AS unique+ count(DISTINCT psid) AS uniqueFROM analyticsWHEREpid = {pid:FixedString(12)}@@ -1760,11 +1759,10 @@ export class AnalyticsService {FROM session_durationsWHERE pid = {pid:FixedString(12)}AND psid IN (- SELECT psid+ SELECT DISTINCT psidFROM analyticsWHERE pid = {pid:FixedString(12)}AND created BETWEEN {periodSubtracted:String} AND {periodFormatted:String}- AND unique = 1${filtersQuery}))@@ -2102,6 +2100,99 @@ export class AnalyticsService {return _map(data, type)}+ async generateParams2(+ parsedFilters: Array<{ [key: string]: string }>,+ subQuery: string,+ customEVFilterApplied: boolean,+ paramsData: any,+ type: 'traffic' | 'performance' | 'captcha' | 'errors' | 'uptime',+ measure?: PerfMeasure,+ ) {+ // We need this to display all the pageview related data (e.g. country, browser) when user applies an inclusive filter on the Page column+ const isPageInclusiveFilterSet = ['captcha', 'performance'].includes(type)+ ? false+ : !_isEmpty(+ _find(+ parsedFilters,+ filter => filter.column === 'pg' && !filter.isExclusive,+ ),+ )++ let columns = TRAFFIC_COLUMNS++ if (type === 'captcha') {+ columns = CAPTCHA_COLUMNS+ }++ if (type === 'errors') {+ columns = ERROR_COLUMNS+ }++ if (type === 'performance') {+ columns = PERFORMANCE_COLUMNS+ }++ if (type === 'uptime') {+ columns = UPTIME_COLUMNS+ }++ // Build a single query combining all metrics+ const withClauses = columns.map(col => {+ const baseQuery = generateParamsQuery(+ col,+ subQuery,+ customEVFilterApplied,+ isPageInclusiveFilterSet,+ type,+ measure,+ )+ return `metrics_${col} AS (${baseQuery})`+ })++ const query = `+ WITH ${withClauses.join(',\n')}+ SELECT+ column_name,+ name,+ count,+ cc+ FROM (+ ${columns+ .map(col => {+ const extraFields =+ col === 'rg' || col === 'ct' ? ', cc' : ', NULL as cc'+ return `+ SELECT+ '${col}' as column_name,+ name,+ count+ ${extraFields}+ FROM metrics_${col}+ `+ })+ .join('\nUNION ALL\n')}+ )+ ORDER BY column_name, count DESC+ `++ console.log(query)++ const { data } = await clickhouse+ .query({+ query,+ query_params: paramsData.params,+ })+ .then(resultSet => resultSet.json<any>())++ // Transform results back into the expected format+ const params = {}+ columns.forEach(col => {+ params[col] = data.filter(row => row.column_name === col)+ })++ return params+ }+async generateParams(parsedFilters: Array<{ [key: string]: string }>,subQuery: string,@@ -2379,12 +2470,11 @@ export class AnalyticsService {${selector},avgOrNull(session_durations.duration) as sdur,count() as pageviews,- sum(unique) as uniques+ count(DISTINCT psid) as uniquesFROM (SELECTpid,psid,- unique,${timeBucketFunc}(toTimeZone(created, '${safeTimezone}')) as tz_createdFROM analyticsPREWHERE pid = {pid:FixedString(12)}@@ -2392,7 +2482,7 @@ export class AnalyticsService {${filtersQuery}) as subqueryLEFT JOIN (- SELECT+ SELECTpid,psid,duration@@ -2692,6 +2782,9 @@ export class AnalyticsService {const promises = [// Getting params(async () => {+ const id = Math.random().toString(16).substring(5, 16)+ console.time(`${id} - groupParamsByTimeBucket`)+params = await this.groupParamsByTimeBucket(subQuery,paramsData,@@ -2699,6 +2792,8 @@ export class AnalyticsService {parsedFilters,)+ console.timeLog(`${id} - groupParamsByTimeBucket`)+if (!_some(_values(params), val => !_isEmpty(val))) {throw new BadRequestException('There are no parameters for the specified time frames',@@ -2708,6 +2803,9 @@ export class AnalyticsService {// Getting chart & average session duration data(async () => {+ const id = Math.random().toString(16).substring(5, 16)+ console.time(`${id} - groupChartByTimeBucket`)+const groupedData = await this.groupChartByTimeBucket(timeBucket,from,@@ -2719,6 +2817,8 @@ export class AnalyticsService {mode,)+ console.timeLog(`${id} - groupChartByTimeBucket`)+// @ts-ignorechart = groupedData.chart})(),@@ -2738,7 +2838,7 @@ export class AnalyticsService {customEVFilterApplied: boolean,parsedFilters: Array<{ [key: string]: string }>,): Promise<object | void> {- return this.generateParams(+ return this.generateParams2(parsedFilters,subQuery,customEVFilterApplied,@@ -2968,7 +3068,7 @@ export class AnalyticsService {const promises = [// Getting params(async () => {- params = await this.generateParams(+ params = await this.generateParams2(null,subQuery,false,@@ -3038,7 +3138,7 @@ export class AnalyticsService {const promises = [// Params(async () => {- params = await this.generateParams(+ params = await this.generateParams2(null,subQuery,false,@@ -3281,7 +3381,7 @@ export class AnalyticsService {const promises = [// Getting params(async () => {- params = await this.generateParams(+ params = await this.generateParams2(null,subQuery,false,@@ -3731,7 +3831,7 @@ export class AnalyticsService {WHEREpid = {pid:FixedString(12)}AND psid = {psid:String}- AND unique = 1+ ORDER BY created ASCLIMIT 1;`diff --git a/backend/apps/production/src/analytics/decorators/validate-period.decorator.ts b/backend/apps/production/src/analytics/decorators/validate-period.decorator.tsindex 917104e61..4e2950c6f 100644--- a/backend/apps/production/src/analytics/decorators/validate-period.decorator.ts+++ b/backend/apps/production/src/analytics/decorators/validate-period.decorator.ts@@ -24,7 +24,7 @@ export function ValidatePeriod(validationOptions?: ValidationOptions) {options: validationOptions,validator: {validate(value: any) {- if (value === undefined || value === '') {+ if (value === undefined || value === '' || value === 'custom') {return true // Allow empty or undefined values}if (!_includes(VALID_PERIODS, value)) {diff --git a/backend/apps/production/src/project/project.service.ts b/backend/apps/production/src/project/project.service.tsindex 56ef46a63..64a4a84f0 100644--- a/backend/apps/production/src/project/project.service.ts+++ b/backend/apps/production/src/project/project.service.ts@@ -1320,8 +1320,14 @@ export class ProjectService {from: string,to: string,): Promise<number> {- const query =- "SELECT count() FROM analytics WHERE pid = {pid:FixedString(12)} AND created BETWEEN {from:String} AND {to:String} AND unique='1'"+ const query = `+ SELECT+ count(DISTINCT psid)+ FROM analytics+ WHERE+ pid = {pid:FixedString(12)}+ AND created BETWEEN {from:String} AND {to:String}+ `const params = { pid, from, to }let count = 0diff --git a/backend/apps/selfhosted/src/analytics/decorators/validate-period.decorator.ts b/backend/apps/selfhosted/src/analytics/decorators/validate-period.decorator.tsindex 917104e61..4e2950c6f 100644--- a/backend/apps/selfhosted/src/analytics/decorators/validate-period.decorator.ts+++ b/backend/apps/selfhosted/src/analytics/decorators/validate-period.decorator.ts@@ -24,7 +24,7 @@ export function ValidatePeriod(validationOptions?: ValidationOptions) {options: validationOptions,validator: {validate(value: any) {- if (value === undefined || value === '') {+ if (value === undefined || value === '' || value === 'custom') {return true // Allow empty or undefined values}if (!_includes(VALID_PERIODS, value)) {diff --git a/backend/meta/clickhouse/generate-dummy-data.js b/backend/meta/clickhouse/generate-dummy-data.jsindex f864f886e..a779e3a8b 100644--- a/backend/meta/clickhouse/generate-dummy-data.js+++ b/backend/meta/clickhouse/generate-dummy-data.js@@ -3,7 +3,7 @@ const { faker } = require('@faker-js/faker')const _ = require('lodash')const chalk = require('chalk')-const { queriesRunner, dbName } = require('../../migrations/clickhouse/setup')+const { clickhouse, dbName } = require('../../migrations/clickhouse/setup')const PID_SIZE = 12const DEFAULT_ROW_COUNT = 10@@ -111,86 +111,76 @@ const CITIES = ['Kobe', // Japan]-const chunkArray = (arr, chunkSize = 1000) => {- const result = []- for (let i = 0; i < arr.length; i += chunkSize) {- result.push(arr.slice(i, i + chunkSize))- }- return result-}-const insertData = async (pid, rowCount, processedRecords, table) => {- const insertQueries = []-- const recordChunks = chunkArray(processedRecords, 1000)-- for (const chunk of recordChunks) {- const insertQuery = `INSERT INTO ${dbName}.${table} (*) VALUES ${chunk.join(- ',',- )}`- insertQueries.push(insertQuery)- }-- if (recordChunks.length > 10) {- console.warn(- chalk.yellow(- '[WARNING] The number of records is big, so it might take longer to insert the records',- ),- )- }-try {- const result = await queriesRunner(insertQueries, false)-- if (!result) {- throw new Error()- }+ await clickhouse.insert({+ table,+ format: 'JSONEachRow',+ values: processedRecords,+ clickhouse_settings: {+ async_insert: 1,+ },+ })console.log(chalk.green(`[SUCCESS] Successfully inserted ${rowCount} records for ${pid}!`,),)- } catch {+ } catch (reason) {console.error(chalk.red('[ERROR] Error occured whilst inserting the records'),)+ console.error(reason)}}+const PSIDS = [+ '123456789012',+ '123456789013',+ '123456789014',+ '123456789015',+ '123456789016',+ '123456789017',+ '123456789018',+]+const generateAnalyticsData = async (pid, rowCount, from, to) => {const records = []for (let i = 0; i < rowCount; ++i) {- const record = [- 0, // psid- null, // sid+ records.push({+ psid:+ Math.random() < 0.05+ ? faker.helpers.arrayElement(PSIDS)+ : Math.floor(Math.random() * 1000000000000000000),pid,- faker.helpers.arrayElement(PAGES), // pg- null, // prev- faker.helpers.arrayElement(DEVICES), // dv- faker.helpers.arrayElement(BROWSERS), // br- faker.helpers.arrayElement(OS), // os- _.toLower(faker.location.countryCode()), // lc- faker.helpers.arrayElement(URLS), // ref- faker.helpers.arrayElement(UTM_SOURCES), // so- faker.helpers.arrayElement(UTM_MEDIUMS), // me- faker.helpers.arrayElement(UTM_CAMPAIGNS), // ca- faker.location.countryCode(), // cc- faker.helpers.arrayElement(REGIONS), // rg- faker.helpers.arrayElement(CITIES), // ct- faker.number.int({ min: 0, max: 10000 }), // sdur- faker.number.int({ min: 0, max: 9 }) === 0 ? 1 : 0, // unique; 10% chance of unique record- _.split(+ pg: faker.helpers.arrayElement(PAGES),+ dv: faker.helpers.arrayElement(DEVICES),+ br: faker.helpers.arrayElement(BROWSERS),+ brv: null,+ os: faker.helpers.arrayElement(OS),+ osv: null,+ lc: _.toLower(faker.location.countryCode()),+ ref: faker.helpers.arrayElement(URLS),+ so: faker.helpers.arrayElement(UTM_SOURCES),+ me: faker.helpers.arrayElement(UTM_MEDIUMS),+ ca: faker.helpers.arrayElement(UTM_CAMPAIGNS),+ te: null,+ co: null,+ cc: faker.location.countryCode(),+ rg: faker.helpers.arrayElement(REGIONS),+ ct: faker.helpers.arrayElement(CITIES),+ unique: faker.number.int({ min: 0, max: 9 }) === 0 ? 1 : 0,+ created: _.split(faker.date.between({ from, to }).toISOString().replace('T', ' ').replace('Z', ''),'.',- )[0], // created; format the date string- ]- records.push(record)+ )[0],+ })}console.info(@@ -199,32 +189,28 @@ const generateAnalyticsData = async (pid, rowCount, from, to) => {),)- const processedRecords = records.map(- record => `(${record.map(item => `'${item}'`).join(',')})`,- )- insertData(pid, rowCount, processedRecords, 'analytics')+ insertData(pid, rowCount, records, 'analytics')}const generateCaptchaData = async (pid, rowCount, from, to) => {const records = []for (let i = 0; i < rowCount; ++i) {- const record = [+ records.push({pid,- faker.helpers.arrayElement(DEVICES), // dv- faker.helpers.arrayElement(BROWSERS), // br- faker.helpers.arrayElement(OS), // os- faker.location.countryCode(), // cc- _.split(+ dv: faker.helpers.arrayElement(DEVICES),+ br: faker.helpers.arrayElement(BROWSERS),+ os: faker.helpers.arrayElement(OS),+ cc: faker.location.countryCode(),+ created: _.split(faker.date.between({ from, to }).toISOString().replace('T', ' ').replace('Z', ''),'.',- )[0], // created; format the date string- ]- records.push(record)+ )[0],+ })}console.info(@@ -233,41 +219,42 @@ const generateCaptchaData = async (pid, rowCount, from, to) => {),)- const processedRecords = records.map(- record => `(${record.map(item => `'${item}'`).join(',')})`,- )- insertData(pid, rowCount, processedRecords, 'captcha')+ insertData(pid, rowCount, records, 'captcha')}const generateCustomEventsData = async (pid, rowCount, from, to) => {const records = []for (let i = 0; i < rowCount; ++i) {- const record = [+ records.push({pid,- faker.helpers.arrayElement(CUSTOM_EVENTS), // ev- faker.helpers.arrayElement(PAGES), // pg- faker.helpers.arrayElement(DEVICES), // dv- faker.helpers.arrayElement(BROWSERS), // br- faker.helpers.arrayElement(OS), // os- _.toLower(faker.location.countryCode()), // lc- faker.helpers.arrayElement(URLS), // ref- faker.helpers.arrayElement(UTM_SOURCES), // so- faker.helpers.arrayElement(UTM_MEDIUMS), // me- faker.helpers.arrayElement(UTM_CAMPAIGNS), // ca- faker.location.countryCode(), // cc- faker.helpers.arrayElement(REGIONS), // rg- faker.helpers.arrayElement(CITIES), // ct- _.split(+ ev: faker.helpers.arrayElement(CUSTOM_EVENTS),+ pg: faker.helpers.arrayElement(PAGES),+ dv: faker.helpers.arrayElement(DEVICES),+ br: faker.helpers.arrayElement(BROWSERS),+ brv: null,+ os: faker.helpers.arrayElement(OS),+ osv: null,+ lc: _.toLower(faker.location.countryCode()),+ ref: faker.helpers.arrayElement(URLS),+ so: faker.helpers.arrayElement(UTM_SOURCES),+ me: faker.helpers.arrayElement(UTM_MEDIUMS),+ ca: faker.helpers.arrayElement(UTM_CAMPAIGNS),+ te: null,+ co: null,+ cc: faker.location.countryCode(),+ rg: faker.helpers.arrayElement(REGIONS),+ ct: faker.helpers.arrayElement(CITIES),+ unique: faker.number.int({ min: 0, max: 9 }) === 0 ? 1 : 0,+ created: _.split(faker.date.between({ from, to }).toISOString().replace('T', ' ').replace('Z', ''),'.',- )[0], // created; format the date string- ]- records.push(record)+ )[0],+ })}console.info(@@ -276,42 +263,39 @@ const generateCustomEventsData = async (pid, rowCount, from, to) => {),)- const processedRecords = records.map(- record => `(${record.map(item => `'${item}'`).join(',')})`,- )- insertData(pid, rowCount, processedRecords, 'customEV')+ insertData(pid, rowCount, records, 'customEV')}const generatePerformanceData = async (pid, rowCount, from, to) => {const records = []for (let i = 0; i < rowCount; ++i) {- const record = [+ records.push({pid,- faker.helpers.arrayElement(PAGES), // pg- faker.helpers.arrayElement(DEVICES), // dv- faker.helpers.arrayElement(BROWSERS), // br- faker.location.countryCode(), // cc- faker.helpers.arrayElement(REGIONS), // rg- faker.helpers.arrayElement(CITIES), // ct- faker.number.int({ min: 0, max: 500 }), // dns- faker.number.int({ min: 0, max: 1000 }), // tls- faker.number.int({ min: 0, max: 100 }), // conn- faker.number.int({ min: 0, max: 500 }), // response- faker.number.int({ min: 0, max: 1500 }), // render- faker.number.int({ min: 300, max: 2000 }), // domLoad- faker.number.int({ min: 300, max: 2000 }), // pageLoad- faker.number.int({ min: 0, max: 500 }), // ttfb- _.split(+ pg: faker.helpers.arrayElement(PAGES),+ dv: faker.helpers.arrayElement(DEVICES),+ br: faker.helpers.arrayElement(BROWSERS),+ brv: null,+ cc: faker.location.countryCode(),+ rg: faker.helpers.arrayElement(REGIONS),+ ct: faker.helpers.arrayElement(CITIES),+ dns: faker.number.int({ min: 0, max: 500 }),+ tls: faker.number.int({ min: 0, max: 1000 }),+ conn: faker.number.int({ min: 0, max: 100 }),+ response: faker.number.int({ min: 0, max: 500 }),+ render: faker.number.int({ min: 0, max: 1500 }),+ domLoad: faker.number.int({ min: 300, max: 2000 }),+ pageLoad: faker.number.int({ min: 300, max: 2000 }),+ ttfb: faker.number.int({ min: 0, max: 500 }),+ created: _.split(faker.date.between({ from, to }).toISOString().replace('T', ' ').replace('Z', ''),'.',- )[0], // created; format the date string- ]- records.push(record)+ )[0],+ })}console.info(@@ -320,10 +304,7 @@ const generatePerformanceData = async (pid, rowCount, from, to) => {),)- const processedRecords = records.map(- record => `(${record.map(item => `'${item}'`).join(',')})`,- )- insertData(pid, rowCount, processedRecords, 'performance')+ insertData(pid, rowCount, records, 'performance')}const main = () => {diff --git a/backend/migrations/clickhouse/2024_10_28_broken.js b/backend/migrations/clickhouse/2024_10_28_broken.jsnew file mode 100644index 000000000..5f0e82c61--- /dev/null+++ b/backend/migrations/clickhouse/2024_10_28_broken.js@@ -0,0 +1,60 @@+const { queriesRunner, dbName } = require('./setup')++const queries = [+ // Step 1: Create Temporary Table+ `+ CREATE TABLE IF NOT EXISTS ${dbName}.analytics_temp AS ${dbName}.analytics+ ENGINE = MergeTree()+ PARTITION BY toYYYYMM(created)+ ORDER BY (pid, created);+ `,++ // Step 2: Insert Data with Backfilled psid+ // TODO: Refactor it. It will fail if there are more than 50 million rows.+ `INSERT INTO ${dbName}.analytics_temp+ SELECT+ IF(psid IS NOT NULL, psid,+ dense_rank() OVER (+ PARTITION BY toDate(created), br, os, lc, cc, rg, ct+ ORDER BY created+ )+ ) AS psid,+ pid,+ pg,+ NULL as prev,+ dv,+ br,+ brv,+ os,+ osv,+ lc,+ ref,+ so,+ me,+ ca,+ te,+ co,+ cc,+ rg,+ ct,+ meta.key,+ meta.value,+ unique,+ created+ FROM ${dbName}.analytics`,++ // Step 3: Swap Tables+ `+ RENAME TABLE ${dbName}.analytics TO ${dbName}.analytics_backup, ${dbName}.analytics_temp TO ${dbName}.analytics;+ `,++ // Step 4: Drop Backup Table (Optional)+ // `+ // DROP TABLE ${dbName}.analytics_backup;+ // `,++ // Step 5: Drop the Obsolete `unique` Column+ // `ALTER TABLE ${dbName}.analytics DROP COLUMN IF EXISTS unique;`,+]++queriesRunner(queries, true)diff --git a/backend/migrations/clickhouse/2024_10_28_sort_of_working.js b/backend/migrations/clickhouse/2024_10_28_sort_of_working.jsnew file mode 100644index 000000000..00dd7ed70--- /dev/null+++ b/backend/migrations/clickhouse/2024_10_28_sort_of_working.js@@ -0,0 +1,107 @@+const { queriesRunner, dbName } = require('./setup')++// Generates insert queries from 2022-01-01 to 2023-10-17 AND where psid is NULL.+const generateInsertQueries = () => {+ const queries = []+ const currentDate = new Date('2021-01-01')+ const endDate = new Date('2023-10-18')++ while (currentDate <= endDate) {+ // Calculate the start of next month+ const nextMonth = new Date(currentDate)+ nextMonth.setMonth(nextMonth.getMonth() + 1)++ // Ensure we don't go past the end date+ const queryEndDate = nextMonth > endDate ? endDate : nextMonth++ queries.push(`INSERT INTO ${dbName}.analytics_temp+ WITH+ SessionStarts AS (+ SELECT+ *,+ cityHash64(+ toString(toDate(created)),+ toString(br),+ toString(os),+ toString(lc),+ toString(cc),+ toString(rg),+ toString(ct)+ ) AS params_hash,+ 1000000 + dense_rank() OVER (+ PARTITION BY toDate(created)+ ORDER BY br, os, lc, cc, rg, ct, created+ ) AS session_id+ FROM ${dbName}.analytics+ WHERE isNull(psid) AND created >= parseDateTimeBestEffort('${currentDate.toISOString()}') AND created < parseDateTimeBestEffort('${queryEndDate.toISOString()}')+ ),+ SessionsAssigned AS (+ SELECT+ a.*,+ LAST_VALUE(IF(a.unique = 1, a.session_id, NULL)) OVER (+ PARTITION BY toDate(a.created), a.br, a.os, a.lc, a.cc, a.rg, a.ct+ ORDER BY a.created+ RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW+ ) AS new_psid+ FROM SessionStarts a+ )+ SELECT+ CASE+ WHEN isNotNull(psid) OR created >= '2023-10-18' THEN psid+ ELSE new_psid+ END AS psid,+ pid,+ pg,+ dv,+ br,+ brv,+ os,+ osv,+ lc,+ ref,+ so,+ me,+ ca,+ te,+ co,+ cc,+ rg,+ ct,+ meta.key,+ meta.value,+ unique,+ created+ FROM SessionsAssigned+ WHERE new_psid IS NOT NULL OR created >= '2023-10-18';`)++ // Move to the next month+ currentDate.setMonth(currentDate.getMonth() + 1)+ }++ return queries+}++const queries = [+ `DROP TABLE IF EXISTS ${dbName}.analytics_temp;`,++ // Step 1: Create Temporary Table+ `CREATE TABLE ${dbName}.analytics_temp AS ${dbName}.analytics+ ENGINE = MergeTree()+ PARTITION BY toYYYYMM(created)+ ORDER BY (pid, created);`,++ ...generateInsertQueries(),+ `INSERT INTO analytics_temp SELECT * FROM analytics WHERE created >= '2023-10-18';`,++ // Step 3: Swap Tables+ `RENAME TABLE ${dbName}.analytics TO ${dbName}.analytics_backup,+ ${dbName}.analytics_temp TO ${dbName}.analytics;`,++ // Step 4: Drop Backup Table (Optional)+ // `DROP TABLE ${dbName}.analytics_backup;`,++ // Step 5: Drop the Obsolete 'unique' Column+ // `ALTER TABLE ${dbName}.analytics DROP COLUMN IF EXISTS unique;`+]++queriesRunner(queries, true)diff --git a/backend/migrations/clickhouse/setup.js b/backend/migrations/clickhouse/setup.jsindex 7d259754d..fc7fc94eb 100644--- a/backend/migrations/clickhouse/setup.js+++ b/backend/migrations/clickhouse/setup.js@@ -62,7 +62,15 @@ const databaselessQueriesRunner = async queries => {}}-const queriesRunner = async (queries, log = true) => {+/**+ * @typedef {import('@clickhouse/client').CommandParams} CommandParams+ *+ * @param {string[]} queries+ * @param {boolean} log+ * @param {Omit<CommandParams, 'query'>} params+ * @returns+ */+const queriesRunner = async (queries, log = true, params = {}) => {let failed = falsefor (const query of queries) {@@ -74,6 +82,7 @@ const queriesRunner = async (queries, log = true) => {try {// eslint-disable-next-line no-await-in-loopawait clickhouse.command({+ ...params,query,})