📈 Open source, privacy-first web analytics — a cookie-less Google Analytics alternative with built-in error tracking and performance monitoring. Self-host it or use our cloud. Get started for free!
TypeScript chartsclickhousedockergdprgoogle-analyticsmetricsmonitoringmysqlnestjsprivacyredisself-hostedselfhostedstatisticstailwindcssweb-analyticswebanalyticswebsite
selfhosted fixed & meta clickhouse improvements
Blue Mousec0564e92aa63
diff --git a/apps/selfhosted/src/analytics/analytics.service.ts b/apps/selfhosted/src/analytics/analytics.service.tsindex 2eb1cff05..d9df423a4 100644--- a/apps/selfhosted/src/analytics/analytics.service.ts+++ b/apps/selfhosted/src/analytics/analytics.service.ts@@ -1170,13 +1170,13 @@ export class AnalyticsService {// eslint-disable-next-linelet _to: string- if (_isEmpty(period) || period === 'custom') {+ if (_isEmpty(period) || ['today', 'yesterday', 'custom'].includes(period)) {const safeTimezone = this.getSafeTimezone(timezone)const { groupFrom, groupTo } = this.getGroupFromTo(from,to,- null,+ ['today', 'yesterday'].includes(period) ? TimeBucketType.HOUR : null,period,safeTimezone,)diff --git a/meta/clickhouse/generate-dummy-data.js b/meta/clickhouse/generate-dummy-data.jsindex 6b679146b..63450ccaa 100644--- a/meta/clickhouse/generate-dummy-data.js+++ b/meta/clickhouse/generate-dummy-data.js@@ -3,9 +3,7 @@ const { faker } = require('@faker-js/faker')const _ = require('lodash')const chalk = require('chalk')-const {- queriesRunner, dbName,-} = require('../../migrations/clickhouse/setup')+const { queriesRunner, dbName } = require('../../migrations/clickhouse/setup')const PID_SIZE = 12const DEFAULT_ROW_COUNT = 10@@ -20,20 +18,97 @@ const PAGES = ['/', '/signup', '/login', '/news', '/some-interesting-page.html']const URLS = Array.from({ length: 36 }, () => faker.internet.url())const DEVICES = ['desktop', 'mobile', 'tablet', 'wearable']const BROWSERS = ['Chrome', 'Firefox', 'Safari', 'Opera', 'Edge', 'IE']-const CUSTOM_EVENTS = ['click', 'hover', 'scroll', 'submit', 'error', 'signup', 'oauth', 'test', 'hello', '235535123433']+const CUSTOM_EVENTS = [+ 'click',+ 'hover',+ 'scroll',+ 'submit',+ 'error',+ 'signup',+ 'oauth',+ 'test',+ 'hello',+ '235535123433',+]const REGIONS = [- 'England', 'Scotland', 'Wales', 'Northern Ireland', // UK- 'Alabama', 'Alaska', 'Arizona', 'Arkansas', 'California', 'Colorado', 'Connecticut', 'Delaware', // USA- 'Kyiv Oblast', 'Kharkiv Oblast', 'Vinnytsia Oblast', 'Donetsk Oblast', 'Odessa Oblast', 'Zaporizhia Oblast', 'Lviv Oblast', 'Kiev', // Ukraine- 'Ontario', 'Quebec', 'Nova Scotia', 'New Brunswick', 'Manitoba', 'British Columbia', 'Prince Edward Island', 'Saskatchewan', // Canada- 'Tokyo', 'Osaka', 'Aichi', 'Hokkaido', 'Fukuoka', 'Kanagawa', 'Ibaraki', 'Saitama', // Japan+ 'England',+ 'Scotland',+ 'Wales',+ 'Northern Ireland', // UK+ 'Alabama',+ 'Alaska',+ 'Arizona',+ 'Arkansas',+ 'California',+ 'Colorado',+ 'Connecticut',+ 'Delaware', // USA+ 'Kyiv Oblast',+ 'Kharkiv Oblast',+ 'Vinnytsia Oblast',+ 'Donetsk Oblast',+ 'Odessa Oblast',+ 'Zaporizhia Oblast',+ 'Lviv Oblast',+ 'Kiev', // Ukraine+ 'Ontario',+ 'Quebec',+ 'Nova Scotia',+ 'New Brunswick',+ 'Manitoba',+ 'British Columbia',+ 'Prince Edward Island',+ 'Saskatchewan', // Canada+ 'Tokyo',+ 'Osaka',+ 'Aichi',+ 'Hokkaido',+ 'Fukuoka',+ 'Kanagawa',+ 'Ibaraki',+ 'Saitama', // Japan]const CITIES = [- 'London', 'Birmingham', 'Dundee', 'Liverpool', 'Bristol', 'Manchester', 'Sheffield', 'Leeds', // UK- 'New York', 'Los Angeles', 'Chicago', 'Houston', 'Phoenix', 'Philadelphia', 'San Antonio', 'San Diego', // USA- 'Kyiv', 'Kharkiv', 'Vinnytsia', 'Donetsk', 'Odessa', 'Zaporizhia', 'Lviv', 'Dnipro', // Ukraine- 'Toronto', 'Montreal', 'Vancouver', 'Ottawa', 'Calgary', 'Edmonton', 'Winnipeg', 'Quebec City', // Canada- 'Tokyo', 'Osaka', 'Nagoya', 'Sapporo', 'Fukuoka', 'Yokohama', 'Sendai', 'Kobe', // Japan+ 'London',+ 'Birmingham',+ 'Dundee',+ 'Liverpool',+ 'Bristol',+ 'Manchester',+ 'Sheffield',+ 'Leeds', // UK+ 'New York',+ 'Los Angeles',+ 'Chicago',+ 'Houston',+ 'Phoenix',+ 'Philadelphia',+ 'San Antonio',+ 'San Diego', // USA+ 'Kyiv',+ 'Kharkiv',+ 'Vinnytsia',+ 'Donetsk',+ 'Odessa',+ 'Zaporizhia',+ 'Lviv',+ 'Dnipro', // Ukraine+ 'Toronto',+ 'Montreal',+ 'Vancouver',+ 'Ottawa',+ 'Calgary',+ 'Edmonton',+ 'Winnipeg',+ 'Quebec City', // Canada+ 'Tokyo',+ 'Osaka',+ 'Nagoya',+ 'Sapporo',+ 'Fukuoka',+ 'Yokohama',+ 'Sendai',+ 'Kobe', // Japan]const chunkArray = (arr, chunkSize = 1000) => {@@ -50,13 +125,17 @@ const insertData = async (pid, rowCount, processedRecords, table) => {const recordChunks = chunkArray(processedRecords, 1000)for (const chunk of recordChunks) {- const insertQuery = `INSERT INTO ${dbName}.${table} (*) VALUES ${chunk.join(',')}`+ const insertQuery = `INSERT INTO ${dbName}.${table} (*) VALUES ${chunk.join(+ ',',+ )}`insertQueries.push(insertQuery)}if (recordChunks.length > 10) {console.warn(- chalk.yellow('[WARNING] The number of records is big, so it might take longer to insert the records')+ chalk.yellow(+ '[WARNING] The number of records is big, so it might take longer to insert the records',+ ),)}@@ -68,11 +147,13 @@ const insertData = async (pid, rowCount, processedRecords, table) => {}console.log(- chalk.green(`[SUCCESS] Successfully inserted ${rowCount} records for ${pid}!`)+ chalk.green(+ `[SUCCESS] Successfully inserted ${rowCount} records for ${pid}!`,+ ),)} catch {console.error(- chalk.red('[ERROR] Error occured whilst inserting the records')+ chalk.red('[ERROR] Error occured whilst inserting the records'),)}}@@ -82,6 +163,7 @@ const generateAnalyticsData = async (pid, rowCount, from, to) => {for (let i = 0; i < rowCount; ++i) {const record = [+ 0, // psid'NULL', // sidpid,faker.helpers.arrayElement(PAGES), // pg@@ -99,16 +181,27 @@ const generateAnalyticsData = async (pid, rowCount, from, to) => {faker.helpers.arrayElement(CITIES), // ctfaker.number.int({ min: 0, max: 10000 }), // sdurfaker.number.int({ min: 0, max: 9 }) === 0 ? 1 : 0, // unique; 10% chance of unique record- _.split(faker.date.between({ from, to }).toISOString().replace('T', ' ').replace('Z', ''), '.')[0], // created; format the date string+ _.split(+ faker.date+ .between({ from, to })+ .toISOString()+ .replace('T', ' ')+ .replace('Z', ''),+ '.',+ )[0], // created; format the date string]records.push(record)}console.info(- chalk.cyan(`[INFO] Inserting the records into Clickhouse "${dbName}" database...`)+ chalk.cyan(+ `[INFO] Inserting the records into Clickhouse "${dbName}" database...`,+ ),)- const processedRecords = records.map(record => `(${record.map(item => `'${item}'`).join(',')})`)+ const processedRecords = records.map(+ record => `(${record.map(item => `'${item}'`).join(',')})`,+ )insertData(pid, rowCount, processedRecords, 'analytics')}@@ -123,16 +216,27 @@ const generateCaptchaData = async (pid, rowCount, from, to) => {faker.helpers.arrayElement(OS), // osfaker.location.countryCode(), // ccfaker.number.int({ min: 0, max: 2 }) === 0 ? 1 : 0, // manuallyPassed; 33% chance of manuallyPassed record- _.split(faker.date.between({ from, to }).toISOString().replace('T', ' ').replace('Z', ''), '.')[0], // created; format the date string+ _.split(+ faker.date+ .between({ from, to })+ .toISOString()+ .replace('T', ' ')+ .replace('Z', ''),+ '.',+ )[0], // created; format the date string]records.push(record)}console.info(- chalk.cyan(`[INFO] Inserting the records into Clickhouse "${dbName}" database...`)+ chalk.cyan(+ `[INFO] Inserting the records into Clickhouse "${dbName}" database...`,+ ),)- const processedRecords = records.map(record => `(${record.map(item => `'${item}'`).join(',')})`)+ const processedRecords = records.map(+ record => `(${record.map(item => `'${item}'`).join(',')})`,+ )insertData(pid, rowCount, processedRecords, 'captcha')}@@ -155,16 +259,27 @@ const generateCustomEventsData = async (pid, rowCount, from, to) => {faker.location.countryCode(), // ccfaker.helpers.arrayElement(REGIONS), // rgfaker.helpers.arrayElement(CITIES), // ct- _.split(faker.date.between({ from, to }).toISOString().replace('T', ' ').replace('Z', ''), '.')[0], // created; format the date string+ _.split(+ faker.date+ .between({ from, to })+ .toISOString()+ .replace('T', ' ')+ .replace('Z', ''),+ '.',+ )[0], // created; format the date string]records.push(record)}console.info(- chalk.cyan(`[INFO] Inserting the records into Clickhouse "${dbName}" database...`)+ chalk.cyan(+ `[INFO] Inserting the records into Clickhouse "${dbName}" database...`,+ ),)- const processedRecords = records.map(record => `(${record.map(item => `'${item}'`).join(',')})`)+ const processedRecords = records.map(+ record => `(${record.map(item => `'${item}'`).join(',')})`,+ )insertData(pid, rowCount, processedRecords, 'customEV')}@@ -188,16 +303,27 @@ const generatePerformanceData = async (pid, rowCount, from, to) => {faker.number.int({ min: 300, max: 2000 }), // domLoadfaker.number.int({ min: 300, max: 2000 }), // pageLoadfaker.number.int({ min: 0, max: 500 }), // ttfb- _.split(faker.date.between({ from, to }).toISOString().replace('T', ' ').replace('Z', ''), '.')[0], // created; format the date string+ _.split(+ faker.date+ .between({ from, to })+ .toISOString()+ .replace('T', ' ')+ .replace('Z', ''),+ '.',+ )[0], // created; format the date string]records.push(record)}console.info(- chalk.cyan(`[INFO] Inserting the records into Clickhouse "${dbName}" database...`)+ chalk.cyan(+ `[INFO] Inserting the records into Clickhouse "${dbName}" database...`,+ ),)- const processedRecords = records.map(record => `(${record.map(item => `'${item}'`).join(',')})`)+ const processedRecords = records.map(+ record => `(${record.map(item => `'${item}'`).join(',')})`,+ )insertData(pid, rowCount, processedRecords, 'performance')}@@ -207,18 +333,28 @@ const main = () => {if (_.includes(args, '--help')) {console.log(- chalk.cyan('Usage: node generate-dummy-data.js --pid <pid> --beginDate <YYYY-MM-DD@HH:mm:ss> --endDate <YYYY-MM-DD@HH:mm:ss> --rows <number of rows>')+ chalk.cyan(+ 'Usage: node generate-dummy-data.js --pid <pid> --beginDate <YYYY-MM-DD@HH:mm:ss> --endDate <YYYY-MM-DD@HH:mm:ss> --rows <number of rows>',+ ),)console.log(- chalk.green('Example: node generate-dummy-data.js --pid STEzHcB1rALV --beginDate 2021-01-01@00:00:00 --endDate 2021-01-31@23:59:59 --rows 1000')+ chalk.green(+ 'Example: node generate-dummy-data.js --pid STEzHcB1rALV --beginDate 2021-01-01@00:00:00 --endDate 2021-01-31@23:59:59 --rows 1000',+ ),)console.log()console.log('Options:')console.log('--pid <pid> - The pid of the project')- console.log('--beginDate <YYYY-MM-DD@HH:mm:ss> - The begin date of the data')+ console.log(+ '--beginDate <YYYY-MM-DD@HH:mm:ss> - The begin date of the data',+ )console.log('--endDate <YYYY-MM-DD@HH:mm:ss> - The end date of the data')- console.log(`--rows <number of rows> - The number of rows to generate; default is ${DEFAULT_ROW_COUNT}`)- console.log(`--mode <analytics | custom_events | performance> - The mode to generate data for; default is ${DEFAULT_MODE}`)+ console.log(+ `--rows <number of rows> - The number of rows to generate; default is ${DEFAULT_ROW_COUNT}`,+ )+ console.log(+ `--mode <analytics | custom_events | performance> - The mode to generate data for; default is ${DEFAULT_MODE}`,+ )return}@@ -230,68 +366,79 @@ const main = () => {}})- const {- pid, beginDate, endDate, rows, mode = DEFAULT_MODE,- } = argMap+ const { pid, beginDate, endDate, rows, mode = DEFAULT_MODE } = argMapconst rowCount = parseInt(rows) || DEFAULT_ROW_COUNTif (_.isEmpty(argMap)) {- console.error(- chalk.red('[ERROR] No arguments specified')- )+ console.error(chalk.red('[ERROR] No arguments specified'))console.log(- chalk.cyan('Usage: node generate-dummy-data.js --pid <pid> --beginDate <YYYY-MM-DD@HH:mm:ss> --endDate <YYYY-MM-DD@HH:mm:ss> --rows <number of rows>')+ chalk.cyan(+ 'Usage: node generate-dummy-data.js --pid <pid> --beginDate <YYYY-MM-DD@HH:mm:ss> --endDate <YYYY-MM-DD@HH:mm:ss> --rows <number of rows>',+ ),)console.log(- chalk.green('Example: node generate-dummy-data.js --pid STEzHcB1rALV --beginDate 2021-01-01@00:00:00 --endDate 2021-01-31@23:59:59 --rows 1000')+ chalk.green(+ 'Example: node generate-dummy-data.js --pid STEzHcB1rALV --beginDate 2021-01-01@00:00:00 --endDate 2021-01-31@23:59:59 --rows 1000',+ ),)return}if (!_.includes(MODES, mode)) {console.error(- chalk.red(`[ERROR] Invalid mode specified, it must be one of ${MODES}`)+ chalk.red(`[ERROR] Invalid mode specified, it must be one of ${MODES}`),)return}if (!pid) {- console.error(- chalk.red('[ERROR] No pid specified')- )+ console.error(chalk.red('[ERROR] No pid specified'))return}if (pid.length !== PID_SIZE) {console.error(- chalk.red(`[ERROR] Invalid pid specified, it must be ${PID_SIZE} characters long`)+ chalk.red(+ `[ERROR] Invalid pid specified, it must be ${PID_SIZE} characters long`,+ ),)return}if (!beginDate) {console.error(- chalk.red('[ERROR] No begin date specified, it must be in the format YYYY-MM-DD@HH:mm:ss')+ chalk.red(+ '[ERROR] No begin date specified, it must be in the format YYYY-MM-DD@HH:mm:ss',+ ),)return}if (!endDate) {console.error(- chalk.red('[ERROR] No end date specified, it must be in the format YYYY-MM-DD@HH:mm:ss')+ chalk.red(+ '[ERROR] No end date specified, it must be in the format YYYY-MM-DD@HH:mm:ss',+ ),)return}+ console.log(chalk.cyan(`[INFO] Operating in ${mode} mode`))console.log(- chalk.cyan(`[INFO] Operating in ${mode} mode`)- )- console.log(- chalk.cyan(`[INFO] Generating ${rowCount} records for ${pid} between ${beginDate} and ${endDate}...`)+ chalk.cyan(+ `[INFO] Generating ${rowCount} records for ${pid} between ${beginDate} and ${endDate}...`,+ ),)- const arguments = [pid, rowCount, _.join(_.split(beginDate, '@'), ' '), _.join(_.split(endDate, '@'), ' ')]+ const arguments = [+ pid,+ rowCount,+ // Date.parse(`${_.join(_.split(beginDate, '@'), 'T')}.000Z`),+ // Date.parse(`${_.join(_.split(endDate, '@'), 'T')}.000Z`),+ _.join(_.split(beginDate, '@'), 'T'),+ _.join(_.split(endDate, '@'), 'T'),+ ]if (mode === 'analytics') {generateAnalyticsData(...arguments)diff --git a/package.json b/package.jsonindex d42c6ea57..a48c7577d 100644--- a/package.json+++ b/package.json@@ -26,7 +26,7 @@"find-deadcode": "ts-prune"},"dependencies": {- "@faker-js/faker": "^8.3.1",+ "@faker-js/faker": "^8.4.1","@nestjs-modules/mailer": "^2.0.2","@nestjs/axios": "^3.0.2","@nestjs/common": "^10.2.10",