Skip to content

📈 Open source, privacy-first web analytics — a cookie-less Google Analytics alternative with built-in error tracking and performance monitoring. Self-host it or use our cloud. Get started for free!

selfhosted fixed & meta clickhouse improvements

Blue Mousec0564e92aa63
diff --git a/apps/selfhosted/src/analytics/analytics.service.ts b/apps/selfhosted/src/analytics/analytics.service.ts
index 2eb1cff05..d9df423a4 100644
--- a/apps/selfhosted/src/analytics/analytics.service.ts
+++ b/apps/selfhosted/src/analytics/analytics.service.ts
@@ -1170,13 +1170,13 @@ export class AnalyticsService {
// eslint-disable-next-line
let _to: string
- if (_isEmpty(period) || period === 'custom') {
+ if (_isEmpty(period) || ['today', 'yesterday', 'custom'].includes(period)) {
const safeTimezone = this.getSafeTimezone(timezone)
const { groupFrom, groupTo } = this.getGroupFromTo(
from,
to,
- null,
+ ['today', 'yesterday'].includes(period) ? TimeBucketType.HOUR : null,
period,
safeTimezone,
)
diff --git a/meta/clickhouse/generate-dummy-data.js b/meta/clickhouse/generate-dummy-data.js
index 6b679146b..63450ccaa 100644
--- a/meta/clickhouse/generate-dummy-data.js
+++ b/meta/clickhouse/generate-dummy-data.js
@@ -3,9 +3,7 @@ const { faker } = require('@faker-js/faker')
const _ = require('lodash')
const chalk = require('chalk')
-const {
- queriesRunner, dbName,
-} = require('../../migrations/clickhouse/setup')
+const { queriesRunner, dbName } = require('../../migrations/clickhouse/setup')
const PID_SIZE = 12
const DEFAULT_ROW_COUNT = 10
@@ -20,20 +18,97 @@ const PAGES = ['/', '/signup', '/login', '/news', '/some-interesting-page.html']
const URLS = Array.from({ length: 36 }, () => faker.internet.url())
const DEVICES = ['desktop', 'mobile', 'tablet', 'wearable']
const BROWSERS = ['Chrome', 'Firefox', 'Safari', 'Opera', 'Edge', 'IE']
-const CUSTOM_EVENTS = ['click', 'hover', 'scroll', 'submit', 'error', 'signup', 'oauth', 'test', 'hello', '235535123433']
+const CUSTOM_EVENTS = [
+ 'click',
+ 'hover',
+ 'scroll',
+ 'submit',
+ 'error',
+ 'signup',
+ 'oauth',
+ 'test',
+ 'hello',
+ '235535123433',
+]
const REGIONS = [
- 'England', 'Scotland', 'Wales', 'Northern Ireland', // UK
- 'Alabama', 'Alaska', 'Arizona', 'Arkansas', 'California', 'Colorado', 'Connecticut', 'Delaware', // USA
- 'Kyiv Oblast', 'Kharkiv Oblast', 'Vinnytsia Oblast', 'Donetsk Oblast', 'Odessa Oblast', 'Zaporizhia Oblast', 'Lviv Oblast', 'Kiev', // Ukraine
- 'Ontario', 'Quebec', 'Nova Scotia', 'New Brunswick', 'Manitoba', 'British Columbia', 'Prince Edward Island', 'Saskatchewan', // Canada
- 'Tokyo', 'Osaka', 'Aichi', 'Hokkaido', 'Fukuoka', 'Kanagawa', 'Ibaraki', 'Saitama', // Japan
+ 'England',
+ 'Scotland',
+ 'Wales',
+ 'Northern Ireland', // UK
+ 'Alabama',
+ 'Alaska',
+ 'Arizona',
+ 'Arkansas',
+ 'California',
+ 'Colorado',
+ 'Connecticut',
+ 'Delaware', // USA
+ 'Kyiv Oblast',
+ 'Kharkiv Oblast',
+ 'Vinnytsia Oblast',
+ 'Donetsk Oblast',
+ 'Odessa Oblast',
+ 'Zaporizhia Oblast',
+ 'Lviv Oblast',
+ 'Kiev', // Ukraine
+ 'Ontario',
+ 'Quebec',
+ 'Nova Scotia',
+ 'New Brunswick',
+ 'Manitoba',
+ 'British Columbia',
+ 'Prince Edward Island',
+ 'Saskatchewan', // Canada
+ 'Tokyo',
+ 'Osaka',
+ 'Aichi',
+ 'Hokkaido',
+ 'Fukuoka',
+ 'Kanagawa',
+ 'Ibaraki',
+ 'Saitama', // Japan
]
const CITIES = [
- 'London', 'Birmingham', 'Dundee', 'Liverpool', 'Bristol', 'Manchester', 'Sheffield', 'Leeds', // UK
- 'New York', 'Los Angeles', 'Chicago', 'Houston', 'Phoenix', 'Philadelphia', 'San Antonio', 'San Diego', // USA
- 'Kyiv', 'Kharkiv', 'Vinnytsia', 'Donetsk', 'Odessa', 'Zaporizhia', 'Lviv', 'Dnipro', // Ukraine
- 'Toronto', 'Montreal', 'Vancouver', 'Ottawa', 'Calgary', 'Edmonton', 'Winnipeg', 'Quebec City', // Canada
- 'Tokyo', 'Osaka', 'Nagoya', 'Sapporo', 'Fukuoka', 'Yokohama', 'Sendai', 'Kobe', // Japan
+ 'London',
+ 'Birmingham',
+ 'Dundee',
+ 'Liverpool',
+ 'Bristol',
+ 'Manchester',
+ 'Sheffield',
+ 'Leeds', // UK
+ 'New York',
+ 'Los Angeles',
+ 'Chicago',
+ 'Houston',
+ 'Phoenix',
+ 'Philadelphia',
+ 'San Antonio',
+ 'San Diego', // USA
+ 'Kyiv',
+ 'Kharkiv',
+ 'Vinnytsia',
+ 'Donetsk',
+ 'Odessa',
+ 'Zaporizhia',
+ 'Lviv',
+ 'Dnipro', // Ukraine
+ 'Toronto',
+ 'Montreal',
+ 'Vancouver',
+ 'Ottawa',
+ 'Calgary',
+ 'Edmonton',
+ 'Winnipeg',
+ 'Quebec City', // Canada
+ 'Tokyo',
+ 'Osaka',
+ 'Nagoya',
+ 'Sapporo',
+ 'Fukuoka',
+ 'Yokohama',
+ 'Sendai',
+ 'Kobe', // Japan
]
const chunkArray = (arr, chunkSize = 1000) => {
@@ -50,13 +125,17 @@ const insertData = async (pid, rowCount, processedRecords, table) => {
const recordChunks = chunkArray(processedRecords, 1000)
for (const chunk of recordChunks) {
- const insertQuery = `INSERT INTO ${dbName}.${table} (*) VALUES ${chunk.join(',')}`
+ const insertQuery = `INSERT INTO ${dbName}.${table} (*) VALUES ${chunk.join(
+ ',',
+ )}`
insertQueries.push(insertQuery)
}
if (recordChunks.length > 10) {
console.warn(
- chalk.yellow('[WARNING] The number of records is big, so it might take longer to insert the records')
+ chalk.yellow(
+ '[WARNING] The number of records is big, so it might take longer to insert the records',
+ ),
)
}
@@ -68,11 +147,13 @@ const insertData = async (pid, rowCount, processedRecords, table) => {
}
console.log(
- chalk.green(`[SUCCESS] Successfully inserted ${rowCount} records for ${pid}!`)
+ chalk.green(
+ `[SUCCESS] Successfully inserted ${rowCount} records for ${pid}!`,
+ ),
)
} catch {
console.error(
- chalk.red('[ERROR] Error occured whilst inserting the records')
+ chalk.red('[ERROR] Error occured whilst inserting the records'),
)
}
}
@@ -82,6 +163,7 @@ const generateAnalyticsData = async (pid, rowCount, from, to) => {
for (let i = 0; i < rowCount; ++i) {
const record = [
+ 0, // psid
'NULL', // sid
pid,
faker.helpers.arrayElement(PAGES), // pg
@@ -99,16 +181,27 @@ const generateAnalyticsData = async (pid, rowCount, from, to) => {
faker.helpers.arrayElement(CITIES), // ct
faker.number.int({ min: 0, max: 10000 }), // sdur
faker.number.int({ min: 0, max: 9 }) === 0 ? 1 : 0, // unique; 10% chance of unique record
- _.split(faker.date.between({ from, to }).toISOString().replace('T', ' ').replace('Z', ''), '.')[0], // created; format the date string
+ _.split(
+ faker.date
+ .between({ from, to })
+ .toISOString()
+ .replace('T', ' ')
+ .replace('Z', ''),
+ '.',
+ )[0], // created; format the date string
]
records.push(record)
}
console.info(
- chalk.cyan(`[INFO] Inserting the records into Clickhouse "${dbName}" database...`)
+ chalk.cyan(
+ `[INFO] Inserting the records into Clickhouse "${dbName}" database...`,
+ ),
)
- const processedRecords = records.map(record => `(${record.map(item => `'${item}'`).join(',')})`)
+ const processedRecords = records.map(
+ record => `(${record.map(item => `'${item}'`).join(',')})`,
+ )
insertData(pid, rowCount, processedRecords, 'analytics')
}
@@ -123,16 +216,27 @@ const generateCaptchaData = async (pid, rowCount, from, to) => {
faker.helpers.arrayElement(OS), // os
faker.location.countryCode(), // cc
faker.number.int({ min: 0, max: 2 }) === 0 ? 1 : 0, // manuallyPassed; 33% chance of manuallyPassed record
- _.split(faker.date.between({ from, to }).toISOString().replace('T', ' ').replace('Z', ''), '.')[0], // created; format the date string
+ _.split(
+ faker.date
+ .between({ from, to })
+ .toISOString()
+ .replace('T', ' ')
+ .replace('Z', ''),
+ '.',
+ )[0], // created; format the date string
]
records.push(record)
}
console.info(
- chalk.cyan(`[INFO] Inserting the records into Clickhouse "${dbName}" database...`)
+ chalk.cyan(
+ `[INFO] Inserting the records into Clickhouse "${dbName}" database...`,
+ ),
)
- const processedRecords = records.map(record => `(${record.map(item => `'${item}'`).join(',')})`)
+ const processedRecords = records.map(
+ record => `(${record.map(item => `'${item}'`).join(',')})`,
+ )
insertData(pid, rowCount, processedRecords, 'captcha')
}
@@ -155,16 +259,27 @@ const generateCustomEventsData = async (pid, rowCount, from, to) => {
faker.location.countryCode(), // cc
faker.helpers.arrayElement(REGIONS), // rg
faker.helpers.arrayElement(CITIES), // ct
- _.split(faker.date.between({ from, to }).toISOString().replace('T', ' ').replace('Z', ''), '.')[0], // created; format the date string
+ _.split(
+ faker.date
+ .between({ from, to })
+ .toISOString()
+ .replace('T', ' ')
+ .replace('Z', ''),
+ '.',
+ )[0], // created; format the date string
]
records.push(record)
}
console.info(
- chalk.cyan(`[INFO] Inserting the records into Clickhouse "${dbName}" database...`)
+ chalk.cyan(
+ `[INFO] Inserting the records into Clickhouse "${dbName}" database...`,
+ ),
)
- const processedRecords = records.map(record => `(${record.map(item => `'${item}'`).join(',')})`)
+ const processedRecords = records.map(
+ record => `(${record.map(item => `'${item}'`).join(',')})`,
+ )
insertData(pid, rowCount, processedRecords, 'customEV')
}
@@ -188,16 +303,27 @@ const generatePerformanceData = async (pid, rowCount, from, to) => {
faker.number.int({ min: 300, max: 2000 }), // domLoad
faker.number.int({ min: 300, max: 2000 }), // pageLoad
faker.number.int({ min: 0, max: 500 }), // ttfb
- _.split(faker.date.between({ from, to }).toISOString().replace('T', ' ').replace('Z', ''), '.')[0], // created; format the date string
+ _.split(
+ faker.date
+ .between({ from, to })
+ .toISOString()
+ .replace('T', ' ')
+ .replace('Z', ''),
+ '.',
+ )[0], // created; format the date string
]
records.push(record)
}
console.info(
- chalk.cyan(`[INFO] Inserting the records into Clickhouse "${dbName}" database...`)
+ chalk.cyan(
+ `[INFO] Inserting the records into Clickhouse "${dbName}" database...`,
+ ),
)
- const processedRecords = records.map(record => `(${record.map(item => `'${item}'`).join(',')})`)
+ const processedRecords = records.map(
+ record => `(${record.map(item => `'${item}'`).join(',')})`,
+ )
insertData(pid, rowCount, processedRecords, 'performance')
}
@@ -207,18 +333,28 @@ const main = () => {
if (_.includes(args, '--help')) {
console.log(
- chalk.cyan('Usage: node generate-dummy-data.js --pid <pid> --beginDate <YYYY-MM-DD@HH:mm:ss> --endDate <YYYY-MM-DD@HH:mm:ss> --rows <number of rows>')
+ chalk.cyan(
+ 'Usage: node generate-dummy-data.js --pid <pid> --beginDate <YYYY-MM-DD@HH:mm:ss> --endDate <YYYY-MM-DD@HH:mm:ss> --rows <number of rows>',
+ ),
)
console.log(
- chalk.green('Example: node generate-dummy-data.js --pid STEzHcB1rALV --beginDate 2021-01-01@00:00:00 --endDate 2021-01-31@23:59:59 --rows 1000')
+ chalk.green(
+ 'Example: node generate-dummy-data.js --pid STEzHcB1rALV --beginDate 2021-01-01@00:00:00 --endDate 2021-01-31@23:59:59 --rows 1000',
+ ),
)
console.log()
console.log('Options:')
console.log('--pid <pid> - The pid of the project')
- console.log('--beginDate <YYYY-MM-DD@HH:mm:ss> - The begin date of the data')
+ console.log(
+ '--beginDate <YYYY-MM-DD@HH:mm:ss> - The begin date of the data',
+ )
console.log('--endDate <YYYY-MM-DD@HH:mm:ss> - The end date of the data')
- console.log(`--rows <number of rows> - The number of rows to generate; default is ${DEFAULT_ROW_COUNT}`)
- console.log(`--mode <analytics | custom_events | performance> - The mode to generate data for; default is ${DEFAULT_MODE}`)
+ console.log(
+ `--rows <number of rows> - The number of rows to generate; default is ${DEFAULT_ROW_COUNT}`,
+ )
+ console.log(
+ `--mode <analytics | custom_events | performance> - The mode to generate data for; default is ${DEFAULT_MODE}`,
+ )
return
}
@@ -230,68 +366,79 @@ const main = () => {
}
})
- const {
- pid, beginDate, endDate, rows, mode = DEFAULT_MODE,
- } = argMap
+ const { pid, beginDate, endDate, rows, mode = DEFAULT_MODE } = argMap
const rowCount = parseInt(rows) || DEFAULT_ROW_COUNT
if (_.isEmpty(argMap)) {
- console.error(
- chalk.red('[ERROR] No arguments specified')
- )
+ console.error(chalk.red('[ERROR] No arguments specified'))
console.log(
- chalk.cyan('Usage: node generate-dummy-data.js --pid <pid> --beginDate <YYYY-MM-DD@HH:mm:ss> --endDate <YYYY-MM-DD@HH:mm:ss> --rows <number of rows>')
+ chalk.cyan(
+ 'Usage: node generate-dummy-data.js --pid <pid> --beginDate <YYYY-MM-DD@HH:mm:ss> --endDate <YYYY-MM-DD@HH:mm:ss> --rows <number of rows>',
+ ),
)
console.log(
- chalk.green('Example: node generate-dummy-data.js --pid STEzHcB1rALV --beginDate 2021-01-01@00:00:00 --endDate 2021-01-31@23:59:59 --rows 1000')
+ chalk.green(
+ 'Example: node generate-dummy-data.js --pid STEzHcB1rALV --beginDate 2021-01-01@00:00:00 --endDate 2021-01-31@23:59:59 --rows 1000',
+ ),
)
return
}
if (!_.includes(MODES, mode)) {
console.error(
- chalk.red(`[ERROR] Invalid mode specified, it must be one of ${MODES}`)
+ chalk.red(`[ERROR] Invalid mode specified, it must be one of ${MODES}`),
)
return
}
if (!pid) {
- console.error(
- chalk.red('[ERROR] No pid specified')
- )
+ console.error(chalk.red('[ERROR] No pid specified'))
return
}
if (pid.length !== PID_SIZE) {
console.error(
- chalk.red(`[ERROR] Invalid pid specified, it must be ${PID_SIZE} characters long`)
+ chalk.red(
+ `[ERROR] Invalid pid specified, it must be ${PID_SIZE} characters long`,
+ ),
)
return
}
if (!beginDate) {
console.error(
- chalk.red('[ERROR] No begin date specified, it must be in the format YYYY-MM-DD@HH:mm:ss')
+ chalk.red(
+ '[ERROR] No begin date specified, it must be in the format YYYY-MM-DD@HH:mm:ss',
+ ),
)
return
}
if (!endDate) {
console.error(
- chalk.red('[ERROR] No end date specified, it must be in the format YYYY-MM-DD@HH:mm:ss')
+ chalk.red(
+ '[ERROR] No end date specified, it must be in the format YYYY-MM-DD@HH:mm:ss',
+ ),
)
return
}
+ console.log(chalk.cyan(`[INFO] Operating in ${mode} mode`))
console.log(
- chalk.cyan(`[INFO] Operating in ${mode} mode`)
- )
- console.log(
- chalk.cyan(`[INFO] Generating ${rowCount} records for ${pid} between ${beginDate} and ${endDate}...`)
+ chalk.cyan(
+ `[INFO] Generating ${rowCount} records for ${pid} between ${beginDate} and ${endDate}...`,
+ ),
)
- const arguments = [pid, rowCount, _.join(_.split(beginDate, '@'), ' '), _.join(_.split(endDate, '@'), ' ')]
+ const arguments = [
+ pid,
+ rowCount,
+ // Date.parse(`${_.join(_.split(beginDate, '@'), 'T')}.000Z`),
+ // Date.parse(`${_.join(_.split(endDate, '@'), 'T')}.000Z`),
+ _.join(_.split(beginDate, '@'), 'T'),
+ _.join(_.split(endDate, '@'), 'T'),
+ ]
if (mode === 'analytics') {
generateAnalyticsData(...arguments)
diff --git a/package.json b/package.json
index d42c6ea57..a48c7577d 100644
--- a/package.json
+++ b/package.json
@@ -26,7 +26,7 @@
"find-deadcode": "ts-prune"
},
"dependencies": {
- "@faker-js/faker": "^8.3.1",
+ "@faker-js/faker": "^8.4.1",
"@nestjs-modules/mailer": "^2.0.2",
"@nestjs/axios": "^3.0.2",
"@nestjs/common": "^10.2.10",