📈 Open source, privacy-first web analytics — a cookie-less Google Analytics alternative with built-in error tracking and performance monitoring. Self-host it or use our cloud. Get started for free!
TypeScript chartsclickhousedockergdprgoogle-analyticsmetricsmonitoringmysqlnestjsprivacyredisself-hostedselfhostedstatisticstailwindcssweb-analyticswebanalyticswebsite
added support for performance & custom events tables
Blue Mousea394dfa5ee93
diff --git a/meta/clickhouse/generate-dummy-data.js b/meta/clickhouse/generate-dummy-data.jsindex bd86f681b..d2230eff1 100644--- a/meta/clickhouse/generate-dummy-data.js+++ b/meta/clickhouse/generate-dummy-data.js@@ -8,6 +8,8 @@ const {const PID_SIZE = 12const DEFAULT_ROW_COUNT = 10+const MODES = ['analytics', 'custom_events', 'performance']+const DEFAULT_MODE = 'analytics'const OS = ['Windows', 'Linux', 'Mac']const UTM_SOURCES = ['organic', 'paid', 'referral']@@ -17,14 +19,11 @@ const PAGES = ['/', '/signup', '/login', '/news', '/some-interesting-page.html']const URLS = _.map(_.times(36, _.constant(null)), faker.internet.url)const DEVICES = ['desktop', 'mobile', 'tablet', 'wearable']const BROWSERS = ['Chrome', 'Firefox', 'Safari', 'Opera', 'Edge', 'IE']+const CUSTOM_EVENTS = ['click', 'hover', 'scroll', 'submit', 'error', 'signup', 'oauth', 'test', 'hello', '235535123433']-const generateData = async (pid, rowCount, beginDate, endDate) => {+const generateAnalyticsData = async (pid, rowCount, beginDate, endDate) => {const records = []- console.info(- chalk.cyan(`[INFO] Generating ${rowCount} records for ${pid}...`)- )-for (let i = 0; i < rowCount; ++i) {const record = ['NULL', // sid@@ -70,7 +69,99 @@ const generateData = async (pid, rowCount, beginDate, endDate) => {}}-// TODO: ADD SUPPORT FOR --type <analytics | custom_events | performance> ARGUMENT TO GENERATE DATA FOR DIFFERENT TABLES+const generateCustomEventsData = async (pid, rowCount, beginDate, endDate) => {+ const records = []++ for (let i = 0; i < rowCount; ++i) {+ const record = [+ pid,+ faker.helpers.arrayElement(CUSTOM_EVENTS), // ev+ faker.helpers.arrayElement(PAGES), // pg+ faker.helpers.arrayElement(DEVICES), // dv+ faker.helpers.arrayElement(BROWSERS), // br+ faker.helpers.arrayElement(OS), // os+ _.toLower(faker.address.countryCode()), // lc+ faker.helpers.arrayElement(URLS), // ref+ faker.helpers.arrayElement(UTM_SOURCES), // so+ faker.helpers.arrayElement(UTM_MEDIUMS), // me+ faker.helpers.arrayElement(UTM_CAMPAIGNS), // ca+ faker.address.countryCode(), // cc+ _.split(faker.date.between(beginDate, endDate).toISOString().replace('T', ' ').replace('Z', ''), '.')[0], // created; format the date string+ ]+ records.push(record)+ }++ console.info(+ chalk.cyan(`[INFO] Inserting the records into Clickhouse "${dbName}" database...`)+ )++ const processedRecords = records.map(record => `(${record.map(item => `'${item}'`).join(',')})`).join(',')+ const insertQuery = `INSERT INTO ${dbName}.customEV (*) VALUES ${processedRecords}`++ try {+ const result = await queriesRunner([insertQuery], false)++ if (!result) {+ throw new Error()+ }++ console.log(+ chalk.green(`[SUCCESS] Successfully inserted ${rowCount} records for ${pid}!`)+ )+ } catch {+ console.error(+ chalk.red('[ERROR] Error occured whilst inserting the records')+ )+ }+}++const generatePerformanceData = async (pid, rowCount, beginDate, endDate) => {+ const records = []++ for (let i = 0; i < rowCount; ++i) {+ const record = [+ pid,+ faker.helpers.arrayElement(PAGES), // pg+ faker.helpers.arrayElement(DEVICES), // dv+ faker.helpers.arrayElement(BROWSERS), // br+ faker.address.countryCode(), // cc+ faker.datatype.number({ min: 0, max: 500 }), // dns+ faker.datatype.number({ min: 0, max: 1000 }), // tls+ faker.datatype.number({ min: 0, max: 100 }), // conn+ faker.datatype.number({ min: 0, max: 500 }), // response+ faker.datatype.number({ min: 0, max: 1500 }), // render+ faker.datatype.number({ min: 300, max: 2000 }), // domLoad+ faker.datatype.number({ min: 300, max: 2000 }), // pageLoad+ faker.datatype.number({ min: 0, max: 500 }), // ttfb+ _.split(faker.date.between(beginDate, endDate).toISOString().replace('T', ' ').replace('Z', ''), '.')[0], // created; format the date string+ ]+ records.push(record)+ }++ console.info(+ chalk.cyan(`[INFO] Inserting the records into Clickhouse "${dbName}" database...`)+ )++ const processedRecords = records.map(record => `(${record.map(item => `'${item}'`).join(',')})`).join(',')+ const insertQuery = `INSERT INTO ${dbName}.performance (*) VALUES ${processedRecords}`++ try {+ const result = await queriesRunner([insertQuery], false)++ if (!result) {+ throw new Error()+ }++ console.log(+ chalk.green(`[SUCCESS] Successfully inserted ${rowCount} records for ${pid}!`)+ )+ } catch {+ console.error(+ chalk.red('[ERROR] Error occured whilst inserting the records')+ )+ }+}+const main = () => {const args = process.argv.slice(2)const argMap = {}@@ -82,6 +173,13 @@ const main = () => {console.log(chalk.green('Example: node generate-dummy-data.js --pid STEzHcB1rALV --beginDate 2021-01-01!00:00:00 --endDate 2021-01-31!23:59:59 --rows 1000'))+ console.log()+ console.log('Options:')+ console.log('--pid <pid> - The pid of the project')+ console.log('--beginDate <YYYY-MM-DD!HH:mm:ss> - The begin date of the data')+ console.log('--endDate <YYYY-MM-DD!HH:mm:ss> - The end date of the data')+ console.log('--rows <number of rows> - The number of rows to generate')+ console.log('--mode <analytics | custom_events | performance> - The mode to generate data for; default is "analytics"')return}@@ -94,7 +192,7 @@ const main = () => {})const {- pid, beginDate, endDate, rows,+ pid, beginDate, endDate, rows, mode = DEFAULT_MODE,} = argMapconst rowCount = parseInt(rows) || DEFAULT_ROW_COUNT@@ -112,6 +210,13 @@ const main = () => {return}+ if (!_.includes(MODES, mode)) {+ console.error(+ chalk.red(`[ERROR] Invalid mode specified, it must be one of ${MODES}`)+ )+ return+ }+if (!pid) {console.error(chalk.red('[ERROR] No pid specified')@@ -140,7 +245,26 @@ const main = () => {return}- generateData(pid, rowCount, _.join(_.split(beginDate, '@'), ' '), _.join(_.split(endDate, '@'), ' '))+ console.log(+ chalk.cyan(`[INFO] Operating in ${mode} mode`)+ )+ console.log(+ chalk.cyan(`[INFO] Generating ${rowCount} records for ${pid} between ${beginDate} and ${endDate}...`)+ )++ const arguments = [pid, rowCount, _.join(_.split(beginDate, '@'), ' '), _.join(_.split(endDate, '@'), ' ')]++ if (mode === 'analytics') {+ generateAnalyticsData(...arguments)+ }++ if (mode === 'custom_events') {+ generateCustomEventsData(...arguments)+ }++ if (mode === 'performance') {+ generatePerformanceData(...arguments)+ }}main()