📈 Open source, privacy-first web analytics — a cookie-less Google Analytics alternative with built-in error tracking and performance monitoring. Self-host it or use our cloud. Get started for free!
TypeScript chartsclickhousedockergdprgoogle-analyticsmetricsmonitoringmysqlnestjsprivacyredisself-hostedselfhostedstatisticstailwindcssweb-analyticswebanalyticswebsite
added support for cities & regions
Blue Mouse89e9bdf39c10
diff --git a/apps/production/src/analytics/analytics.controller.ts b/apps/production/src/analytics/analytics.controller.tsindex aba6f3028..02ab94c8b 100644--- a/apps/production/src/analytics/analytics.controller.ts+++ b/apps/production/src/analytics/analytics.controller.ts@@ -9,7 +9,7 @@ import * as dayjs from 'dayjs'import * as utc from 'dayjs/plugin/utc'import * as dayjsTimezone from 'dayjs/plugin/timezone'import { hash } from 'blake3'-import ct from 'countries-and-timezones'+import timezones from 'countries-and-timezones'import {Controller,Body,@@ -92,6 +92,8 @@ const analyticsDTO = (me: string,ca: string,cc: string,+ rg: string,+ ct: string,sdur: number | string,unique: number,): Array<string | number> => {@@ -109,6 +111,8 @@ const analyticsDTO = (me,ca,cc,+ rg,+ ct,sdur,unique,dayjs.utc().format('YYYY-MM-DD HH:mm:ss'),@@ -121,6 +125,8 @@ const performanceDTO = (dv: string,br: string,cc: string,+ rg: string,+ ct: string,dns: number,tls: number,conn: number,@@ -136,6 +142,8 @@ const performanceDTO = (dv,br,cc,+ rg,+ ct,_round(dns),_round(tls),_round(conn),@@ -161,6 +169,8 @@ const customLogDTO = (me: string,ca: string,cc: string,+ rg: string,+ ct: string,): Array<string | number> => {return [pid,@@ -175,6 +185,8 @@ const customLogDTO = (me,ca,cc,+ rg,+ ct,dayjs.utc().format('YYYY-MM-DD HH:mm:ss'),]}@@ -835,13 +847,16 @@ export class AnalyticsController {}}+ const { city = 'NULL', region = 'NULL' } =+ this.analyticsService.getIPDetails(ip)+const ua = UAParser(userAgent)const dv = ua.device.type || 'desktop'const br = ua.browser.nameconst os = ua.os.name// trying to get country from timezome, otherwise using CloudFlare's IP based country code as a fallbackconst cc =- ct.getCountryForTimezone(eventsDTO.tz)?.id ||+ timezones.getCountryForTimezone(eventsDTO.tz)?.id ||(headers['cf-ipcountry'] === 'XX' ? 'NULL' : headers['cf-ipcountry'])const dto = customLogDTO(@@ -857,6 +872,8 @@ export class AnalyticsController {eventsDTO.me,eventsDTO.ca,cc,+ region,+ city,)try {@@ -930,9 +947,11 @@ export class AnalyticsController {// trying to get country from timezome, otherwise using CloudFlare's IP based country code as a fallbackconst cc =- ct.getCountryForTimezone(logDTO.tz)?.id ||+ timezones.getCountryForTimezone(logDTO.tz)?.id ||(headers['cf-ipcountry'] === 'XX' ? 'NULL' : headers['cf-ipcountry'])+ const { city = 'NULL', region = 'NULL' } =+ this.analyticsService.getIPDetails(ip)const ua = UAParser(userAgent)const dv = ua.device.type || 'desktop'const br = ua.browser.name@@ -952,6 +971,8 @@ export class AnalyticsController {logDTO.me,logDTO.ca,cc,+ region,+ city,0,Number(unique),)@@ -976,6 +997,8 @@ export class AnalyticsController {dv,br,cc,+ region,+ city,dns,tls,conn,@@ -1036,6 +1059,8 @@ export class AnalyticsController {await this.analyticsService.processInteractionSD(sessionHash, logDTO.pid)+ const { city = 'NULL', region = 'NULL' } =+ this.analyticsService.getIPDetails(ip)const ua = UAParser(userAgent)const dv = ua.device.type || 'desktop'const br = ua.browser.name@@ -1058,6 +1083,8 @@ export class AnalyticsController {'NULL','NULL',cc,+ region,+ city,0,Number(unique),)diff --git a/apps/production/src/analytics/analytics.service.ts b/apps/production/src/analytics/analytics.service.tsindex 69c11809e..067889d56 100644--- a/apps/production/src/analytics/analytics.service.ts+++ b/apps/production/src/analytics/analytics.service.ts@@ -45,6 +45,7 @@ import {import {calculateRelativePercentage,millisecondsToSeconds,+ lookup,} from '../common/utils'import { PageviewsDTO } from './dto/pageviews.dto'import { EventsDTO } from './dto/events.dto'@@ -233,6 +234,12 @@ const isValidOrigin = (origins: string[], origin: string) => {return false}+interface IPGeoDetails {+ country?: string+ region?: string+ city?: string+}+@Injectable()export class AnalyticsService {constructor(private readonly projectService: ProjectService) {}@@ -813,6 +820,20 @@ export class AnalyticsService {return result}+ getIPDetails(ip: string): IPGeoDetails {+ const data = lookup.get(ip)++ const country = data?.country?.names?.en+ const city = data?.city?.names?.en+ const region = data?.subdivisions?.[0]?.names?.en++ return {+ country,+ city,+ region,+ }+ }+async getFilters(pid: string, type: string): Promise<Array<string>> {if (!_includes(TRAFFIC_COLUMNS, type)) {throw new UnprocessableEntityException(diff --git a/apps/production/src/common/utils.ts b/apps/production/src/common/utils.tsindex 71b79da75..e3a083d9e 100644--- a/apps/production/src/common/utils.ts+++ b/apps/production/src/common/utils.ts@@ -1,3 +1,6 @@+import { Reader, CityResponse } from 'maxmind'+import * as path from 'path'+import * as fs from 'fs'import { HttpException } from '@nestjs/common'import { hash } from 'blake3'import * as randomstring from 'randomstring'@@ -130,6 +133,38 @@ const millisecondsToSeconds = (milliseconds: number) => milliseconds / 1000const generateRandomString = (length: number): string =>randomstring.generate(length)+const dummyLookup = () => ({+ country: {+ names: {+ en: 'NULL',+ },+ },+ city: {+ names: {+ en: 'NULL',+ },+ },+ subdivisions: [+ {+ names: {+ en: 'NULL',+ },+ },+ ],+})++const GEOIP_DB_PATH = path.join(__dirname, '../../../..', 'dbip-city-lite.mmdb')++// @ts-ignore+// eslint-disable-next-line+let lookup: Reader<CityResponse> = dummyLookup++// TODO: CHECK IF GEO-IP DB EXISTS+if (fs.existsSync(GEOIP_DB_PATH)) {+ const buffer = fs.readFileSync(GEOIP_DB_PATH)+ lookup = new Reader<CityResponse>(buffer)+}+export {getRandomTip,checkRateLimit,@@ -138,4 +173,5 @@ export {millisecondsToSeconds,generateRandomString,nFormatter,+ lookup,}diff --git a/meta/clickhouse/generate-dummy-data.js b/meta/clickhouse/generate-dummy-data.jsindex c8afb6dc3..735c2d582 100644--- a/meta/clickhouse/generate-dummy-data.js+++ b/meta/clickhouse/generate-dummy-data.js@@ -80,6 +80,8 @@ const generateAnalyticsData = async (pid, rowCount, from, to) => {faker.helpers.arrayElement(UTM_MEDIUMS), // mefaker.helpers.arrayElement(UTM_CAMPAIGNS), // cafaker.location.countryCode(), // cc+ faker.location.state(), // rg+ faker.location.city(), // ctfaker.number.int({ min: 0, max: 10000 }), // sdurfaker.number.int({ min: 0, max: 9 }) === 0 ? 1 : 0, // unique; 10% chance of unique record_.split(faker.date.between({ from, to }).toISOString().replace('T', ' ').replace('Z', ''), '.')[0], // created; format the date string@@ -136,6 +138,8 @@ const generateCustomEventsData = async (pid, rowCount, from, to) => {faker.helpers.arrayElement(UTM_MEDIUMS), // mefaker.helpers.arrayElement(UTM_CAMPAIGNS), // cafaker.location.countryCode(), // cc+ faker.location.state(), // rg+ faker.location.city(), // ct_.split(faker.date.between({ from, to }).toISOString().replace('T', ' ').replace('Z', ''), '.')[0], // created; format the date string]records.push(record)@@ -159,6 +163,8 @@ const generatePerformanceData = async (pid, rowCount, from, to) => {faker.helpers.arrayElement(DEVICES), // dvfaker.helpers.arrayElement(BROWSERS), // brfaker.location.countryCode(), // cc+ faker.location.state(), // rg+ faker.location.city(), // ctfaker.number.int({ min: 0, max: 500 }), // dnsfaker.number.int({ min: 0, max: 1000 }), // tlsfaker.number.int({ min: 0, max: 100 }), // conndiff --git a/migrations/clickhouse/2023_06_26.js b/migrations/clickhouse/2023_06_26.jsnew file mode 100644index 000000000..b401f4e18--- /dev/null+++ b/migrations/clickhouse/2023_06_26.js@@ -0,0 +1,99 @@+const { queriesRunner, dbName } = require('./setup')++const queries = [+ // Analytics table: added 'rg' and 'ct' columns+ `DROP TABLE IF EXISTS ${dbName}.analytics_temp`,+ `CREATE TABLE IF NOT EXISTS ${dbName}.analytics_temp+ (+ sid Nullable(String),+ pid FixedString(12),+ pg Nullable(String),+ prev Nullable(String),+ dv LowCardinality(Nullable(String)),+ br LowCardinality(Nullable(String)),+ os LowCardinality(Nullable(String)),+ lc LowCardinality(Nullable(String)),+ ref Nullable(String),+ so Nullable(String),+ me Nullable(String),+ ca Nullable(String),+ cc Nullable(FixedString(2)),+ rg LowCardinality(Nullable(String)),+ ct Nullable(String),+ sdur Nullable(UInt32),+ unique UInt8,+ created DateTime('UTC')+ )+ ENGINE = MergeTree()+ PARTITION BY toYYYYMM(created)+ ORDER BY (pid, created);`,++ `INSERT INTO ${dbName}.analytics_temp (sid, pid, pg, prev, dv, br, os, lc, ref, so, me, ca, cc, rg, ct, sdur, unique, created)+ SELECT sid, pid, pg, prev, dv, br, os, lc, ref, so, me, ca, cc, NULL, NULL, sdur, unique, created FROM ${dbName}.analytics`,++ `DROP TABLE ${dbName}.analytics`,+ `RENAME TABLE ${dbName}.analytics_temp TO ${dbName}.analytics`,++ // Custom events table: added 'rg' and 'ct' columns+ `DROP TABLE IF EXISTS ${dbName}.customEV_temp`,+ `CREATE TABLE IF NOT EXISTS ${dbName}.customEV+ (+ pid FixedString(12),+ ev String,+ pg Nullable(String),+ dv LowCardinality(Nullable(String)),+ br LowCardinality(Nullable(String)),+ os LowCardinality(Nullable(String)),+ lc LowCardinality(Nullable(String)),+ ref Nullable(String),+ so Nullable(String),+ me Nullable(String),+ ca Nullable(String),+ cc Nullable(FixedString(2)),+ rg LowCardinality(Nullable(String)),+ ct Nullable(String),+ created DateTime('UTC')+ )+ ENGINE = MergeTree()+ PARTITION BY toYYYYMM(created)+ ORDER BY (pid, created);`,++ `INSERT INTO ${dbName}.customEV_temp (pid, ev, pg, dv, br, os, lc, ref, so, me, ca, cc, rg, ct, created)+ SELECT pid, ev, pg, dv, br, os, lc, ref, so, me, ca, cc, NULL, NULL, created FROM ${dbName}.customEV`,++ `DROP TABLE ${dbName}.customEV`,+ `RENAME TABLE ${dbName}.customEV_temp TO ${dbName}.customEV`,++ // Performance table: added 'rg' and 'ct' columns+ `DROP TABLE IF EXISTS ${dbName}.performance_temp`,+ `CREATE TABLE IF NOT EXISTS ${dbName}.performance+ (+ pid FixedString(12),+ pg Nullable(String),+ dv LowCardinality(Nullable(String)),+ br LowCardinality(Nullable(String)),+ cc Nullable(FixedString(2)),+ rg LowCardinality(Nullable(String)),+ ct Nullable(String),+ dns Nullable(UInt32),+ tls Nullable(UInt32),+ conn Nullable(UInt32),+ response Nullable(UInt32),+ render Nullable(UInt32),+ domLoad Nullable(UInt32),+ pageLoad Nullable(UInt32),+ ttfb Nullable(UInt32),+ created DateTime('UTC')+ )+ ENGINE = MergeTree()+ PARTITION BY toYYYYMM(created)+ ORDER BY (pid, created);`,++ `INSERT INTO ${dbName}.performance_temp (pid, pg, dv, br, cc, rg, ct, dns, tls, conn, response, render, domLoad, pageLoad, ttfb, created)+ SELECT pid, pg, dv, br, cc, NULL, NULL, dns, tls, conn, response, render, domLoad, pageLoad, ttfb, created FROM ${dbName}.performance`,++ `DROP TABLE ${dbName}.performance`,+ `RENAME TABLE ${dbName}.performance_temp TO ${dbName}.performance`,+]++queriesRunner(queries)diff --git a/migrations/clickhouse/initialise_database.js b/migrations/clickhouse/initialise_database.jsindex 1d529f370..fde352fce 100644--- a/migrations/clickhouse/initialise_database.js+++ b/migrations/clickhouse/initialise_database.js@@ -22,6 +22,8 @@ const CLICKHOUSE_INIT_QUERIES = [me Nullable(String),ca Nullable(String),cc Nullable(FixedString(2)),+ rg LowCardinality(Nullable(String)),+ ct Nullable(String),sdur Nullable(UInt32),unique UInt8,created DateTime('UTC')@@ -45,6 +47,8 @@ const CLICKHOUSE_INIT_QUERIES = [me Nullable(String),ca Nullable(String),cc Nullable(FixedString(2)),+ rg LowCardinality(Nullable(String)),+ ct Nullable(String),created DateTime('UTC'))ENGINE = MergeTree()@@ -59,6 +63,8 @@ const CLICKHOUSE_INIT_QUERIES = [dv LowCardinality(Nullable(String)),br LowCardinality(Nullable(String)),cc Nullable(FixedString(2)),+ rg LowCardinality(Nullable(String)),+ ct Nullable(String),dns Nullable(UInt32),tls Nullable(UInt32),conn Nullable(UInt32),