Skip to content

📈 Open source, privacy-first web analytics — a cookie-less Google Analytics alternative with built-in error tracking and performance monitoring. Self-host it or use our cloud. Get started for free!

added support for cities & regions

Blue Mouse89e9bdf39c10
diff --git a/apps/production/src/analytics/analytics.controller.ts b/apps/production/src/analytics/analytics.controller.ts
index aba6f3028..02ab94c8b 100644
--- a/apps/production/src/analytics/analytics.controller.ts
+++ b/apps/production/src/analytics/analytics.controller.ts
@@ -9,7 +9,7 @@ import * as dayjs from 'dayjs'
import * as utc from 'dayjs/plugin/utc'
import * as dayjsTimezone from 'dayjs/plugin/timezone'
import { hash } from 'blake3'
-import ct from 'countries-and-timezones'
+import timezones from 'countries-and-timezones'
import {
Controller,
Body,
@@ -92,6 +92,8 @@ const analyticsDTO = (
me: string,
ca: string,
cc: string,
+ rg: string,
+ ct: string,
sdur: number | string,
unique: number,
): Array<string | number> => {
@@ -109,6 +111,8 @@ const analyticsDTO = (
me,
ca,
cc,
+ rg,
+ ct,
sdur,
unique,
dayjs.utc().format('YYYY-MM-DD HH:mm:ss'),
@@ -121,6 +125,8 @@ const performanceDTO = (
dv: string,
br: string,
cc: string,
+ rg: string,
+ ct: string,
dns: number,
tls: number,
conn: number,
@@ -136,6 +142,8 @@ const performanceDTO = (
dv,
br,
cc,
+ rg,
+ ct,
_round(dns),
_round(tls),
_round(conn),
@@ -161,6 +169,8 @@ const customLogDTO = (
me: string,
ca: string,
cc: string,
+ rg: string,
+ ct: string,
): Array<string | number> => {
return [
pid,
@@ -175,6 +185,8 @@ const customLogDTO = (
me,
ca,
cc,
+ rg,
+ ct,
dayjs.utc().format('YYYY-MM-DD HH:mm:ss'),
]
}
@@ -835,13 +847,16 @@ export class AnalyticsController {
}
}
+ const { city = 'NULL', region = 'NULL' } =
+ this.analyticsService.getIPDetails(ip)
+
const ua = UAParser(userAgent)
const dv = ua.device.type || 'desktop'
const br = ua.browser.name
const os = ua.os.name
// trying to get country from timezome, otherwise using CloudFlare's IP based country code as a fallback
const cc =
- ct.getCountryForTimezone(eventsDTO.tz)?.id ||
+ timezones.getCountryForTimezone(eventsDTO.tz)?.id ||
(headers['cf-ipcountry'] === 'XX' ? 'NULL' : headers['cf-ipcountry'])
const dto = customLogDTO(
@@ -857,6 +872,8 @@ export class AnalyticsController {
eventsDTO.me,
eventsDTO.ca,
cc,
+ region,
+ city,
)
try {
@@ -930,9 +947,11 @@ export class AnalyticsController {
// trying to get country from timezome, otherwise using CloudFlare's IP based country code as a fallback
const cc =
- ct.getCountryForTimezone(logDTO.tz)?.id ||
+ timezones.getCountryForTimezone(logDTO.tz)?.id ||
(headers['cf-ipcountry'] === 'XX' ? 'NULL' : headers['cf-ipcountry'])
+ const { city = 'NULL', region = 'NULL' } =
+ this.analyticsService.getIPDetails(ip)
const ua = UAParser(userAgent)
const dv = ua.device.type || 'desktop'
const br = ua.browser.name
@@ -952,6 +971,8 @@ export class AnalyticsController {
logDTO.me,
logDTO.ca,
cc,
+ region,
+ city,
0,
Number(unique),
)
@@ -976,6 +997,8 @@ export class AnalyticsController {
dv,
br,
cc,
+ region,
+ city,
dns,
tls,
conn,
@@ -1036,6 +1059,8 @@ export class AnalyticsController {
await this.analyticsService.processInteractionSD(sessionHash, logDTO.pid)
+ const { city = 'NULL', region = 'NULL' } =
+ this.analyticsService.getIPDetails(ip)
const ua = UAParser(userAgent)
const dv = ua.device.type || 'desktop'
const br = ua.browser.name
@@ -1058,6 +1083,8 @@ export class AnalyticsController {
'NULL',
'NULL',
cc,
+ region,
+ city,
0,
Number(unique),
)
diff --git a/apps/production/src/analytics/analytics.service.ts b/apps/production/src/analytics/analytics.service.ts
index 69c11809e..067889d56 100644
--- a/apps/production/src/analytics/analytics.service.ts
+++ b/apps/production/src/analytics/analytics.service.ts
@@ -45,6 +45,7 @@ import {
import {
calculateRelativePercentage,
millisecondsToSeconds,
+ lookup,
} from '../common/utils'
import { PageviewsDTO } from './dto/pageviews.dto'
import { EventsDTO } from './dto/events.dto'
@@ -233,6 +234,12 @@ const isValidOrigin = (origins: string[], origin: string) => {
return false
}
+interface IPGeoDetails {
+ country?: string
+ region?: string
+ city?: string
+}
+
@Injectable()
export class AnalyticsService {
constructor(private readonly projectService: ProjectService) {}
@@ -813,6 +820,20 @@ export class AnalyticsService {
return result
}
+ getIPDetails(ip: string): IPGeoDetails {
+ const data = lookup.get(ip)
+
+ const country = data?.country?.names?.en
+ const city = data?.city?.names?.en
+ const region = data?.subdivisions?.[0]?.names?.en
+
+ return {
+ country,
+ city,
+ region,
+ }
+ }
+
async getFilters(pid: string, type: string): Promise<Array<string>> {
if (!_includes(TRAFFIC_COLUMNS, type)) {
throw new UnprocessableEntityException(
diff --git a/apps/production/src/common/utils.ts b/apps/production/src/common/utils.ts
index 71b79da75..e3a083d9e 100644
--- a/apps/production/src/common/utils.ts
+++ b/apps/production/src/common/utils.ts
@@ -1,3 +1,6 @@
+import { Reader, CityResponse } from 'maxmind'
+import * as path from 'path'
+import * as fs from 'fs'
import { HttpException } from '@nestjs/common'
import { hash } from 'blake3'
import * as randomstring from 'randomstring'
@@ -130,6 +133,38 @@ const millisecondsToSeconds = (milliseconds: number) => milliseconds / 1000
const generateRandomString = (length: number): string =>
randomstring.generate(length)
+const dummyLookup = () => ({
+ country: {
+ names: {
+ en: 'NULL',
+ },
+ },
+ city: {
+ names: {
+ en: 'NULL',
+ },
+ },
+ subdivisions: [
+ {
+ names: {
+ en: 'NULL',
+ },
+ },
+ ],
+})
+
+const GEOIP_DB_PATH = path.join(__dirname, '../../../..', 'dbip-city-lite.mmdb')
+
+// @ts-ignore
+// eslint-disable-next-line
+let lookup: Reader<CityResponse> = dummyLookup
+
+// TODO: CHECK IF GEO-IP DB EXISTS
+if (fs.existsSync(GEOIP_DB_PATH)) {
+ const buffer = fs.readFileSync(GEOIP_DB_PATH)
+ lookup = new Reader<CityResponse>(buffer)
+}
+
export {
getRandomTip,
checkRateLimit,
@@ -138,4 +173,5 @@ export {
millisecondsToSeconds,
generateRandomString,
nFormatter,
+ lookup,
}
diff --git a/meta/clickhouse/generate-dummy-data.js b/meta/clickhouse/generate-dummy-data.js
index c8afb6dc3..735c2d582 100644
--- a/meta/clickhouse/generate-dummy-data.js
+++ b/meta/clickhouse/generate-dummy-data.js
@@ -80,6 +80,8 @@ const generateAnalyticsData = async (pid, rowCount, from, to) => {
faker.helpers.arrayElement(UTM_MEDIUMS), // me
faker.helpers.arrayElement(UTM_CAMPAIGNS), // ca
faker.location.countryCode(), // cc
+ faker.location.state(), // rg
+ faker.location.city(), // ct
faker.number.int({ min: 0, max: 10000 }), // sdur
faker.number.int({ min: 0, max: 9 }) === 0 ? 1 : 0, // unique; 10% chance of unique record
_.split(faker.date.between({ from, to }).toISOString().replace('T', ' ').replace('Z', ''), '.')[0], // created; format the date string
@@ -136,6 +138,8 @@ const generateCustomEventsData = async (pid, rowCount, from, to) => {
faker.helpers.arrayElement(UTM_MEDIUMS), // me
faker.helpers.arrayElement(UTM_CAMPAIGNS), // ca
faker.location.countryCode(), // cc
+ faker.location.state(), // rg
+ faker.location.city(), // ct
_.split(faker.date.between({ from, to }).toISOString().replace('T', ' ').replace('Z', ''), '.')[0], // created; format the date string
]
records.push(record)
@@ -159,6 +163,8 @@ const generatePerformanceData = async (pid, rowCount, from, to) => {
faker.helpers.arrayElement(DEVICES), // dv
faker.helpers.arrayElement(BROWSERS), // br
faker.location.countryCode(), // cc
+ faker.location.state(), // rg
+ faker.location.city(), // ct
faker.number.int({ min: 0, max: 500 }), // dns
faker.number.int({ min: 0, max: 1000 }), // tls
faker.number.int({ min: 0, max: 100 }), // conn
diff --git a/migrations/clickhouse/2023_06_26.js b/migrations/clickhouse/2023_06_26.js
new file mode 100644
index 000000000..b401f4e18
--- /dev/null
+++ b/migrations/clickhouse/2023_06_26.js
@@ -0,0 +1,99 @@
+const { queriesRunner, dbName } = require('./setup')
+
+const queries = [
+ // Analytics table: added 'rg' and 'ct' columns
+ `DROP TABLE IF EXISTS ${dbName}.analytics_temp`,
+ `CREATE TABLE IF NOT EXISTS ${dbName}.analytics_temp
+ (
+ sid Nullable(String),
+ pid FixedString(12),
+ pg Nullable(String),
+ prev Nullable(String),
+ dv LowCardinality(Nullable(String)),
+ br LowCardinality(Nullable(String)),
+ os LowCardinality(Nullable(String)),
+ lc LowCardinality(Nullable(String)),
+ ref Nullable(String),
+ so Nullable(String),
+ me Nullable(String),
+ ca Nullable(String),
+ cc Nullable(FixedString(2)),
+ rg LowCardinality(Nullable(String)),
+ ct Nullable(String),
+ sdur Nullable(UInt32),
+ unique UInt8,
+ created DateTime('UTC')
+ )
+ ENGINE = MergeTree()
+ PARTITION BY toYYYYMM(created)
+ ORDER BY (pid, created);`,
+
+ `INSERT INTO ${dbName}.analytics_temp (sid, pid, pg, prev, dv, br, os, lc, ref, so, me, ca, cc, rg, ct, sdur, unique, created)
+ SELECT sid, pid, pg, prev, dv, br, os, lc, ref, so, me, ca, cc, NULL, NULL, sdur, unique, created FROM ${dbName}.analytics`,
+
+ `DROP TABLE ${dbName}.analytics`,
+ `RENAME TABLE ${dbName}.analytics_temp TO ${dbName}.analytics`,
+
+ // Custom events table: added 'rg' and 'ct' columns
+ `DROP TABLE IF EXISTS ${dbName}.customEV_temp`,
+ `CREATE TABLE IF NOT EXISTS ${dbName}.customEV
+ (
+ pid FixedString(12),
+ ev String,
+ pg Nullable(String),
+ dv LowCardinality(Nullable(String)),
+ br LowCardinality(Nullable(String)),
+ os LowCardinality(Nullable(String)),
+ lc LowCardinality(Nullable(String)),
+ ref Nullable(String),
+ so Nullable(String),
+ me Nullable(String),
+ ca Nullable(String),
+ cc Nullable(FixedString(2)),
+ rg LowCardinality(Nullable(String)),
+ ct Nullable(String),
+ created DateTime('UTC')
+ )
+ ENGINE = MergeTree()
+ PARTITION BY toYYYYMM(created)
+ ORDER BY (pid, created);`,
+
+ `INSERT INTO ${dbName}.customEV_temp (pid, ev, pg, dv, br, os, lc, ref, so, me, ca, cc, rg, ct, created)
+ SELECT pid, ev, pg, dv, br, os, lc, ref, so, me, ca, cc, NULL, NULL, created FROM ${dbName}.customEV`,
+
+ `DROP TABLE ${dbName}.customEV`,
+ `RENAME TABLE ${dbName}.customEV_temp TO ${dbName}.customEV`,
+
+ // Performance table: added 'rg' and 'ct' columns
+ `DROP TABLE IF EXISTS ${dbName}.performance_temp`,
+ `CREATE TABLE IF NOT EXISTS ${dbName}.performance
+ (
+ pid FixedString(12),
+ pg Nullable(String),
+ dv LowCardinality(Nullable(String)),
+ br LowCardinality(Nullable(String)),
+ cc Nullable(FixedString(2)),
+ rg LowCardinality(Nullable(String)),
+ ct Nullable(String),
+ dns Nullable(UInt32),
+ tls Nullable(UInt32),
+ conn Nullable(UInt32),
+ response Nullable(UInt32),
+ render Nullable(UInt32),
+ domLoad Nullable(UInt32),
+ pageLoad Nullable(UInt32),
+ ttfb Nullable(UInt32),
+ created DateTime('UTC')
+ )
+ ENGINE = MergeTree()
+ PARTITION BY toYYYYMM(created)
+ ORDER BY (pid, created);`,
+
+ `INSERT INTO ${dbName}.performance_temp (pid, pg, dv, br, cc, rg, ct, dns, tls, conn, response, render, domLoad, pageLoad, ttfb, created)
+ SELECT pid, pg, dv, br, cc, NULL, NULL, dns, tls, conn, response, render, domLoad, pageLoad, ttfb, created FROM ${dbName}.performance`,
+
+ `DROP TABLE ${dbName}.performance`,
+ `RENAME TABLE ${dbName}.performance_temp TO ${dbName}.performance`,
+]
+
+queriesRunner(queries)
diff --git a/migrations/clickhouse/initialise_database.js b/migrations/clickhouse/initialise_database.js
index 1d529f370..fde352fce 100644
--- a/migrations/clickhouse/initialise_database.js
+++ b/migrations/clickhouse/initialise_database.js
@@ -22,6 +22,8 @@ const CLICKHOUSE_INIT_QUERIES = [
me Nullable(String),
ca Nullable(String),
cc Nullable(FixedString(2)),
+ rg LowCardinality(Nullable(String)),
+ ct Nullable(String),
sdur Nullable(UInt32),
unique UInt8,
created DateTime('UTC')
@@ -45,6 +47,8 @@ const CLICKHOUSE_INIT_QUERIES = [
me Nullable(String),
ca Nullable(String),
cc Nullable(FixedString(2)),
+ rg LowCardinality(Nullable(String)),
+ ct Nullable(String),
created DateTime('UTC')
)
ENGINE = MergeTree()
@@ -59,6 +63,8 @@ const CLICKHOUSE_INIT_QUERIES = [
dv LowCardinality(Nullable(String)),
br LowCardinality(Nullable(String)),
cc Nullable(FixedString(2)),
+ rg LowCardinality(Nullable(String)),
+ ct Nullable(String),
dns Nullable(UInt32),
tls Nullable(UInt32),
conn Nullable(UInt32),