Skip to content

📈 Open source, privacy-first web analytics — a cookie-less Google Analytics alternative with built-in error tracking and performance monitoring. Self-host it or use our cloud. Get started for free!

Replace old Clickhouse 'NULL' values with null

Blue Moused46cc2ba788f
diff --git a/apps/production/src/analytics/analytics.controller.ts b/apps/production/src/analytics/analytics.controller.ts
index 487cef0b4..4b1fe4a1a 100644
--- a/apps/production/src/analytics/analytics.controller.ts
+++ b/apps/production/src/analytics/analytics.controller.ts
@@ -1133,11 +1133,7 @@ export class AnalyticsController {
await this.analyticsService.validate(errorDTO, origin, 'error', ip)
- const {
- city = 'NULL',
- region = 'NULL',
- country = 'NULL',
- } = getGeoDetails(ip, errorDTO.tz)
+ const { city, region, country } = getGeoDetails(ip, errorDTO.tz)
const ua = UAParser(userAgent)
const dv = ua.device.type || 'desktop'
@@ -1242,11 +1238,7 @@ export class AnalyticsController {
}
}
- const {
- city = 'NULL',
- region = 'NULL',
- country = 'NULL',
- } = getGeoDetails(ip, eventsDTO.tz)
+ const { city, region, country } = getGeoDetails(ip, eventsDTO.tz)
const ua = UAParser(userAgent)
const dv = ua.device.type || 'desktop'
@@ -1470,11 +1462,7 @@ export class AnalyticsController {
await this.analyticsService.processInteractionSD(sessionHash, logDTO.pid)
- const {
- city = 'NULL',
- region = 'NULL',
- country = 'NULL',
- } = getGeoDetails(ip)
+ const { city, region, country } = getGeoDetails(ip)
const ua = UAParser(userAgent)
const dv = ua.device.type || 'desktop'
const br = ua.browser.name
diff --git a/apps/production/src/analytics/analytics.service.ts b/apps/production/src/analytics/analytics.service.ts
index 06aae447e..8b8afe180 100644
--- a/apps/production/src/analytics/analytics.service.ts
+++ b/apps/production/src/analytics/analytics.service.ts
@@ -496,7 +496,7 @@ export class AnalyticsService {
logDTO.lc = _join(lcParted, '-')
} else {
- logDTO.lc = 'NULL'
+ logDTO.lc = null
}
}
diff --git a/apps/production/src/common/utils.ts b/apps/production/src/common/utils.ts
index 990cb65af..77bf81a95 100644
--- a/apps/production/src/common/utils.ts
+++ b/apps/production/src/common/utils.ts
@@ -185,19 +185,19 @@ if (fs.existsSync(PRODUCTION_GEOIP_DB_PATH)) {
}
interface IPGeoDetails {
- country?: string
- region?: string
- city?: string
+ country: string | null
+ region: string | null
+ city: string | null
}
const getGeoDetails = (ip: string, tz?: string): IPGeoDetails => {
// Stage 1: Using IP address based geo lookup
const data = lookup.get(ip)
- const country = data?.country?.iso_code
+ const country = data?.country?.iso_code || null
// TODO: Add city overrides, for example, Colinton -> Edinburgh, etc.
- const city = data?.city?.names?.en
- const region = data?.subdivisions?.[0]?.names?.en
+ const city = data?.city?.names?.en || null
+ const region = data?.subdivisions?.[0]?.names?.en || null
if (country) {
return {
diff --git a/apps/selfhosted/src/analytics/analytics.controller.ts b/apps/selfhosted/src/analytics/analytics.controller.ts
index 77069b624..cbb7f2dac 100644
--- a/apps/selfhosted/src/analytics/analytics.controller.ts
+++ b/apps/selfhosted/src/analytics/analytics.controller.ts
@@ -960,11 +960,7 @@ export class AnalyticsController {
const br = ua.browser.name
const os = ua.os.name
- const {
- city = 'NULL',
- region = 'NULL',
- country = 'NULL',
- } = getGeoDetails(ip, eventsDTO.tz, headers)
+ const { city, region, country } = getGeoDetails(ip, eventsDTO.tz, headers)
const sessionHash = getSessionKey(ip, userAgent, eventsDTO.pid, salt)
const [, psid] = await this.analyticsService.isUnique(sessionHash)
@@ -1061,11 +1057,7 @@ export class AnalyticsController {
)
}
- const {
- city = 'NULL',
- region = 'NULL',
- country = 'NULL',
- } = getGeoDetails(ip, logDTO.tz, headers)
+ const { city, region, country } = getGeoDetails(ip, logDTO.tz, headers)
const ua = UAParser(userAgent)
const dv = ua.device.type || 'desktop'
@@ -1192,11 +1184,7 @@ export class AnalyticsController {
const br = ua.browser.name
const os = ua.os.name
- const {
- city = 'NULL',
- region = 'NULL',
- country = 'NULL',
- } = getGeoDetails(ip, null, headers)
+ const { city, region, country } = getGeoDetails(ip, null, headers)
const transformed = trafficTransformer(
psid,
@@ -1495,11 +1483,7 @@ export class AnalyticsController {
await this.analyticsService.validate(errorDTO, origin, 'error', ip)
- const {
- city = 'NULL',
- region = 'NULL',
- country = 'NULL',
- } = getGeoDetails(ip, errorDTO.tz)
+ const { city, region, country } = getGeoDetails(ip, errorDTO.tz)
const ua = UAParser(userAgent)
const dv = ua.device.type || 'desktop'
diff --git a/apps/selfhosted/src/analytics/analytics.service.ts b/apps/selfhosted/src/analytics/analytics.service.ts
index 5272f7e54..2c69e2d71 100644
--- a/apps/selfhosted/src/analytics/analytics.service.ts
+++ b/apps/selfhosted/src/analytics/analytics.service.ts
@@ -409,6 +409,8 @@ export class AnalyticsService {
type: 'custom' | 'log' | 'error' = 'log',
ip?: string,
): Promise<string | null> {
+ console.log('validate!')
+ console.log('req origin:', origin)
if (_isEmpty(logDTO)) {
throw new BadRequestException('The request cannot be empty')
}
@@ -445,7 +447,7 @@ export class AnalyticsService {
logDTO.lc = _join(lcParted, '-')
} else {
- logDTO.lc = 'NULL'
+ logDTO.lc = null
}
}
diff --git a/apps/selfhosted/src/common/utils.ts b/apps/selfhosted/src/common/utils.ts
index 7a4c2fbbc..0780bf6d2 100644
--- a/apps/selfhosted/src/common/utils.ts
+++ b/apps/selfhosted/src/common/utils.ts
@@ -670,9 +670,9 @@ const getSelfhostedUUID = (): string => {
}
interface IPGeoDetails {
- country?: string
- region?: string
- city?: string
+ country: string | null
+ region: string | null
+ city: string | null
}
const getGeoDetails = (
@@ -680,27 +680,12 @@ const getGeoDetails = (
tz?: string,
headers?: unknown,
): IPGeoDetails => {
- // TODO: Add support for DBIP for self-hosted
-
// Stage 1: Using IP address based geo lookup
- // const data = lookup.get(ip)
-
- // const country = data?.country?.iso_code
- // // TODO: Add city overrides, for example, Colinton -> Edinburgh, etc.
- // const city = data?.city?.names?.en
- // const region = data?.subdivisions?.[0]?.names?.en
-
- // if (country) {
- // return {
- // country,
- // city,
- // region,
- // }
- // }
+ // TODO: Add support for DBIP for self-hosted
// Stage 2: If Cloudflare is enabled, use their headers
if (isProxiedByCloudflare && headers?.['cf-ipcountry'] !== 'XX') {
- return headers['cf-ipcountry']
+ return headers['cf-ipcountry'] || null
}
// Stage 3: Using timezone based geo lookup as a fallback
diff --git a/meta/clickhouse/generate-dummy-data.js b/meta/clickhouse/generate-dummy-data.js
index 63450ccaa..02d9c416f 100644
--- a/meta/clickhouse/generate-dummy-data.js
+++ b/meta/clickhouse/generate-dummy-data.js
@@ -164,10 +164,10 @@ const generateAnalyticsData = async (pid, rowCount, from, to) => {
for (let i = 0; i < rowCount; ++i) {
const record = [
0, // psid
- 'NULL', // sid
+ null, // sid
pid,
faker.helpers.arrayElement(PAGES), // pg
- 'NULL', // prev
+ null, // prev
faker.helpers.arrayElement(DEVICES), // dv
faker.helpers.arrayElement(BROWSERS), // br
faker.helpers.arrayElement(OS), // os