Skip to content

📈 Open source, privacy-first web analytics — a cookie-less Google Analytics alternative with built-in error tracking and performance monitoring. Self-host it or use our cloud. Get started for free!

Add support for commercial DB-IP database

Blue Mouse04898d40aca2
diff --git a/backend/Dockerfile b/backend/Dockerfile
index 8462ed110..f9f5df20c 100644
--- a/backend/Dockerfile
+++ b/backend/Dockerfile
@@ -5,7 +5,7 @@ COPY package*.json .
RUN npm install --force
COPY . .
RUN npm run deploy:community
-RUN node ./meta/dbip-sync.js
+RUN node ./meta/dbip-free-sync.js
#Stage 2: Runtime
FROM node:lts-alpine as run
@@ -50,7 +50,7 @@ WORKDIR /app
# Copy files (updated to remove user:group ownership since we're not creating them)
COPY --from=build /build/package*.json ./
COPY --from=build /build/dist/ ./dist/
-COPY --from=build /build/dbip-city-lite.mmdb ./dbip-city-lite.mmdb
+COPY --from=build /build/ip-geolocation-db.mmdb ./ip-geolocation-db.mmdb
COPY --from=build /build/migrations ./migrations
COPY --from=build /build/node_modules/ ./node_modules/
diff --git a/backend/apps/cloud/src/common/utils.ts b/backend/apps/cloud/src/common/utils.ts
index b3e5899b3..d9f1069c8 100644
--- a/backend/apps/cloud/src/common/utils.ts
+++ b/backend/apps/cloud/src/common/utils.ts
@@ -223,12 +223,12 @@ const dummyLookup = () => ({
const DEVELOPMENT_GEOIP_DB_PATH = path.join(
__dirname,
'../../../..',
- 'dbip-city-lite.mmdb',
+ 'ip-geolocation-db.mmdb',
)
const PRODUCTION_GEOIP_DB_PATH = path.join(
__dirname,
'../..',
- 'dbip-city-lite.mmdb',
+ 'ip-geolocation-db.mmdb',
)
let lookup: Reader<CityResponse> = {
@@ -254,6 +254,8 @@ export const getGeoDetails = (ip: string, tz?: string): IPGeoDetails => {
// Stage 1: Using IP address based geo lookup
const data = lookup.get(ip)
+ console.log('data:', JSON.stringify(data, null, 2))
+
const country = data?.country?.iso_code || null
// TODO: Add city overrides, for example, Colinton -> Edinburgh, etc.
const city = data?.city?.names?.en || null
diff --git a/backend/apps/community/src/common/utils.ts b/backend/apps/community/src/common/utils.ts
index fc64403d1..fcf1c8194 100644
--- a/backend/apps/community/src/common/utils.ts
+++ b/backend/apps/community/src/common/utils.ts
@@ -948,12 +948,12 @@ const dummyLookup = () => ({
const DEVELOPMENT_GEOIP_DB_PATH = path.join(
__dirname,
'../../../..',
- 'dbip-city-lite.mmdb',
+ 'ip-geolocation-db.mmdb',
)
const PRODUCTION_GEOIP_DB_PATH = path.join(
__dirname,
'../..',
- 'dbip-city-lite.mmdb',
+ 'ip-geolocation-db.mmdb',
)
let lookup: Reader<CityResponse> = {
diff --git a/backend/meta/dbip-commercial-sync.js b/backend/meta/dbip-commercial-sync.js
new file mode 100644
index 000000000..f6502e0ae
--- /dev/null
+++ b/backend/meta/dbip-commercial-sync.js
@@ -0,0 +1,164 @@
+const https = require('https')
+const fs = require('fs')
+const zlib = require('zlib')
+const path = require('path')
+
+const DESTINATION_FILE = 'ip-geolocation-db.mmdb'
+const DESTINATION_FOLDER = path.join(__dirname, '../')
+const DESTINATION_PATH = path.join(DESTINATION_FOLDER, DESTINATION_FILE)
+
+function parseApiKeyFromArgs() {
+ const args = process.argv.slice(2)
+ for (let i = 0; i < args.length; i += 1) {
+ const arg = args[i]
+ if (arg === '--api-key' || arg === '-k') {
+ return args[i + 1]
+ }
+ if (arg.startsWith('--api-key=')) {
+ return arg.split('=')[1]
+ }
+ }
+ return undefined
+}
+
+function ensureFolderExists(folderPath) {
+ if (!fs.existsSync(folderPath)) {
+ fs.mkdirSync(folderPath, { recursive: true })
+ }
+}
+
+function followableGet(url, options = {}, maxRedirects = 5) {
+ return new Promise((resolve, reject) => {
+ const request = https.get(url, options, response => {
+ const status = response.statusCode || 0
+ if ([301, 302, 303, 307, 308].includes(status)) {
+ if (maxRedirects <= 0) {
+ reject(new Error('Too many redirects'))
+ return
+ }
+ const location = response.headers.location
+ if (!location) {
+ reject(new Error('Redirect with no Location header'))
+ return
+ }
+ // Drain response before redirecting
+ response.resume()
+ followableGet(location, options, maxRedirects - 1)
+ .then(resolve)
+ .catch(reject)
+ return
+ }
+ resolve(response)
+ })
+ request.on('error', reject)
+ })
+}
+
+async function fetchDbipAccountInfo(apiKey) {
+ const metadataUrl = `https://db-ip.com/account/${apiKey}/db/ip-to-location/`
+ const response = await followableGet(metadataUrl, {
+ headers: { Accept: 'application/json' },
+ })
+
+ return new Promise((resolve, reject) => {
+ if ((response.statusCode || 0) !== 200) {
+ reject(
+ new Error(
+ `Failed to fetch metadata. HTTP ${response.statusCode}. Is the API key valid?`,
+ ),
+ )
+ return
+ }
+ const chunks = []
+ response.on('data', chunk => chunks.push(chunk))
+ response.on('end', () => {
+ try {
+ const text = Buffer.concat(chunks).toString('utf8')
+ const json = JSON.parse(text)
+ resolve(json)
+ } catch (e) {
+ reject(new Error('Failed to parse metadata JSON'))
+ }
+ })
+ response.on('error', reject)
+ })
+}
+
+function downloadMmdb(url, shouldGunzip, destinationPath) {
+ console.log(`Downloading ${url} ...`)
+ return new Promise((resolve, reject) => {
+ followableGet(url)
+ .then(response => {
+ if ((response.statusCode || 0) !== 200) {
+ reject(new Error(`Download failed. HTTP ${response.statusCode}`))
+ return
+ }
+
+ const tempPath = `${destinationPath}.download`
+ const fileStream = fs.createWriteStream(tempPath)
+
+ const onFinish = () => {
+ try {
+ if (fs.existsSync(destinationPath)) fs.unlinkSync(destinationPath)
+ fs.renameSync(tempPath, destinationPath)
+ console.log('[OK] Download finished!')
+ resolve()
+ } catch (e) {
+ if (fs.existsSync(tempPath)) fs.unlinkSync(tempPath)
+ reject(e)
+ }
+ }
+
+ const onError = error => {
+ if (fs.existsSync(tempPath)) fs.unlinkSync(tempPath)
+ reject(error)
+ }
+
+ if (shouldGunzip) {
+ const gunzip = zlib.createGunzip()
+ response.pipe(gunzip).pipe(fileStream)
+ fileStream.on('finish', onFinish)
+ fileStream.on('error', onError)
+ gunzip.on('error', onError)
+ } else {
+ response.pipe(fileStream)
+ fileStream.on('finish', onFinish)
+ fileStream.on('error', onError)
+ }
+ })
+ .catch(reject)
+ })
+}
+
+;(async () => {
+ try {
+ const apiKey = parseApiKeyFromArgs()
+ if (!apiKey) {
+ console.error(
+ 'Usage: node backend/meta/dbip-commercial-sync.js --api-key <YOUR_DBIP_API_KEY>',
+ )
+ process.exit(1)
+ }
+
+ ensureFolderExists(DESTINATION_FOLDER)
+
+ const metadata = await fetchDbipAccountInfo(apiKey)
+ if (!metadata || !metadata.mmdb || !metadata.mmdb.url) {
+ throw new Error('Metadata JSON does not contain mmdb.url')
+ }
+
+ const mmdbUrl = metadata.mmdb.url
+ const mmdbDate = metadata.mmdb.date || 'Unknown date'
+ const mmdbName = metadata.mmdb.name || ''
+
+ console.log(`[INFO] Downloading DB-IP mmdb for: ${mmdbDate}`)
+
+ const shouldGunzip = mmdbName.endsWith('.gz')
+ await downloadMmdb(mmdbUrl, shouldGunzip, DESTINATION_PATH)
+
+ console.log(`[OK] Saved to ${DESTINATION_PATH}`)
+ } catch (error) {
+ console.error('[ERROR] An error occurred:', error.message)
+ process.exitCode = 1
+ }
+})()
diff --git a/backend/meta/dbip-sync.js b/backend/meta/dbip-free-sync.js
similarity index 95%
rename from backend/meta/dbip-sync.js
rename to backend/meta/dbip-free-sync.js
index 7bc5cf0b6..4b846c790 100644
--- a/backend/meta/dbip-sync.js
+++ b/backend/meta/dbip-free-sync.js
@@ -1,4 +1,3 @@
-/* eslint-disable */
const https = require('https')
const fs = require('fs')
const zlib = require('zlib')
@@ -14,8 +13,8 @@ function getDownloadLink(date) {
// Update the DOWNLOAD_LINK to use the helper function
const DOWNLOAD_LINK = getDownloadLink(new Date())
-const DESTINATION_FILE = 'dbip-city-lite.mmdb'
-const ORIGINAL_FILE = 'dbip-city-lite.mmdb.gz'
+const DESTINATION_FILE = 'ip-geolocation-db.mmdb'
+const ORIGINAL_FILE = 'ip-geolocation-db.mmdb.gz'
const DESTINATION_FOLDER = path.join(__dirname, '../')
const GEOIP_DB_PATH = path.join(DESTINATION_FOLDER, ORIGINAL_FILE)
const UNZIPPED_PATH = path.join(DESTINATION_FOLDER, DESTINATION_FILE)