Skip to content

🐸 Rybbit - open-source and privacy-friendly alternative to Google Analytics that is 10x more intuitive.

Update package.json and package-lock.json to include @maxmind/geoip2-node dependency; refactor pageviewQueue to utilize local geolocation service for IP address tracking

Bill Yang03152db28454
diff --git a/server/package-lock.json b/server/package-lock.json
index e4f5e2205..a767451dd 100644
--- a/server/package-lock.json
+++ b/server/package-lock.json
@@ -13,6 +13,7 @@
"@fastify/cors": "^10.0.2",
"@fastify/one-line-logger": "^1.4.0",
"@fastify/static": "^8.0.4",
+ "@maxmind/geoip2-node": "^6.0.0",
"better-auth": "^1.2.7",
"dotenv": "^16.4.7",
"drizzle-orm": "^0.40.0",
@@ -1127,6 +1128,14 @@
"node": ">=8"
}
},
+ "node_modules/@maxmind/geoip2-node": {
+ "version": "6.0.0",
+ "resolved": "https://registry.npmjs.org/@maxmind/geoip2-node/-/geoip2-node-6.0.0.tgz",
+ "integrity": "sha512-b1zcdxRm13HDgNfHnRpET7E3r1NlSK+VpQhN9dKn8aUUH71Ug6rNmgASYh7WrHDZLDqSXMHryfs5Aw0ufMtBYw==",
+ "dependencies": {
+ "maxmind": "^4.2.0"
+ }
+ },
"node_modules/@noble/ciphers": {
"version": "0.6.0",
"resolved": "https://registry.npmjs.org/@noble/ciphers/-/ciphers-0.6.0.tgz",
@@ -2718,6 +2727,19 @@
"node": ">= 0.4"
}
},
+ "node_modules/maxmind": {
+ "version": "4.3.24",
+ "resolved": "https://registry.npmjs.org/maxmind/-/maxmind-4.3.24.tgz",
+ "integrity": "sha512-dexrLcjfS2xDGOvdV8XcfQYmyQVpGidMwEG2ld19lXlsB+i+lXRWPzQi81HfwRXR4hxzFr5gT0oAIFyqAAb/Ww==",
+ "dependencies": {
+ "mmdb-lib": "2.1.1",
+ "tiny-lru": "11.2.11"
+ },
+ "engines": {
+ "node": ">=12",
+ "npm": ">=6"
+ }
+ },
"node_modules/mime": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/mime/-/mime-3.0.0.tgz",
@@ -2771,6 +2793,15 @@
"node": ">=10"
}
},
+ "node_modules/mmdb-lib": {
+ "version": "2.1.1",
+ "resolved": "https://registry.npmjs.org/mmdb-lib/-/mmdb-lib-2.1.1.tgz",
+ "integrity": "sha512-yx8H/1H5AfnufiLnzzPqPf4yr/dKU9IFT1rPVwSkrKWHsQEeVVd6+X+L0nUbXhlEFTu3y/7hu38CFmEVgzvyeg==",
+ "engines": {
+ "node": ">=10",
+ "npm": ">=6"
+ }
+ },
"node_modules/mnemonist": {
"version": "0.39.8",
"resolved": "https://registry.npmjs.org/mnemonist/-/mnemonist-0.39.8.tgz",
@@ -3731,6 +3762,14 @@
"real-require": "^0.2.0"
}
},
+ "node_modules/tiny-lru": {
+ "version": "11.2.11",
+ "resolved": "https://registry.npmjs.org/tiny-lru/-/tiny-lru-11.2.11.tgz",
+ "integrity": "sha512-27BIW0dIWTYYoWNnqSmoNMKe5WIbkXsc0xaCQHd3/3xT2XMuMJrzHdrO9QBFR14emBz1Bu0dOAs2sCBBrvgPQA==",
+ "engines": {
+ "node": ">=12"
+ }
+ },
"node_modules/to-regex-range": {
"version": "5.0.1",
"resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz",
diff --git a/server/package.json b/server/package.json
index 1aecb8ab8..18340ddfb 100644
--- a/server/package.json
+++ b/server/package.json
@@ -21,6 +21,7 @@
"@fastify/cors": "^10.0.2",
"@fastify/one-line-logger": "^1.4.0",
"@fastify/static": "^8.0.4",
+ "@maxmind/geoip2-node": "^6.0.0",
"better-auth": "^1.2.7",
"dotenv": "^16.4.7",
"drizzle-orm": "^0.40.0",
diff --git a/server/src/db/geolocation/GeoLite2-City.mmdb b/server/src/db/geolocation/GeoLite2-City.mmdb
new file mode 100644
index 000000000..4f8d9fee0
Binary files /dev/null and b/server/src/db/geolocation/GeoLite2-City.mmdb differ
diff --git a/server/src/db/geolocation/geolocation.ts b/server/src/db/geolocation/geolocation.ts
new file mode 100644
index 000000000..45aa3fdcf
--- /dev/null
+++ b/server/src/db/geolocation/geolocation.ts
@@ -0,0 +1,59 @@
+import type { City, SubdivisionsRecord } from "@maxmind/geoip2-node";
+import { Reader } from "@maxmind/geoip2-node";
+import { readFile } from "fs/promises";
+import path from "path";
+
+interface LocationResponse {
+ city?: string;
+ country?: string;
+ countryIso?: string;
+ latitude?: number;
+ longitude?: number;
+ timezone?: string;
+ error?: string;
+ subdivisions?: SubdivisionsRecord[];
+}
+
+let reader: Reader;
+
+// Extend the Reader type to include the city method
+interface ExtendedReader extends Reader {
+ city(ip: string): City;
+}
+
+async function loadDatabase(dbPath: string): Promise<void> {
+ try {
+ const dbBuffer = await readFile(dbPath);
+ reader = Reader.openBuffer(dbBuffer);
+ console.log("GeoIP database loaded successfully");
+ } catch (error) {
+ console.error("Failed to load GeoIP database:", error);
+ throw error;
+ }
+}
+
+await loadDatabase(path.join(process.cwd(), "GeoLite2-City.mmdb"));
+
+// Utility function to extract response data
+function extractLocationData(response: City): LocationResponse {
+ console.info(JSON.stringify(response, null, 2));
+ return {
+ city: response.city?.names?.en,
+ country: response.country?.names?.en,
+ countryIso: response.country?.isoCode,
+ latitude: response.location?.latitude,
+ longitude: response.location?.longitude,
+ timezone: response.location?.timeZone,
+ subdivisions: response.subdivisions,
+ };
+}
+
+export async function getLocation(ip: string): Promise<LocationResponse> {
+ try {
+ const response = await (reader as ExtendedReader).city(ip);
+ return extractLocationData(response);
+ } catch (error) {
+ console.error("Error fetching location:", error);
+ return { error: "Failed to fetch location" };
+ }
+}
diff --git a/server/src/tracker/pageviewQueue.ts b/server/src/tracker/pageviewQueue.ts
index fecf0e4af..f91d8824d 100644
--- a/server/src/tracker/pageviewQueue.ts
+++ b/server/src/tracker/pageviewQueue.ts
@@ -4,6 +4,7 @@ import { TrackingPayload } from "../types.js";
import { getDeviceType } from "../utils.js";
import { getChannel } from "./getChannel.js";
import { clearSelfReferrer, getUTMParams } from "./trackingUtils.js";
+import { getLocation } from "../db/geolocation/geolocation.js";
type TotalPayload = TrackingPayload & {
userId: string;
@@ -40,20 +41,21 @@ class PageviewQueue {
const batch = this.queue.splice(0, this.batchSize);
const ips = [...new Set(batch.map((pv) => pv.ipAddress))];
- let geoData: any;
+ let geoData: Record<string, { data: any }> = {};
try {
- // Get geo data for all IPs in batch
- const geoResponse = await fetch("https://tracking.tomato.gg/geoip/bulk", {
- method: "POST",
-
- headers: {
- "Content-Type": "application/json",
- // "Accept-Encoding": "gzip",
- },
- body: JSON.stringify({ ips }),
+ // Process each IP to get geo data using local implementation
+ const geoPromises = ips.map(async (ip) => {
+ const locationData = await getLocation(ip);
+ return { ip, locationData };
+ });
+
+ const results = await Promise.all(geoPromises);
+
+ // Format results to match expected structure
+ results.forEach(({ ip, locationData }) => {
+ geoData[ip] = { data: locationData };
});
- geoData = await geoResponse.json();
} catch (error) {
console.error("Error getting geo data:", error);
}