🐸 Rybbit - open-source and privacy-friendly alternative to Google Analytics that is 10x more intuitive.
ip geolocation
Bill Yanga8ac7900b7e1
diff --git a/server/src/actions/pageviewQueue.ts b/server/src/actions/pageviewQueue.tsnew file mode 100644index 000000000..cba99ca12--- /dev/null+++ b/server/src/actions/pageviewQueue.ts@@ -0,0 +1,92 @@+import { DateTime } from "luxon";+import clickhouse from "../db/clickhouse/clickhouse";+import { TrackingPayload } from "../types";+import { getDeviceType } from "../utils";++type TotalPayload = TrackingPayload & {+ userId: string;+ timestamp: string;+ sessionId: string;+ ua: UAParser.IResult;+ referrer: string;+ ipAddress: string;+};++class PageviewQueue {+ private queue: TotalPayload[] = [];+ private batchSize = 5000;+ private interval = 10000; // 10 seconds+ private processing = false;++ constructor() {+ // Start processing interval+ setInterval(() => this.processQueue(), this.interval);+ }++ async add(pageview: TotalPayload) {+ this.queue.push(pageview);+ }++ private async processQueue() {+ if (this.processing || this.queue.length === 0) return;+ this.processing = true;++ try {+ // Get batch of pageviews+ const batch = this.queue.splice(0, this.batchSize);+ const ips = [...new Set(batch.map((pv) => pv.ipAddress))];++ // Get geo data for all IPs in batch+ const geoResponse = await fetch("https://tracking.tomato.gg/geoip/bulk", {+ method: "POST",+ headers: {+ "Content-Type": "application/json",+ // "Accept-Encoding": "gzip",+ },+ body: JSON.stringify({ ips }),+ });++ console.info("ips: ", ips);++ const geoData: any = await geoResponse.json();+ console.info("geoData: ", geoData);++ // Process each pageview with its geo data+ const processedPageviews = batch.map((pv) => ({+ timestamp: DateTime.fromISO(pv.timestamp).toFormat(+ "yyyy-MM-dd HH:mm:ss"+ ),+ session_id: pv.sessionId,+ user_id: pv.userId,+ hostname: pv.hostname || "",+ pathname: pv.pathname || "",+ querystring: pv.querystring || "",+ page_title: pv.page_title || "",+ referrer: pv.referrer || "",+ browser: pv.ua.browser.name,+ operating_system: pv.ua.os.name,+ language: pv.language || "",+ screen_width: pv.screenWidth || 0,+ screen_height: pv.screenHeight || 0,+ device_type: getDeviceType(pv.screenWidth, pv.screenHeight),+ country: geoData[pv.ipAddress]?.data?.country || "",+ // countryIso: geoData[pv.ip]?.data?.countryIso || null,+ }));++ console.info("bulk insert: ", processedPageviews.length);+ // Bulk insert into database+ await clickhouse.insert({+ table: "pageviews",+ values: processedPageviews,+ format: "JSONEachRow",+ });+ } catch (error) {+ console.error("Error processing pageview queue:", error);+ } finally {+ this.processing = false;+ }+ }+}++// Create singleton instance+export const pageviewQueue = new PageviewQueue();diff --git a/server/src/actions/trackPageView.ts b/server/src/actions/trackPageView.tsindex 81abeb2a6..0cc2ae302 100644--- a/server/src/actions/trackPageView.ts+++ b/server/src/actions/trackPageView.ts@@ -7,6 +7,7 @@ import { DateTime } from "luxon";import { sql } from "../db/postgres/postgres";import UAParser, { UAParser as userAgentParser } from "ua-parser-js";import { Pageview } from "../db/clickhouse/types";+import { pageviewQueue } from "./pageviewQueue";type TotalPayload = TrackingPayload & {userId: string;@@ -14,6 +15,7 @@ type TotalPayload = TrackingPayload & {sessionId: string;ua: UAParser.IResult;referrer: string;+ ipAddress: string;};const insertPageview = async (pageview: TotalPayload) => {@@ -98,18 +100,19 @@ export async function trackPageView(request: FastifyRequest<{ Body: TrackingPayload }>) {const userAgent = request.headers["user-agent"] || "";- const ip_address = getIpAddress(request);- const userId = getUserId(ip_address, userAgent);+ const ipAddress = getIpAddress(request);+ const userId = getUserId(ipAddress, userAgent);const existingSession = await getExistingSession(userId);const payload = {...request.body,- ip_address: ip_address,+ ipAddress: ipAddress,timestamp: new Date().toISOString(),ua: userAgentParser(userAgent),userId: userId,sessionId: existingSession?.session_id || crypto.randomUUID(),};- insertPageview(payload);+ pageviewQueue.add(payload);+ // insertPageview(payload);updateSession(payload, existingSession);}diff --git a/server/src/db/clickhouse/clickhouse.ts b/server/src/db/clickhouse/clickhouse.tsindex c05fe1ff2..43a8e23ec 100644--- a/server/src/db/clickhouse/clickhouse.ts+++ b/server/src/db/clickhouse/clickhouse.ts@@ -22,6 +22,7 @@ export const initializeClickhouse = async () => {browser LowCardinality(String),operating_system LowCardinality(String),language LowCardinality(String),+ country LowCardinality(String),screen_width UInt16,screen_height UInt16,device_type LowCardinality(String)@@ -49,7 +50,7 @@ export const initializeClickhouse = async () => {ip_address String,browser String,language String,- country_code String+ country LowCardinality(String))ENGINE = MergeTree()PARTITION BY toYYYYMM(start_time)