Skip to content

🐸 Rybbit - open-source and privacy-friendly alternative to Google Analytics that is 10x more intuitive.

ip geolocation

Bill Yanga8ac7900b7e1
diff --git a/server/src/actions/pageviewQueue.ts b/server/src/actions/pageviewQueue.ts
new file mode 100644
index 000000000..cba99ca12
--- /dev/null
+++ b/server/src/actions/pageviewQueue.ts
@@ -0,0 +1,92 @@
+import { DateTime } from "luxon";
+import clickhouse from "../db/clickhouse/clickhouse";
+import { TrackingPayload } from "../types";
+import { getDeviceType } from "../utils";
+
+type TotalPayload = TrackingPayload & {
+ userId: string;
+ timestamp: string;
+ sessionId: string;
+ ua: UAParser.IResult;
+ referrer: string;
+ ipAddress: string;
+};
+
+class PageviewQueue {
+ private queue: TotalPayload[] = [];
+ private batchSize = 5000;
+ private interval = 10000; // 10 seconds
+ private processing = false;
+
+ constructor() {
+ // Start processing interval
+ setInterval(() => this.processQueue(), this.interval);
+ }
+
+ async add(pageview: TotalPayload) {
+ this.queue.push(pageview);
+ }
+
+ private async processQueue() {
+ if (this.processing || this.queue.length === 0) return;
+ this.processing = true;
+
+ try {
+ // Get batch of pageviews
+ const batch = this.queue.splice(0, this.batchSize);
+ const ips = [...new Set(batch.map((pv) => pv.ipAddress))];
+
+ // Get geo data for all IPs in batch
+ const geoResponse = await fetch("https://tracking.tomato.gg/geoip/bulk", {
+ method: "POST",
+ headers: {
+ "Content-Type": "application/json",
+ // "Accept-Encoding": "gzip",
+ },
+ body: JSON.stringify({ ips }),
+ });
+
+ console.info("ips: ", ips);
+
+ const geoData: any = await geoResponse.json();
+ console.info("geoData: ", geoData);
+
+ // Process each pageview with its geo data
+ const processedPageviews = batch.map((pv) => ({
+ timestamp: DateTime.fromISO(pv.timestamp).toFormat(
+ "yyyy-MM-dd HH:mm:ss"
+ ),
+ session_id: pv.sessionId,
+ user_id: pv.userId,
+ hostname: pv.hostname || "",
+ pathname: pv.pathname || "",
+ querystring: pv.querystring || "",
+ page_title: pv.page_title || "",
+ referrer: pv.referrer || "",
+ browser: pv.ua.browser.name,
+ operating_system: pv.ua.os.name,
+ language: pv.language || "",
+ screen_width: pv.screenWidth || 0,
+ screen_height: pv.screenHeight || 0,
+ device_type: getDeviceType(pv.screenWidth, pv.screenHeight),
+ country: geoData[pv.ipAddress]?.data?.country || "",
+ // countryIso: geoData[pv.ip]?.data?.countryIso || null,
+ }));
+
+ console.info("bulk insert: ", processedPageviews.length);
+ // Bulk insert into database
+ await clickhouse.insert({
+ table: "pageviews",
+ values: processedPageviews,
+ format: "JSONEachRow",
+ });
+ } catch (error) {
+ console.error("Error processing pageview queue:", error);
+ } finally {
+ this.processing = false;
+ }
+ }
+}
+
+// Create singleton instance
+export const pageviewQueue = new PageviewQueue();
diff --git a/server/src/actions/trackPageView.ts b/server/src/actions/trackPageView.ts
index 81abeb2a6..0cc2ae302 100644
--- a/server/src/actions/trackPageView.ts
+++ b/server/src/actions/trackPageView.ts
@@ -7,6 +7,7 @@ import { DateTime } from "luxon";
import { sql } from "../db/postgres/postgres";
import UAParser, { UAParser as userAgentParser } from "ua-parser-js";
import { Pageview } from "../db/clickhouse/types";
+import { pageviewQueue } from "./pageviewQueue";
type TotalPayload = TrackingPayload & {
userId: string;
@@ -14,6 +15,7 @@ type TotalPayload = TrackingPayload & {
sessionId: string;
ua: UAParser.IResult;
referrer: string;
+ ipAddress: string;
};
const insertPageview = async (pageview: TotalPayload) => {
@@ -98,18 +100,19 @@ export async function trackPageView(
request: FastifyRequest<{ Body: TrackingPayload }>
) {
const userAgent = request.headers["user-agent"] || "";
- const ip_address = getIpAddress(request);
- const userId = getUserId(ip_address, userAgent);
+ const ipAddress = getIpAddress(request);
+ const userId = getUserId(ipAddress, userAgent);
const existingSession = await getExistingSession(userId);
const payload = {
...request.body,
- ip_address: ip_address,
+ ipAddress: ipAddress,
timestamp: new Date().toISOString(),
ua: userAgentParser(userAgent),
userId: userId,
sessionId: existingSession?.session_id || crypto.randomUUID(),
};
- insertPageview(payload);
+ pageviewQueue.add(payload);
+ // insertPageview(payload);
updateSession(payload, existingSession);
}
diff --git a/server/src/db/clickhouse/clickhouse.ts b/server/src/db/clickhouse/clickhouse.ts
index c05fe1ff2..43a8e23ec 100644
--- a/server/src/db/clickhouse/clickhouse.ts
+++ b/server/src/db/clickhouse/clickhouse.ts
@@ -22,6 +22,7 @@ export const initializeClickhouse = async () => {
browser LowCardinality(String),
operating_system LowCardinality(String),
language LowCardinality(String),
+ country LowCardinality(String),
screen_width UInt16,
screen_height UInt16,
device_type LowCardinality(String)
@@ -49,7 +50,7 @@ export const initializeClickhouse = async () => {
ip_address String,
browser String,
language String,
- country_code String
+ country LowCardinality(String)
)
ENGINE = MergeTree()
PARTITION BY toYYYYMM(start_time)