Skip to content

🐸 Rybbit - open-source and privacy-friendly alternative to Google Analytics that is 10x more intuitive.

import session logic

Bill Yang0d798d655675
diff --git a/server/package-lock.json b/server/package-lock.json
index e1243e289..6bb473418 100644
--- a/server/package-lock.json
+++ b/server/package-lock.json
@@ -14,12 +14,14 @@
"@fastify/static": "^8.0.4",
"fastify": "^5.1.0",
"luxon": "^3.5.0",
+ "node-cron": "^3.0.3",
"postgres": "^3.4.5",
"ua-parser-js": "^2.0.0"
},
"devDependencies": {
"@types/luxon": "^3.4.2",
"@types/node": "^20.10.0",
+ "@types/node-cron": "^3.0.11",
"ts-node-dev": "^2.0.0",
"typescript": "^5.7.3"
}
@@ -302,6 +304,12 @@
"undici-types": "~5.26.4"
}
},
+ "node_modules/@types/node-cron": {
+ "version": "3.0.11",
+ "resolved": "https://registry.npmjs.org/@types/node-cron/-/node-cron-3.0.11.tgz",
+ "integrity": "sha512-0ikrnug3/IyneSHqCBeslAhlK2aBfYek1fGo4bP4QnZPmiqSGRK+Oy7ZMisLWkesffJvQ1cqAcBnJC+8+nxIAg==",
+ "dev": true
+ },
"node_modules/@types/strip-bom": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/@types/strip-bom/-/strip-bom-3.0.0.tgz",
@@ -1251,6 +1259,17 @@
"obliterator": "^2.0.1"
}
},
+ "node_modules/node-cron": {
+ "version": "3.0.3",
+ "resolved": "https://registry.npmjs.org/node-cron/-/node-cron-3.0.3.tgz",
+ "integrity": "sha512-dOal67//nohNgYWb+nWmg5dkFdIwDm8EpeGYMekPMrngV3637lqnX0lbUcCtgibHTz6SEz7DAIjKvKDFYCnO1A==",
+ "dependencies": {
+ "uuid": "8.3.2"
+ },
+ "engines": {
+ "node": ">=6.0.0"
+ }
+ },
"node_modules/normalize-path": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz",
@@ -2082,6 +2101,14 @@
"integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==",
"dev": true
},
+ "node_modules/uuid": {
+ "version": "8.3.2",
+ "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz",
+ "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==",
+ "bin": {
+ "uuid": "dist/bin/uuid"
+ }
+ },
"node_modules/v8-compile-cache-lib": {
"version": "3.0.1",
"resolved": "https://registry.npmjs.org/v8-compile-cache-lib/-/v8-compile-cache-lib-3.0.1.tgz",
diff --git a/server/package.json b/server/package.json
index 56bd5e537..d8f68eac9 100644
--- a/server/package.json
+++ b/server/package.json
@@ -15,12 +15,14 @@
"@fastify/static": "^8.0.4",
"fastify": "^5.1.0",
"luxon": "^3.5.0",
+ "node-cron": "^3.0.3",
"postgres": "^3.4.5",
"ua-parser-js": "^2.0.0"
},
"devDependencies": {
"@types/luxon": "^3.4.2",
"@types/node": "^20.10.0",
+ "@types/node-cron": "^3.0.11",
"ts-node-dev": "^2.0.0",
"typescript": "^5.7.3"
}
diff --git a/server/public/analytics.js b/server/public/analytics.js
index a1c5fc4a7..730ce43e5 100644
--- a/server/public/analytics.js
+++ b/server/public/analytics.js
@@ -4,19 +4,32 @@
const scriptTag = document.currentScript;
const ANALYTICS_URL = scriptTag.getAttribute("src").split("/analytics.js")[0];
- // Track pageview
+ // Get debounce duration from data attribute or default to 300ms
+ const debounceDuration = scriptTag.getAttribute("data-debounce")
+ ? Math.max(0, parseInt(scriptTag.getAttribute("data-debounce")))
+ : 500;
+
+ // Debounce function
+ function debounce(func, wait) {
+ let timeout;
+ return (...args) => {
+ clearTimeout(timeout);
+ timeout = setTimeout(() => func.apply(this, args), wait);
+ };
+ }
+
+ // Track pageview (core functionality)
const trackPageview = () => {
const url = new URL(window.location.href);
const payload = {
hostname: url.hostname,
pathname: url.pathname,
querystring: url.search,
- referrer: document.referrer,
- timestamp: new Date().toISOString(),
- userAgent: navigator.userAgent,
screenWidth: window.innerWidth,
screenHeight: window.innerHeight,
language: navigator.language,
+ page_title: document.title,
+ referrer: document.referrer,
};
fetch(`${ANALYTICS_URL}/track/pageview`, {
@@ -30,49 +43,38 @@
}).catch(console.error);
};
- // Track custom events
- const trackEvent = (eventName, eventData = {}) => {
- const payload = {
- eventName,
- eventData,
- url: window.location.href,
- timestamp: new Date().toISOString(),
- };
+ // Create debounced version with configured duration
+ const debouncedTrackPageview =
+ debounceDuration > 0
+ ? debounce(trackPageview, debounceDuration)
+ : trackPageview;
- fetch(`${ANALYTICS_URL}/track/event`, {
- method: "POST",
- headers: {
- "Content-Type": "application/json",
- },
- body: JSON.stringify(payload),
- mode: "cors",
- keepalive: true,
- }).catch(console.error);
+ // History API override
+ const originalPushState = history.pushState;
+ const originalReplaceState = history.replaceState;
+
+ history.pushState = function (...args) {
+ console.info("pushState");
+ originalPushState.apply(this, args);
+ debouncedTrackPageview();
};
- // Track page duration
- let pageLoadTime = Date.now();
- window.addEventListener("beforeunload", () => {
- const duration = Math.floor((Date.now() - pageLoadTime) / 1000);
- const payload = {
- url: window.location.href,
- timestamp: new Date().toISOString(),
- duration,
- };
+ history.replaceState = function (...args) {
+ console.info("replaceState");
+ originalReplaceState.apply(this, args);
+ debouncedTrackPageview();
+ };
- // Using sendBeacon for more reliable data sending on page unload
- // navigator.sendBeacon(
- // `${ANALYTICS_URL}/track/pageview`,
- // JSON.stringify(payload)
- // );
- });
+ // Handle navigation events
+ window.addEventListener("popstate", debouncedTrackPageview);
- // Expose the tracking functions globally
+ // Expose API
window.frogstats = {
- trackEvent,
- trackPageview,
+ trackPageview: debouncedTrackPageview,
+ // Add direct access to non-debounced version if needed
+ _trackImmediately: trackPageview,
};
- // Track initial pageview
+ // Initial pageview
trackPageview();
})();
diff --git a/server/src/actions/trackPageView.ts b/server/src/actions/trackPageView.ts
index e27d63a58..7934fb79b 100644
--- a/server/src/actions/trackPageView.ts
+++ b/server/src/actions/trackPageView.ts
@@ -1,20 +1,22 @@
import { FastifyRequest } from "fastify";
import { TrackingPayload } from "../types";
-import { generateUserId, getDeviceType, getIpAddress } from "../utils";
+import { getUserId, getDeviceType, getIpAddress } from "../utils";
import crypto from "crypto";
-import clickhouse from "../db/clickhouse";
+import clickhouse from "../db/clickhouse/clickhouse";
import { DateTime } from "luxon";
-import { sql } from "../db/postgres";
+import { sql } from "../db/postgres/postgres";
import UAParser, { UAParser as userAgentParser } from "ua-parser-js";
+import { Pageview } from "../db/clickhouse/types";
-const insertPageview = async (
- pageview: TrackingPayload & {
- userId: string;
- timestamp: string;
- sessionId: string;
- ua: UAParser.IResult;
- }
-) => {
+type TotalPayload = TrackingPayload & {
+ userId: string;
+ timestamp: string;
+ sessionId: string;
+ ua: UAParser.IResult;
+ referrer: string;
+};
+
+const insertPageview = async (pageview: TotalPayload) => {
try {
const formattedTimestamp = DateTime.fromISO(pageview.timestamp).toFormat(
"yyyy-MM-dd HH:mm:ss"
@@ -29,8 +31,8 @@ const insertPageview = async (
hostname: pageview.hostname || "",
pathname: pageview.pathname || "",
querystring: pageview.querystring || "",
+ page_title: pageview.page_title || "",
referrer: pageview.referrer || "",
- user_agent: pageview.userAgent || "",
browser: pageview.ua.browser.name,
operating_system: pageview.ua.os.name,
language: pageview.language || "",
@@ -51,17 +53,17 @@ const insertPageview = async (
}
};
+const getExistingSession = async (userId: string): Promise<Pageview | null> => {
+ const [existingSession] = await sql<Pageview[]>`
+ SELECT * FROM active_sessions WHERE user_id = ${userId}
+ `;
+ return existingSession;
+};
+
const updateSession = async (
- pageview: TrackingPayload & {
- userId: string;
- timestamp: string;
- sessionId: string;
- ua: UAParser.IResult;
- }
+ pageview: TotalPayload,
+ existingSession: Pageview | null
) => {
- const [existingSession] = await sql`
- SELECT * FROM active_sessions WHERE user_id = ${pageview.userId}
- `;
if (existingSession) {
await sql`
UPDATE active_sessions SET last_activity = ${pageview.timestamp}, pageviews = pageviews + 1 WHERE user_id = ${pageview.userId}
@@ -92,27 +94,24 @@ const updateSession = async (
`;
};
-export function trackPageView(
+export async function trackPageView(
request: FastifyRequest<{ Body: TrackingPayload }>
) {
+ const userAgent = request.headers["user-agent"] || "";
+ const referrer = request.headers["referer"] || "";
+ const ip_address = getIpAddress(request);
+ const userId = getUserId(ip_address, userAgent);
+ const existingSession = await getExistingSession(userId);
const payload = {
...request.body,
- ip_address: getIpAddress(request),
+ ip_address: ip_address,
timestamp: new Date().toISOString(),
- ua: userAgentParser(request.body.userAgent),
- };
-
- const userId = generateUserId(payload.ip_address, payload.userAgent);
- const timestamp = new Date().toISOString();
- const sessionId = crypto.randomUUID();
-
- const insertPayload = {
- ...payload,
- userId,
- timestamp,
- sessionId,
+ ua: userAgentParser(userAgent),
+ referrer: referrer,
+ userId: userId,
+ sessionId: existingSession?.session_id || crypto.randomUUID(),
};
- insertPageview(insertPayload);
- updateSession(insertPayload);
+ insertPageview(payload);
+ updateSession(payload, existingSession);
}
diff --git a/server/src/db/clickhouse.ts b/server/src/db/clickhouse/clickhouse.ts
similarity index 50%
rename from server/src/db/clickhouse.ts
rename to server/src/db/clickhouse/clickhouse.ts
index 567d4ee45..c05fe1ff2 100644
--- a/server/src/db/clickhouse.ts
+++ b/server/src/db/clickhouse/clickhouse.ts
@@ -1,4 +1,5 @@
import { createClient } from "@clickhouse/client";
+import { Session } from "../postgres/types";
export const clickhouse = createClient({
host: process.env.CLICKHOUSE_HOST,
@@ -16,8 +17,8 @@ export const initializeClickhouse = async () => {
hostname String,
pathname String,
querystring String,
+ page_title String,
referrer String,
- user_agent String,
browser LowCardinality(String),
operating_system LowCardinality(String),
language LowCardinality(String),
@@ -31,23 +32,6 @@ export const initializeClickhouse = async () => {
`,
});
- // Create events table
- await clickhouse.exec({
- query: `
- CREATE TABLE IF NOT EXISTS events (
- timestamp DateTime,
- session_id String,
- event_name String,
- event_data String,
- page_url String,
- ip_address String
- )
- ENGINE = MergeTree()
- PARTITION BY toYYYYMM(timestamp)
- ORDER BY (timestamp, session_id)
- `,
- });
-
// Create sessions table
await clickhouse.exec({
query: `
@@ -62,7 +46,6 @@ export const initializeClickhouse = async () => {
events UInt32,
duration UInt32,
referrer String,
- user_agent String,
ip_address String,
browser String,
language String,
@@ -75,52 +58,12 @@ export const initializeClickhouse = async () => {
});
};
-export const insertEvent = async (event: Record<string, any>) => {
- try {
- await clickhouse.insert({
- table: "events",
- values: [
- {
- timestamp: event.timestamp,
- session_id: event.sessionId,
- event_name: event.eventName,
- event_data: JSON.stringify(event.eventData || {}),
- page_url: event.url,
- ip_address: event.ip_address || "",
- },
- ],
- format: "JSONEachRow",
- });
- return true;
- } catch (error) {
- console.error("Error inserting event:", error);
- return false;
- }
-};
-
// Function to insert session data
-export const insertSession = async (session: Record<string, any>) => {
+export const insertSessions = async (sessions: Session[]) => {
try {
await clickhouse.insert({
table: "sessions",
- values: [
- {
- session_id: session.sessionId,
- start_time: session.startTime,
- end_time: session.endTime,
- entry_page: session.entryPage,
- exit_page: session.exitPage,
- pageviews: session.pageviews,
- events: session.events,
- duration: session.duration,
- referrer: session.referrer,
- user_agent: session.userAgent,
- ip_address: session.ipAddress,
- browser: session.browser,
- language: session.language,
- country_code: session.countryCode,
- },
- ],
+ values: sessions,
format: "JSONEachRow",
});
return true;
diff --git a/server/src/db/clickhouse/types.ts b/server/src/db/clickhouse/types.ts
new file mode 100644
index 000000000..43b677c9e
--- /dev/null
+++ b/server/src/db/clickhouse/types.ts
@@ -0,0 +1,18 @@
+import { DateTime } from "luxon";
+
+export type Pageview = {
+ timestamp: DateTime;
+ session_id: string;
+ user_id: string;
+ hostname: string;
+ pathname: string;
+ querystring: string;
+ page_title: string;
+ referrer: string;
+ browser: string;
+ operating_system: string;
+ language: string;
+ screen_width: number;
+ screen_height: number;
+ device_type: string;
+};
diff --git a/server/src/db/postgres.ts b/server/src/db/postgres/postgres.ts
similarity index 94%
rename from server/src/db/postgres.ts
rename to server/src/db/postgres/postgres.ts
index 4d1f0644f..dab3da6a2 100644
--- a/server/src/db/postgres.ts
+++ b/server/src/db/postgres/postgres.ts
@@ -1,4 +1,5 @@
import postgres from "postgres";
+import { Session } from "./types";
export const sql = postgres({
host: process.env.POSTGRES_HOST || "postgres",
@@ -10,7 +11,7 @@ export const sql = postgres({
export async function initializePostgres() {
try {
- await sql`
+ await sql<Session[]>`
CREATE TABLE IF NOT EXISTS active_sessions (
session_id TEXT PRIMARY KEY,
user_id TEXT,
diff --git a/server/src/db/postgres/session-cleanup.ts b/server/src/db/postgres/session-cleanup.ts
new file mode 100644
index 000000000..c2e346c59
--- /dev/null
+++ b/server/src/db/postgres/session-cleanup.ts
@@ -0,0 +1,24 @@
+import { insertSessions } from "../clickhouse/clickhouse";
+import { sql } from "./postgres";
+import { Session } from "./types";
+import { DateTime } from "luxon";
+
+function convertPostgresToClickhouse(postgresTimestamp: string): string {
+ return DateTime.fromISO(postgresTimestamp).toFormat("yyyy-MM-dd HH:mm:ss");
+}
+
+export async function cleanupOldSessions() {
+ const deletedSessions = await sql<Session[]>`
+ DELETE FROM active_sessions
+ WHERE last_activity < NOW() - INTERVAL '1 minute'
+ RETURNING *
+ `;
+ console.log(`Cleaned up ${deletedSessions.length} sessions`);
+ await insertSessions(
+ deletedSessions.map((e) => ({
+ ...e,
+ start_time: convertPostgresToClickhouse(e.start_time),
+ last_activity: convertPostgresToClickhouse(e.last_activity),
+ }))
+ );
+}
diff --git a/server/src/db/postgres/types.ts b/server/src/db/postgres/types.ts
new file mode 100644
index 000000000..ad2d33f2b
--- /dev/null
+++ b/server/src/db/postgres/types.ts
@@ -0,0 +1,17 @@
+export type Session = {
+ session_id: string;
+ user_id: string;
+ hostname: string;
+ start_time: string;
+ last_activity: string;
+ pageviews: number;
+ entry_page: string;
+ exit_page: string;
+ device_type: string;
+ screen_width: number;
+ screen_height: number;
+ browser: string;
+ operating_system: string;
+ language: string;
+ referrer: string;
+};
diff --git a/server/src/index.ts b/server/src/index.ts
index 9e01ce59c..6d0f687b3 100644
--- a/server/src/index.ts
+++ b/server/src/index.ts
@@ -3,11 +3,11 @@ import fastifyStatic from "@fastify/static";
import Fastify, { FastifyReply, FastifyRequest } from "fastify";
import path from "path";
import { trackPageView } from "./actions/trackPageView";
-import { initializeClickhouse, insertEvent } from "./db/clickhouse";
+import { initializeClickhouse } from "./db/clickhouse/clickhouse";
import { TrackingPayload } from "./types";
-import { getIpAddress } from "./utils";
-import { initializePostgres } from "./db/postgres";
-import { UAParser } from "ua-parser-js";
+import { initializePostgres } from "./db/postgres/postgres";
+import cron from "node-cron";
+import { cleanupOldSessions } from "./db/postgres/session-cleanup";
const server = Fastify({
logger: {
@@ -55,43 +55,11 @@ server.post<{ Body: TrackingPayload }>(
}
);
-const uaParser = new UAParser();
-
-// Track event endpoint
-server.post<{ Body: TrackingPayload }>(
- "/track/event",
- async (
- request: FastifyRequest<{ Body: TrackingPayload }>,
- reply: FastifyReply
- ) => {
- try {
- const payload = {
- ...request.body,
- ip_address: getIpAddress(request),
- timestamp: new Date().toISOString(),
- };
-
- const success = await insertEvent(payload);
- if (!success) {
- throw new Error("Failed to insert event");
- }
-
- return { success: true };
- } catch (error) {
- request.log.error(error);
- return reply.status(500).send({
- success: false,
- error: "Failed to track event",
- });
- }
- }
-);
-
const start = async () => {
try {
+ console.info("Starting server...");
// Initialize the database
await Promise.allSettled([initializeClickhouse(), initializePostgres()]);
-
// Start the server
await server.listen({ port: 3001, host: "0.0.0.0" });
} catch (err) {
@@ -101,3 +69,8 @@ const start = async () => {
};
start();
+
+cron.schedule("* * * * *", () => {
+ console.log("Cleaning up old sessions");
+ cleanupOldSessions();
+});
diff --git a/server/src/types.ts b/server/src/types.ts
index 7581b6488..288aa7540 100644
--- a/server/src/types.ts
+++ b/server/src/types.ts
@@ -2,10 +2,10 @@ export interface TrackingPayload {
hostname: string;
pathname: string;
querystring: string;
- referrer: string;
timestamp: string;
- userAgent: string;
screenWidth: number;
screenHeight: number;
language: string;
+ page_title: string;
+ referrer: string;
}
diff --git a/server/src/utils.ts b/server/src/utils.ts
index b3e7c0010..51540e80b 100644
--- a/server/src/utils.ts
+++ b/server/src/utils.ts
@@ -1,7 +1,7 @@
import { FastifyRequest } from "fastify";
import crypto from "crypto";
-export function generateUserId(ip: string, userAgent: string) {
+export function getUserId(ip: string, userAgent: string) {
return crypto
.createHash("sha256")
.update(ip + userAgent)