🐸 Rybbit - open-source and privacy-friendly alternative to Google Analytics that is 10x more intuitive.
import session logic
Bill Yang0d798d655675
diff --git a/server/package-lock.json b/server/package-lock.jsonindex e1243e289..6bb473418 100644--- a/server/package-lock.json+++ b/server/package-lock.json@@ -14,12 +14,14 @@"@fastify/static": "^8.0.4","fastify": "^5.1.0","luxon": "^3.5.0",+ "node-cron": "^3.0.3","postgres": "^3.4.5","ua-parser-js": "^2.0.0"},"devDependencies": {"@types/luxon": "^3.4.2","@types/node": "^20.10.0",+ "@types/node-cron": "^3.0.11","ts-node-dev": "^2.0.0","typescript": "^5.7.3"}@@ -302,6 +304,12 @@"undici-types": "~5.26.4"}},+ "node_modules/@types/node-cron": {+ "version": "3.0.11",+ "resolved": "https://registry.npmjs.org/@types/node-cron/-/node-cron-3.0.11.tgz",+ "integrity": "sha512-0ikrnug3/IyneSHqCBeslAhlK2aBfYek1fGo4bP4QnZPmiqSGRK+Oy7ZMisLWkesffJvQ1cqAcBnJC+8+nxIAg==",+ "dev": true+ },"node_modules/@types/strip-bom": {"version": "3.0.0","resolved": "https://registry.npmjs.org/@types/strip-bom/-/strip-bom-3.0.0.tgz",@@ -1251,6 +1259,17 @@"obliterator": "^2.0.1"}},+ "node_modules/node-cron": {+ "version": "3.0.3",+ "resolved": "https://registry.npmjs.org/node-cron/-/node-cron-3.0.3.tgz",+ "integrity": "sha512-dOal67//nohNgYWb+nWmg5dkFdIwDm8EpeGYMekPMrngV3637lqnX0lbUcCtgibHTz6SEz7DAIjKvKDFYCnO1A==",+ "dependencies": {+ "uuid": "8.3.2"+ },+ "engines": {+ "node": ">=6.0.0"+ }+ },"node_modules/normalize-path": {"version": "3.0.0","resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz",@@ -2082,6 +2101,14 @@"integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==","dev": true},+ "node_modules/uuid": {+ "version": "8.3.2",+ "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz",+ "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==",+ "bin": {+ "uuid": "dist/bin/uuid"+ }+ },"node_modules/v8-compile-cache-lib": {"version": "3.0.1","resolved": "https://registry.npmjs.org/v8-compile-cache-lib/-/v8-compile-cache-lib-3.0.1.tgz",diff --git a/server/package.json b/server/package.jsonindex 56bd5e537..d8f68eac9 100644--- a/server/package.json+++ b/server/package.json@@ -15,12 +15,14 @@"@fastify/static": "^8.0.4","fastify": "^5.1.0","luxon": "^3.5.0",+ "node-cron": "^3.0.3","postgres": "^3.4.5","ua-parser-js": "^2.0.0"},"devDependencies": {"@types/luxon": "^3.4.2","@types/node": "^20.10.0",+ "@types/node-cron": "^3.0.11","ts-node-dev": "^2.0.0","typescript": "^5.7.3"}diff --git a/server/public/analytics.js b/server/public/analytics.jsindex a1c5fc4a7..730ce43e5 100644--- a/server/public/analytics.js+++ b/server/public/analytics.js@@ -4,19 +4,32 @@const scriptTag = document.currentScript;const ANALYTICS_URL = scriptTag.getAttribute("src").split("/analytics.js")[0];- // Track pageview+ // Get debounce duration from data attribute or default to 300ms+ const debounceDuration = scriptTag.getAttribute("data-debounce")+ ? Math.max(0, parseInt(scriptTag.getAttribute("data-debounce")))+ : 500;++ // Debounce function+ function debounce(func, wait) {+ let timeout;+ return (...args) => {+ clearTimeout(timeout);+ timeout = setTimeout(() => func.apply(this, args), wait);+ };+ }++ // Track pageview (core functionality)const trackPageview = () => {const url = new URL(window.location.href);const payload = {hostname: url.hostname,pathname: url.pathname,querystring: url.search,- referrer: document.referrer,- timestamp: new Date().toISOString(),- userAgent: navigator.userAgent,screenWidth: window.innerWidth,screenHeight: window.innerHeight,language: navigator.language,+ page_title: document.title,+ referrer: document.referrer,};fetch(`${ANALYTICS_URL}/track/pageview`, {@@ -30,49 +43,38 @@}).catch(console.error);};- // Track custom events- const trackEvent = (eventName, eventData = {}) => {- const payload = {- eventName,- eventData,- url: window.location.href,- timestamp: new Date().toISOString(),- };+ // Create debounced version with configured duration+ const debouncedTrackPageview =+ debounceDuration > 0+ ? debounce(trackPageview, debounceDuration)+ : trackPageview;- fetch(`${ANALYTICS_URL}/track/event`, {- method: "POST",- headers: {- "Content-Type": "application/json",- },- body: JSON.stringify(payload),- mode: "cors",- keepalive: true,- }).catch(console.error);+ // History API override+ const originalPushState = history.pushState;+ const originalReplaceState = history.replaceState;++ history.pushState = function (...args) {+ console.info("pushState");+ originalPushState.apply(this, args);+ debouncedTrackPageview();};- // Track page duration- let pageLoadTime = Date.now();- window.addEventListener("beforeunload", () => {- const duration = Math.floor((Date.now() - pageLoadTime) / 1000);- const payload = {- url: window.location.href,- timestamp: new Date().toISOString(),- duration,- };+ history.replaceState = function (...args) {+ console.info("replaceState");+ originalReplaceState.apply(this, args);+ debouncedTrackPageview();+ };- // Using sendBeacon for more reliable data sending on page unload- // navigator.sendBeacon(- // `${ANALYTICS_URL}/track/pageview`,- // JSON.stringify(payload)- // );- });+ // Handle navigation events+ window.addEventListener("popstate", debouncedTrackPageview);- // Expose the tracking functions globally+ // Expose APIwindow.frogstats = {- trackEvent,- trackPageview,+ trackPageview: debouncedTrackPageview,+ // Add direct access to non-debounced version if needed+ _trackImmediately: trackPageview,};- // Track initial pageview+ // Initial pageviewtrackPageview();})();diff --git a/server/src/actions/trackPageView.ts b/server/src/actions/trackPageView.tsindex e27d63a58..7934fb79b 100644--- a/server/src/actions/trackPageView.ts+++ b/server/src/actions/trackPageView.ts@@ -1,20 +1,22 @@import { FastifyRequest } from "fastify";import { TrackingPayload } from "../types";-import { generateUserId, getDeviceType, getIpAddress } from "../utils";+import { getUserId, getDeviceType, getIpAddress } from "../utils";import crypto from "crypto";-import clickhouse from "../db/clickhouse";+import clickhouse from "../db/clickhouse/clickhouse";import { DateTime } from "luxon";-import { sql } from "../db/postgres";+import { sql } from "../db/postgres/postgres";import UAParser, { UAParser as userAgentParser } from "ua-parser-js";+import { Pageview } from "../db/clickhouse/types";-const insertPageview = async (- pageview: TrackingPayload & {- userId: string;- timestamp: string;- sessionId: string;- ua: UAParser.IResult;- }-) => {+type TotalPayload = TrackingPayload & {+ userId: string;+ timestamp: string;+ sessionId: string;+ ua: UAParser.IResult;+ referrer: string;+};++const insertPageview = async (pageview: TotalPayload) => {try {const formattedTimestamp = DateTime.fromISO(pageview.timestamp).toFormat("yyyy-MM-dd HH:mm:ss"@@ -29,8 +31,8 @@ const insertPageview = async (hostname: pageview.hostname || "",pathname: pageview.pathname || "",querystring: pageview.querystring || "",+ page_title: pageview.page_title || "",referrer: pageview.referrer || "",- user_agent: pageview.userAgent || "",browser: pageview.ua.browser.name,operating_system: pageview.ua.os.name,language: pageview.language || "",@@ -51,17 +53,17 @@ const insertPageview = async (}};+const getExistingSession = async (userId: string): Promise<Pageview | null> => {+ const [existingSession] = await sql<Pageview[]>`+ SELECT * FROM active_sessions WHERE user_id = ${userId}+ `;+ return existingSession;+};+const updateSession = async (- pageview: TrackingPayload & {- userId: string;- timestamp: string;- sessionId: string;- ua: UAParser.IResult;- }+ pageview: TotalPayload,+ existingSession: Pageview | null) => {- const [existingSession] = await sql`- SELECT * FROM active_sessions WHERE user_id = ${pageview.userId}- `;if (existingSession) {await sql`UPDATE active_sessions SET last_activity = ${pageview.timestamp}, pageviews = pageviews + 1 WHERE user_id = ${pageview.userId}@@ -92,27 +94,24 @@ const updateSession = async (`;};-export function trackPageView(+export async function trackPageView(request: FastifyRequest<{ Body: TrackingPayload }>) {+ const userAgent = request.headers["user-agent"] || "";+ const referrer = request.headers["referer"] || "";+ const ip_address = getIpAddress(request);+ const userId = getUserId(ip_address, userAgent);+ const existingSession = await getExistingSession(userId);const payload = {...request.body,- ip_address: getIpAddress(request),+ ip_address: ip_address,timestamp: new Date().toISOString(),- ua: userAgentParser(request.body.userAgent),- };-- const userId = generateUserId(payload.ip_address, payload.userAgent);- const timestamp = new Date().toISOString();- const sessionId = crypto.randomUUID();-- const insertPayload = {- ...payload,- userId,- timestamp,- sessionId,+ ua: userAgentParser(userAgent),+ referrer: referrer,+ userId: userId,+ sessionId: existingSession?.session_id || crypto.randomUUID(),};- insertPageview(insertPayload);- updateSession(insertPayload);+ insertPageview(payload);+ updateSession(payload, existingSession);}diff --git a/server/src/db/clickhouse.ts b/server/src/db/clickhouse/clickhouse.tssimilarity index 50%rename from server/src/db/clickhouse.tsrename to server/src/db/clickhouse/clickhouse.tsindex 567d4ee45..c05fe1ff2 100644--- a/server/src/db/clickhouse.ts+++ b/server/src/db/clickhouse/clickhouse.ts@@ -1,4 +1,5 @@import { createClient } from "@clickhouse/client";+import { Session } from "../postgres/types";export const clickhouse = createClient({host: process.env.CLICKHOUSE_HOST,@@ -16,8 +17,8 @@ export const initializeClickhouse = async () => {hostname String,pathname String,querystring String,+ page_title String,referrer String,- user_agent String,browser LowCardinality(String),operating_system LowCardinality(String),language LowCardinality(String),@@ -31,23 +32,6 @@ export const initializeClickhouse = async () => {`,});- // Create events table- await clickhouse.exec({- query: `- CREATE TABLE IF NOT EXISTS events (- timestamp DateTime,- session_id String,- event_name String,- event_data String,- page_url String,- ip_address String- )- ENGINE = MergeTree()- PARTITION BY toYYYYMM(timestamp)- ORDER BY (timestamp, session_id)- `,- });-// Create sessions tableawait clickhouse.exec({query: `@@ -62,7 +46,6 @@ export const initializeClickhouse = async () => {events UInt32,duration UInt32,referrer String,- user_agent String,ip_address String,browser String,language String,@@ -75,52 +58,12 @@ export const initializeClickhouse = async () => {});};-export const insertEvent = async (event: Record<string, any>) => {- try {- await clickhouse.insert({- table: "events",- values: [- {- timestamp: event.timestamp,- session_id: event.sessionId,- event_name: event.eventName,- event_data: JSON.stringify(event.eventData || {}),- page_url: event.url,- ip_address: event.ip_address || "",- },- ],- format: "JSONEachRow",- });- return true;- } catch (error) {- console.error("Error inserting event:", error);- return false;- }-};-// Function to insert session data-export const insertSession = async (session: Record<string, any>) => {+export const insertSessions = async (sessions: Session[]) => {try {await clickhouse.insert({table: "sessions",- values: [- {- session_id: session.sessionId,- start_time: session.startTime,- end_time: session.endTime,- entry_page: session.entryPage,- exit_page: session.exitPage,- pageviews: session.pageviews,- events: session.events,- duration: session.duration,- referrer: session.referrer,- user_agent: session.userAgent,- ip_address: session.ipAddress,- browser: session.browser,- language: session.language,- country_code: session.countryCode,- },- ],+ values: sessions,format: "JSONEachRow",});return true;diff --git a/server/src/db/clickhouse/types.ts b/server/src/db/clickhouse/types.tsnew file mode 100644index 000000000..43b677c9e--- /dev/null+++ b/server/src/db/clickhouse/types.ts@@ -0,0 +1,18 @@+import { DateTime } from "luxon";++export type Pageview = {+ timestamp: DateTime;+ session_id: string;+ user_id: string;+ hostname: string;+ pathname: string;+ querystring: string;+ page_title: string;+ referrer: string;+ browser: string;+ operating_system: string;+ language: string;+ screen_width: number;+ screen_height: number;+ device_type: string;+};diff --git a/server/src/db/postgres.ts b/server/src/db/postgres/postgres.tssimilarity index 94%rename from server/src/db/postgres.tsrename to server/src/db/postgres/postgres.tsindex 4d1f0644f..dab3da6a2 100644--- a/server/src/db/postgres.ts+++ b/server/src/db/postgres/postgres.ts@@ -1,4 +1,5 @@import postgres from "postgres";+import { Session } from "./types";export const sql = postgres({host: process.env.POSTGRES_HOST || "postgres",@@ -10,7 +11,7 @@ export const sql = postgres({export async function initializePostgres() {try {- await sql`+ await sql<Session[]>`CREATE TABLE IF NOT EXISTS active_sessions (session_id TEXT PRIMARY KEY,user_id TEXT,diff --git a/server/src/db/postgres/session-cleanup.ts b/server/src/db/postgres/session-cleanup.tsnew file mode 100644index 000000000..c2e346c59--- /dev/null+++ b/server/src/db/postgres/session-cleanup.ts@@ -0,0 +1,24 @@+import { insertSessions } from "../clickhouse/clickhouse";+import { sql } from "./postgres";+import { Session } from "./types";+import { DateTime } from "luxon";++function convertPostgresToClickhouse(postgresTimestamp: string): string {+ return DateTime.fromISO(postgresTimestamp).toFormat("yyyy-MM-dd HH:mm:ss");+}++export async function cleanupOldSessions() {+ const deletedSessions = await sql<Session[]>`+ DELETE FROM active_sessions+ WHERE last_activity < NOW() - INTERVAL '1 minute'+ RETURNING *+ `;+ console.log(`Cleaned up ${deletedSessions.length} sessions`);+ await insertSessions(+ deletedSessions.map((e) => ({+ ...e,+ start_time: convertPostgresToClickhouse(e.start_time),+ last_activity: convertPostgresToClickhouse(e.last_activity),+ }))+ );+}diff --git a/server/src/db/postgres/types.ts b/server/src/db/postgres/types.tsnew file mode 100644index 000000000..ad2d33f2b--- /dev/null+++ b/server/src/db/postgres/types.ts@@ -0,0 +1,17 @@+export type Session = {+ session_id: string;+ user_id: string;+ hostname: string;+ start_time: string;+ last_activity: string;+ pageviews: number;+ entry_page: string;+ exit_page: string;+ device_type: string;+ screen_width: number;+ screen_height: number;+ browser: string;+ operating_system: string;+ language: string;+ referrer: string;+};diff --git a/server/src/index.ts b/server/src/index.tsindex 9e01ce59c..6d0f687b3 100644--- a/server/src/index.ts+++ b/server/src/index.ts@@ -3,11 +3,11 @@ import fastifyStatic from "@fastify/static";import Fastify, { FastifyReply, FastifyRequest } from "fastify";import path from "path";import { trackPageView } from "./actions/trackPageView";-import { initializeClickhouse, insertEvent } from "./db/clickhouse";+import { initializeClickhouse } from "./db/clickhouse/clickhouse";import { TrackingPayload } from "./types";-import { getIpAddress } from "./utils";-import { initializePostgres } from "./db/postgres";-import { UAParser } from "ua-parser-js";+import { initializePostgres } from "./db/postgres/postgres";+import cron from "node-cron";+import { cleanupOldSessions } from "./db/postgres/session-cleanup";const server = Fastify({logger: {@@ -55,43 +55,11 @@ server.post<{ Body: TrackingPayload }>(});-const uaParser = new UAParser();--// Track event endpoint-server.post<{ Body: TrackingPayload }>(- "/track/event",- async (- request: FastifyRequest<{ Body: TrackingPayload }>,- reply: FastifyReply- ) => {- try {- const payload = {- ...request.body,- ip_address: getIpAddress(request),- timestamp: new Date().toISOString(),- };-- const success = await insertEvent(payload);- if (!success) {- throw new Error("Failed to insert event");- }-- return { success: true };- } catch (error) {- request.log.error(error);- return reply.status(500).send({- success: false,- error: "Failed to track event",- });- }- }-);-const start = async () => {try {+ console.info("Starting server...");// Initialize the databaseawait Promise.allSettled([initializeClickhouse(), initializePostgres()]);-// Start the serverawait server.listen({ port: 3001, host: "0.0.0.0" });} catch (err) {@@ -101,3 +69,8 @@ const start = async () => {};start();++cron.schedule("* * * * *", () => {+ console.log("Cleaning up old sessions");+ cleanupOldSessions();+});diff --git a/server/src/types.ts b/server/src/types.tsindex 7581b6488..288aa7540 100644--- a/server/src/types.ts+++ b/server/src/types.ts@@ -2,10 +2,10 @@ export interface TrackingPayload {hostname: string;pathname: string;querystring: string;- referrer: string;timestamp: string;- userAgent: string;screenWidth: number;screenHeight: number;language: string;+ page_title: string;+ referrer: string;}diff --git a/server/src/utils.ts b/server/src/utils.tsindex b3e7c0010..51540e80b 100644--- a/server/src/utils.ts+++ b/server/src/utils.ts@@ -1,7 +1,7 @@import { FastifyRequest } from "fastify";import crypto from "crypto";-export function generateUserId(ip: string, userAgent: string) {+export function getUserId(ip: string, userAgent: string) {return crypto.createHash("sha256").update(ip + userAgent)