🐸 Rybbit - open-source and privacy-friendly alternative to Google Analytics that is 10x more intuitive.
wip
Bill Yangac2e78f15235
diff --git a/server/package-lock.json b/server/package-lock.jsonindex 1b340e35d..a725f5ea9 100644--- a/server/package-lock.json+++ b/server/package-lock.json@@ -12,6 +12,7 @@"@fastify/cors": "^10.0.2","@fastify/one-line-logger": "^1.4.0","@fastify/static": "^8.0.4",+ "date-fns": "^2.30.0","fastify": "^5.1.0"},"devDependencies": {@@ -20,6 +21,17 @@"typescript": "^5.7.3"}},+ "node_modules/@babel/runtime": {+ "version": "7.26.0",+ "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.26.0.tgz",+ "integrity": "sha512-FDSOghenHTiToteC/QRlv2q3DhPZ/oOXTBoirfWNx1Cx3TMVcGWQtMMmQcSvb/JjpNeGzx8Pq/b4fKEJuWm1sw==",+ "dependencies": {+ "regenerator-runtime": "^0.14.0"+ },+ "engines": {+ "node": ">=6.9.0"+ }+ },"node_modules/@clickhouse/client": {"version": "0.2.7","resolved": "https://registry.npmjs.org/@clickhouse/client/-/client-0.2.7.tgz",@@ -607,6 +619,21 @@"node": ">= 8"}},+ "node_modules/date-fns": {+ "version": "2.30.0",+ "resolved": "https://registry.npmjs.org/date-fns/-/date-fns-2.30.0.tgz",+ "integrity": "sha512-fnULvOpxnC5/Vg3NCiWelDsLiUc9bRwAPs/+LfTLNvetFCtCTN+yQz15C/fs4AwX1R9K5GLtLfn8QW+dWisaAw==",+ "dependencies": {+ "@babel/runtime": "^7.21.0"+ },+ "engines": {+ "node": ">=0.11"+ },+ "funding": {+ "type": "opencollective",+ "url": "https://opencollective.com/date-fns"+ }+ },"node_modules/dateformat": {"version": "4.6.3","resolved": "https://registry.npmjs.org/dateformat/-/dateformat-4.6.3.tgz",@@ -1426,6 +1453,11 @@"node": ">= 12.13.0"}},+ "node_modules/regenerator-runtime": {+ "version": "0.14.1",+ "resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.14.1.tgz",+ "integrity": "sha512-dYnhHh0nJoMfnkZs6GmmhFknAGRrLznOu5nc9ML+EJxGvrx6H7teuevqVqCuPcPK//3eDrrjQhehXVx9cnkGdw=="+ },"node_modules/require-from-string": {"version": "2.0.2","resolved": "https://registry.npmjs.org/require-from-string/-/require-from-string-2.0.2.tgz",diff --git a/server/package.json b/server/package.jsonindex 5009ce343..a31928436 100644--- a/server/package.json+++ b/server/package.json@@ -13,7 +13,8 @@"@fastify/cors": "^10.0.2","@fastify/static": "^8.0.4","@fastify/one-line-logger": "^1.4.0",- "fastify": "^5.1.0"+ "fastify": "^5.1.0",+ "date-fns": "^2.30.0"},"devDependencies": {"@types/node": "^20.10.0",diff --git a/server/public/analytics.js b/server/public/analytics.jsindex aa494ea37..a84943b96 100644--- a/server/public/analytics.js+++ b/server/public/analytics.js@@ -1,27 +1,45 @@// Frogstats Analytics Script(function () {- const ANALYTICS_URL = "http://localhost:3001"; // Replace with your actual analytics server URL+ // Get the script tag that loaded this script+ const scriptTag = document.currentScript;+ const ANALYTICS_URL = scriptTag.getAttribute("src").split("/analytics.js")[0];// Generate a unique session IDconst generateSessionId = () => {const existingId = localStorage.getItem("frogstats_session_id");- if (existingId) return existingId;+ const lastActivity = localStorage.getItem("frogstats_last_activity");+ const now = Date.now();- const newId =- Math.random().toString(36).substring(2) + Date.now().toString(36);+ // Check if we have an existing session that's less than 30 minutes old+ if (+ existingId &&+ lastActivity &&+ now - parseInt(lastActivity) < 30 * 60 * 1000+ ) {+ localStorage.setItem("frogstats_last_activity", now.toString());+ return existingId;+ }++ // Generate new session ID if none exists or if expired+ const newId = Math.random().toString(36).substring(2) + now.toString(36);localStorage.setItem("frogstats_session_id", newId);+ localStorage.setItem("frogstats_last_activity", now.toString());return newId;};// Track pageviewconst trackPageview = () => {+ const url = new URL(window.location.href);const payload = {- url: window.location.href,+ hostname: url.hostname,+ pathname: url.pathname,+ querystring: url.search,referrer: document.referrer,timestamp: new Date().toISOString(),sessionId: generateSessionId(),userAgent: navigator.userAgent,- screenSize: `${window.innerWidth}x${window.innerHeight}`,+ screenWidth: window.innerWidth,+ screenHeight: window.innerHeight,language: navigator.language,};@@ -69,10 +87,10 @@};// Using sendBeacon for more reliable data sending on page unload- navigator.sendBeacon(- `${ANALYTICS_URL}/track/pageview`,- JSON.stringify(payload)- );+ // navigator.sendBeacon(+ // `${ANALYTICS_URL}/track/pageview`,+ // JSON.stringify(payload)+ // );});// Expose the tracking functions globallydiff --git a/server/src/db/clickhouse.ts b/server/src/db/clickhouse.tsindex aa29ed3ac..a45e191b7 100644--- a/server/src/db/clickhouse.ts+++ b/server/src/db/clickhouse.ts@@ -1,17 +1,11 @@import { createClient } from "@clickhouse/client";+import { format } from "date-fns";+import { getBrowserDetails } from "../utils";-console.info("Testing");const client = createClient({host: process.env.CLICKHOUSE_HOST,database: process.env.CLICKHOUSE_DB,});-// Add some debug logging-// console.info("ClickHouse config:", {-// host: process.env.CLICKHOUSE_HOST,-// database: process.env.CLICKHOUSE_DB,-// username: process.env.CLICKHOUSE_USER,-// password: process.env.CLICKHOUSE_PASSWORD, // This will help verify what's being used-// });export const initializeDatabase = async () => {// Create pageviews table@@ -20,15 +14,16 @@ export const initializeDatabase = async () => {CREATE TABLE IF NOT EXISTS pageviews (timestamp DateTime,session_id String,- page_url String,+ hostname String,+ pathname String,+ querystring String,referrer String,user_agent String,ip_address String,- country String,- device_type String,browser String,- os String,- duration UInt32+ language String,+ screen_width UInt16,+ screen_height UInt16,)ENGINE = MergeTree()PARTITION BY toYYYYMM(timestamp)@@ -44,13 +39,129 @@ export const initializeDatabase = async () => {session_id String,event_name String,event_data String,- page_url String+ page_url String,+ ip_address String)ENGINE = MergeTree()PARTITION BY toYYYYMM(timestamp)ORDER BY (timestamp, session_id)`,});++ // Create sessions table+ await client.exec({+ query: `+ CREATE TABLE IF NOT EXISTS sessions (+ hostname String,+ session_id String,+ start_time DateTime,+ end_time DateTime,+ entry_page String,+ exit_page String,+ pageviews UInt32,+ events UInt32,+ duration UInt32,+ referrer String,+ user_agent String,+ ip_address String,+ browser String,+ language String,+ country_code String+ )+ ENGINE = MergeTree()+ PARTITION BY toYYYYMM(start_time)+ ORDER BY (session_id, start_time)+ `,+ });+};++export const insertPageview = async (pageview: Record<string, any>) => {+ try {+ const formattedTimestamp = format(+ new Date(pageview.timestamp),+ "yyyy-MM-dd HH:mm:ss"+ );+ await client.insert({+ table: "pageviews",+ values: [+ {+ timestamp: formattedTimestamp,+ session_id: pageview.sessionId,+ hostname: pageview.hostname || "",+ pathname: pageview.pathname || "",+ querystring: pageview.querystring || "",+ referrer: pageview.referrer || "",+ user_agent: pageview.userAgent || "",+ ip_address: pageview.ip_address || "",+ browser: getBrowserDetails(pageview.userAgent || ""),+ language: pageview.language || "",+ screen_width: pageview.screenWidth || 0,+ screen_height: pageview.screenHeight || 0,+ duration: pageview.duration || 0,+ },+ ],+ format: "JSONEachRow",+ });+ return true;+ } catch (error) {+ console.error("Error inserting pageview:", error);+ return false;+ }+};++export const insertEvent = async (event: Record<string, any>) => {+ try {+ await client.insert({+ table: "events",+ values: [+ {+ timestamp: event.timestamp,+ session_id: event.sessionId,+ event_name: event.eventName,+ event_data: JSON.stringify(event.eventData || {}),+ page_url: event.url,+ ip_address: event.ip_address || "",+ },+ ],+ format: "JSONEachRow",+ });+ return true;+ } catch (error) {+ console.error("Error inserting event:", error);+ return false;+ }+};++// Function to insert session data+export const insertSession = async (session: Record<string, any>) => {+ try {+ await client.insert({+ table: "sessions",+ values: [+ {+ session_id: session.sessionId,+ start_time: session.startTime,+ end_time: session.endTime,+ entry_page: session.entryPage,+ exit_page: session.exitPage,+ pageviews: session.pageviews,+ events: session.events,+ duration: session.duration,+ referrer: session.referrer,+ user_agent: session.userAgent,+ ip_address: session.ipAddress,+ browser: session.browser,+ language: session.language,+ country_code: session.countryCode,+ },+ ],+ format: "JSONEachRow",+ });+ return true;+ } catch (error) {+ console.error("Error inserting session:", error);+ return false;+ }};export default client;diff --git a/server/src/index.ts b/server/src/index.tsindex f4bc13e6b..490795c4c 100644--- a/server/src/index.ts+++ b/server/src/index.ts@@ -2,7 +2,11 @@ import Fastify, { FastifyRequest, FastifyReply } from "fastify";import cors from "@fastify/cors";import fastifyStatic from "@fastify/static";import path from "path";-import { initializeDatabase } from "./db/clickhouse";+import {+ initializeDatabase,+ insertPageview,+ insertEvent,+} from "./db/clickhouse";const server = Fastify({logger: {@@ -27,15 +31,18 @@ server.register(fastifyStatic, {interface TrackingPayload {url: string;+ pathname?: string;+ querystring?: string;timestamp: string;sessionId: string;referrer?: string;userAgent?: string;- screenSize?: string;+ screenWidth?: number;+ screenHeight?: number;language?: string;+ browser?: string;eventName?: string;eventData?: Record<string, unknown>;- duration?: number;[key: string]: unknown;}@@ -69,14 +76,27 @@ server.post<{ Body: TrackingPayload }>(request: FastifyRequest<{ Body: TrackingPayload }>,reply: FastifyReply) => {- const payload = {- ...request.body,- ip_address: getIpAddress(request),- timestamp: new Date().toISOString(),- };- console.info(payload);- // TODO: Implement pageview tracking logic- return { success: true };+ try {+ console.info(request.body);+ const payload = {+ ...request.body,+ ip_address: getIpAddress(request),+ timestamp: new Date().toISOString(),+ };++ const success = await insertPageview(payload);+ if (!success) {+ throw new Error("Failed to insert pageview");+ }++ return { success: true };+ } catch (error) {+ request.log.error(error);+ return reply.status(500).send({+ success: false,+ error: "Failed to track pageview",+ });+ }});@@ -87,13 +107,26 @@ server.post<{ Body: TrackingPayload }>(request: FastifyRequest<{ Body: TrackingPayload }>,reply: FastifyReply) => {- const payload = {- ...request.body,- ip_address: getIpAddress(request),- timestamp: new Date().toISOString(),- };- // TODO: Implement event tracking logic- return { success: true };+ try {+ const payload = {+ ...request.body,+ ip_address: getIpAddress(request),+ timestamp: new Date().toISOString(),+ };++ const success = await insertEvent(payload);+ if (!success) {+ throw new Error("Failed to insert event");+ }++ return { success: true };+ } catch (error) {+ request.log.error(error);+ return reply.status(500).send({+ success: false,+ error: "Failed to track event",+ });+ }});@@ -103,7 +136,7 @@ const start = async () => {await initializeDatabase();// Start the server- await server.listen({ port: 5000, host: "0.0.0.0" });+ await server.listen({ port: 3001, host: "0.0.0.0" });} catch (err) {server.log.error(err);process.exit(1);diff --git a/server/src/utils.ts b/server/src/utils.tsnew file mode 100644index 000000000..7d68a47db--- /dev/null+++ b/server/src/utils.ts@@ -0,0 +1,17 @@+// Get browser details+export const getBrowserDetails = (userAgent: string) => {+ let browser = "Unknown";+ if (userAgent.indexOf("Chrome") > -1) {+ browser = "Chrome";+ } else if (userAgent.indexOf("Safari") > -1) {+ browser = "Safari";+ } else if (userAgent.indexOf("Firefox") > -1) {+ browser = "Firefox";+ } else if (+ userAgent.indexOf("MSIE") > -1 ||+ userAgent.indexOf("Trident/") > -1+ ) {+ browser = "Internet Explorer";+ }+ return browser;+};