Skip to content

🐸 Rybbit - open-source and privacy-friendly alternative to Google Analytics that is 10x more intuitive.

wip

Bill Yangac2e78f15235
diff --git a/server/package-lock.json b/server/package-lock.json
index 1b340e35d..a725f5ea9 100644
--- a/server/package-lock.json
+++ b/server/package-lock.json
@@ -12,6 +12,7 @@
"@fastify/cors": "^10.0.2",
"@fastify/one-line-logger": "^1.4.0",
"@fastify/static": "^8.0.4",
+ "date-fns": "^2.30.0",
"fastify": "^5.1.0"
},
"devDependencies": {
@@ -20,6 +21,17 @@
"typescript": "^5.7.3"
}
},
+ "node_modules/@babel/runtime": {
+ "version": "7.26.0",
+ "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.26.0.tgz",
+ "integrity": "sha512-FDSOghenHTiToteC/QRlv2q3DhPZ/oOXTBoirfWNx1Cx3TMVcGWQtMMmQcSvb/JjpNeGzx8Pq/b4fKEJuWm1sw==",
+ "dependencies": {
+ "regenerator-runtime": "^0.14.0"
+ },
+ "engines": {
+ "node": ">=6.9.0"
+ }
+ },
"node_modules/@clickhouse/client": {
"version": "0.2.7",
"resolved": "https://registry.npmjs.org/@clickhouse/client/-/client-0.2.7.tgz",
@@ -607,6 +619,21 @@
"node": ">= 8"
}
},
+ "node_modules/date-fns": {
+ "version": "2.30.0",
+ "resolved": "https://registry.npmjs.org/date-fns/-/date-fns-2.30.0.tgz",
+ "integrity": "sha512-fnULvOpxnC5/Vg3NCiWelDsLiUc9bRwAPs/+LfTLNvetFCtCTN+yQz15C/fs4AwX1R9K5GLtLfn8QW+dWisaAw==",
+ "dependencies": {
+ "@babel/runtime": "^7.21.0"
+ },
+ "engines": {
+ "node": ">=0.11"
+ },
+ "funding": {
+ "type": "opencollective",
+ "url": "https://opencollective.com/date-fns"
+ }
+ },
"node_modules/dateformat": {
"version": "4.6.3",
"resolved": "https://registry.npmjs.org/dateformat/-/dateformat-4.6.3.tgz",
@@ -1426,6 +1453,11 @@
"node": ">= 12.13.0"
}
},
+ "node_modules/regenerator-runtime": {
+ "version": "0.14.1",
+ "resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.14.1.tgz",
+ "integrity": "sha512-dYnhHh0nJoMfnkZs6GmmhFknAGRrLznOu5nc9ML+EJxGvrx6H7teuevqVqCuPcPK//3eDrrjQhehXVx9cnkGdw=="
+ },
"node_modules/require-from-string": {
"version": "2.0.2",
"resolved": "https://registry.npmjs.org/require-from-string/-/require-from-string-2.0.2.tgz",
diff --git a/server/package.json b/server/package.json
index 5009ce343..a31928436 100644
--- a/server/package.json
+++ b/server/package.json
@@ -13,7 +13,8 @@
"@fastify/cors": "^10.0.2",
"@fastify/static": "^8.0.4",
"@fastify/one-line-logger": "^1.4.0",
- "fastify": "^5.1.0"
+ "fastify": "^5.1.0",
+ "date-fns": "^2.30.0"
},
"devDependencies": {
"@types/node": "^20.10.0",
diff --git a/server/public/analytics.js b/server/public/analytics.js
index aa494ea37..a84943b96 100644
--- a/server/public/analytics.js
+++ b/server/public/analytics.js
@@ -1,27 +1,45 @@
// Frogstats Analytics Script
(function () {
- const ANALYTICS_URL = "http://localhost:3001"; // Replace with your actual analytics server URL
+ // Get the script tag that loaded this script
+ const scriptTag = document.currentScript;
+ const ANALYTICS_URL = scriptTag.getAttribute("src").split("/analytics.js")[0];
// Generate a unique session ID
const generateSessionId = () => {
const existingId = localStorage.getItem("frogstats_session_id");
- if (existingId) return existingId;
+ const lastActivity = localStorage.getItem("frogstats_last_activity");
+ const now = Date.now();
- const newId =
- Math.random().toString(36).substring(2) + Date.now().toString(36);
+ // Check if we have an existing session that's less than 30 minutes old
+ if (
+ existingId &&
+ lastActivity &&
+ now - parseInt(lastActivity) < 30 * 60 * 1000
+ ) {
+ localStorage.setItem("frogstats_last_activity", now.toString());
+ return existingId;
+ }
+
+ // Generate new session ID if none exists or if expired
+ const newId = Math.random().toString(36).substring(2) + now.toString(36);
localStorage.setItem("frogstats_session_id", newId);
+ localStorage.setItem("frogstats_last_activity", now.toString());
return newId;
};
// Track pageview
const trackPageview = () => {
+ const url = new URL(window.location.href);
const payload = {
- url: window.location.href,
+ hostname: url.hostname,
+ pathname: url.pathname,
+ querystring: url.search,
referrer: document.referrer,
timestamp: new Date().toISOString(),
sessionId: generateSessionId(),
userAgent: navigator.userAgent,
- screenSize: `${window.innerWidth}x${window.innerHeight}`,
+ screenWidth: window.innerWidth,
+ screenHeight: window.innerHeight,
language: navigator.language,
};
@@ -69,10 +87,10 @@
};
// Using sendBeacon for more reliable data sending on page unload
- navigator.sendBeacon(
- `${ANALYTICS_URL}/track/pageview`,
- JSON.stringify(payload)
- );
+ // navigator.sendBeacon(
+ // `${ANALYTICS_URL}/track/pageview`,
+ // JSON.stringify(payload)
+ // );
});
// Expose the tracking functions globally
diff --git a/server/src/db/clickhouse.ts b/server/src/db/clickhouse.ts
index aa29ed3ac..a45e191b7 100644
--- a/server/src/db/clickhouse.ts
+++ b/server/src/db/clickhouse.ts
@@ -1,17 +1,11 @@
import { createClient } from "@clickhouse/client";
+import { format } from "date-fns";
+import { getBrowserDetails } from "../utils";
-console.info("Testing");
const client = createClient({
host: process.env.CLICKHOUSE_HOST,
database: process.env.CLICKHOUSE_DB,
});
-// Add some debug logging
-// console.info("ClickHouse config:", {
-// host: process.env.CLICKHOUSE_HOST,
-// database: process.env.CLICKHOUSE_DB,
-// username: process.env.CLICKHOUSE_USER,
-// password: process.env.CLICKHOUSE_PASSWORD, // This will help verify what's being used
-// });
export const initializeDatabase = async () => {
// Create pageviews table
@@ -20,15 +14,16 @@ export const initializeDatabase = async () => {
CREATE TABLE IF NOT EXISTS pageviews (
timestamp DateTime,
session_id String,
- page_url String,
+ hostname String,
+ pathname String,
+ querystring String,
referrer String,
user_agent String,
ip_address String,
- country String,
- device_type String,
browser String,
- os String,
- duration UInt32
+ language String,
+ screen_width UInt16,
+ screen_height UInt16,
)
ENGINE = MergeTree()
PARTITION BY toYYYYMM(timestamp)
@@ -44,13 +39,129 @@ export const initializeDatabase = async () => {
session_id String,
event_name String,
event_data String,
- page_url String
+ page_url String,
+ ip_address String
)
ENGINE = MergeTree()
PARTITION BY toYYYYMM(timestamp)
ORDER BY (timestamp, session_id)
`,
});
+
+ // Create sessions table
+ await client.exec({
+ query: `
+ CREATE TABLE IF NOT EXISTS sessions (
+ hostname String,
+ session_id String,
+ start_time DateTime,
+ end_time DateTime,
+ entry_page String,
+ exit_page String,
+ pageviews UInt32,
+ events UInt32,
+ duration UInt32,
+ referrer String,
+ user_agent String,
+ ip_address String,
+ browser String,
+ language String,
+ country_code String
+ )
+ ENGINE = MergeTree()
+ PARTITION BY toYYYYMM(start_time)
+ ORDER BY (session_id, start_time)
+ `,
+ });
+};
+
+export const insertPageview = async (pageview: Record<string, any>) => {
+ try {
+ const formattedTimestamp = format(
+ new Date(pageview.timestamp),
+ "yyyy-MM-dd HH:mm:ss"
+ );
+ await client.insert({
+ table: "pageviews",
+ values: [
+ {
+ timestamp: formattedTimestamp,
+ session_id: pageview.sessionId,
+ hostname: pageview.hostname || "",
+ pathname: pageview.pathname || "",
+ querystring: pageview.querystring || "",
+ referrer: pageview.referrer || "",
+ user_agent: pageview.userAgent || "",
+ ip_address: pageview.ip_address || "",
+ browser: getBrowserDetails(pageview.userAgent || ""),
+ language: pageview.language || "",
+ screen_width: pageview.screenWidth || 0,
+ screen_height: pageview.screenHeight || 0,
+ duration: pageview.duration || 0,
+ },
+ ],
+ format: "JSONEachRow",
+ });
+ return true;
+ } catch (error) {
+ console.error("Error inserting pageview:", error);
+ return false;
+ }
+};
+
+export const insertEvent = async (event: Record<string, any>) => {
+ try {
+ await client.insert({
+ table: "events",
+ values: [
+ {
+ timestamp: event.timestamp,
+ session_id: event.sessionId,
+ event_name: event.eventName,
+ event_data: JSON.stringify(event.eventData || {}),
+ page_url: event.url,
+ ip_address: event.ip_address || "",
+ },
+ ],
+ format: "JSONEachRow",
+ });
+ return true;
+ } catch (error) {
+ console.error("Error inserting event:", error);
+ return false;
+ }
+};
+
+// Function to insert session data
+export const insertSession = async (session: Record<string, any>) => {
+ try {
+ await client.insert({
+ table: "sessions",
+ values: [
+ {
+ session_id: session.sessionId,
+ start_time: session.startTime,
+ end_time: session.endTime,
+ entry_page: session.entryPage,
+ exit_page: session.exitPage,
+ pageviews: session.pageviews,
+ events: session.events,
+ duration: session.duration,
+ referrer: session.referrer,
+ user_agent: session.userAgent,
+ ip_address: session.ipAddress,
+ browser: session.browser,
+ language: session.language,
+ country_code: session.countryCode,
+ },
+ ],
+ format: "JSONEachRow",
+ });
+ return true;
+ } catch (error) {
+ console.error("Error inserting session:", error);
+ return false;
+ }
};
export default client;
diff --git a/server/src/index.ts b/server/src/index.ts
index f4bc13e6b..490795c4c 100644
--- a/server/src/index.ts
+++ b/server/src/index.ts
@@ -2,7 +2,11 @@ import Fastify, { FastifyRequest, FastifyReply } from "fastify";
import cors from "@fastify/cors";
import fastifyStatic from "@fastify/static";
import path from "path";
-import { initializeDatabase } from "./db/clickhouse";
+import {
+ initializeDatabase,
+ insertPageview,
+ insertEvent,
+} from "./db/clickhouse";
const server = Fastify({
logger: {
@@ -27,15 +31,18 @@ server.register(fastifyStatic, {
interface TrackingPayload {
url: string;
+ pathname?: string;
+ querystring?: string;
timestamp: string;
sessionId: string;
referrer?: string;
userAgent?: string;
- screenSize?: string;
+ screenWidth?: number;
+ screenHeight?: number;
language?: string;
+ browser?: string;
eventName?: string;
eventData?: Record<string, unknown>;
- duration?: number;
[key: string]: unknown;
}
@@ -69,14 +76,27 @@ server.post<{ Body: TrackingPayload }>(
request: FastifyRequest<{ Body: TrackingPayload }>,
reply: FastifyReply
) => {
- const payload = {
- ...request.body,
- ip_address: getIpAddress(request),
- timestamp: new Date().toISOString(),
- };
- console.info(payload);
- // TODO: Implement pageview tracking logic
- return { success: true };
+ try {
+ console.info(request.body);
+ const payload = {
+ ...request.body,
+ ip_address: getIpAddress(request),
+ timestamp: new Date().toISOString(),
+ };
+
+ const success = await insertPageview(payload);
+ if (!success) {
+ throw new Error("Failed to insert pageview");
+ }
+
+ return { success: true };
+ } catch (error) {
+ request.log.error(error);
+ return reply.status(500).send({
+ success: false,
+ error: "Failed to track pageview",
+ });
+ }
}
);
@@ -87,13 +107,26 @@ server.post<{ Body: TrackingPayload }>(
request: FastifyRequest<{ Body: TrackingPayload }>,
reply: FastifyReply
) => {
- const payload = {
- ...request.body,
- ip_address: getIpAddress(request),
- timestamp: new Date().toISOString(),
- };
- // TODO: Implement event tracking logic
- return { success: true };
+ try {
+ const payload = {
+ ...request.body,
+ ip_address: getIpAddress(request),
+ timestamp: new Date().toISOString(),
+ };
+
+ const success = await insertEvent(payload);
+ if (!success) {
+ throw new Error("Failed to insert event");
+ }
+
+ return { success: true };
+ } catch (error) {
+ request.log.error(error);
+ return reply.status(500).send({
+ success: false,
+ error: "Failed to track event",
+ });
+ }
}
);
@@ -103,7 +136,7 @@ const start = async () => {
await initializeDatabase();
// Start the server
- await server.listen({ port: 5000, host: "0.0.0.0" });
+ await server.listen({ port: 3001, host: "0.0.0.0" });
} catch (err) {
server.log.error(err);
process.exit(1);
diff --git a/server/src/utils.ts b/server/src/utils.ts
new file mode 100644
index 000000000..7d68a47db
--- /dev/null
+++ b/server/src/utils.ts
@@ -0,0 +1,17 @@
+// Get browser details
+export const getBrowserDetails = (userAgent: string) => {
+ let browser = "Unknown";
+ if (userAgent.indexOf("Chrome") > -1) {
+ browser = "Chrome";
+ } else if (userAgent.indexOf("Safari") > -1) {
+ browser = "Safari";
+ } else if (userAgent.indexOf("Firefox") > -1) {
+ browser = "Firefox";
+ } else if (
+ userAgent.indexOf("MSIE") > -1 ||
+ userAgent.indexOf("Trident/") > -1
+ ) {
+ browser = "Internet Explorer";
+ }
+ return browser;
+};