🐸 Rybbit - open-source and privacy-friendly alternative to Google Analytics that is 10x more intuitive.
Refactor analytics to utilize structured URL parameters
Bill Yang611d8ac088b3
diff --git a/server/src/api/analytics/getSessions.ts b/server/src/api/analytics/getSessions.tsindex 99175c445..ad239127d 100644--- a/server/src/api/analytics/getSessions.ts+++ b/server/src/api/analytics/getSessions.ts@@ -79,11 +79,12 @@ export async function getSessions(argMax(screen_height, timestamp) AS screen_height,argMin(referrer, timestamp) AS referrer,argMin(channel, timestamp) AS channel,- argMin(utm_source, timestamp) AS utm_source,- argMin(utm_medium, timestamp) AS utm_medium,- argMin(utm_campaign, timestamp) AS utm_campaign,- argMin(utm_term, timestamp) AS utm_term,- argMin(utm_content, timestamp) AS utm_content,+ /* UTM parameters from url_parameters map */+ argMin(url_parameters, timestamp)['utm_source'] AS utm_source,+ argMin(url_parameters, timestamp)['utm_medium'] AS utm_medium,+ argMin(url_parameters, timestamp)['utm_campaign'] AS utm_campaign,+ argMin(url_parameters, timestamp)['utm_term'] AS utm_term,+ argMin(url_parameters, timestamp)['utm_content'] AS utm_content,MAX(timestamp) AS session_end,MIN(timestamp) AS session_start,dateDiff('second', MIN(timestamp), MAX(timestamp)) AS session_duration,diff --git a/server/src/api/analytics/getSingleCol.ts b/server/src/api/analytics/getSingleCol.tsindex 7095280ea..0505ba470 100644--- a/server/src/api/analytics/getSingleCol.ts+++ b/server/src/api/analytics/getSingleCol.ts@@ -1,14 +1,13 @@import { FastifyReply, FastifyRequest } from "fastify";import clickhouse from "../../db/clickhouse/clickhouse.js";+import { getUserHasAccessToSitePublic } from "../../lib/auth-utils.js";+import { FilterParameter } from "./types.js";import {- geSqlParam,getFilterStatement,+ getSqlParam,getTimeStatement,processResults,} from "./utils.js";-import { getUserHasAccessToSitePublic } from "../../lib/auth-utils.js";-import { FilterParameter } from "./types.js";-import SqlString from "sqlstring";interface GenericRequest {Params: {@@ -209,14 +208,14 @@ const getQuery = (request: FastifyRequest<GenericRequest>) => {return `WITH PageStats AS (SELECT- ${geSqlParam(parameter)} as value,+ ${getSqlParam(parameter)} as value,COUNT(distinct(session_id)) as unique_sessions,COUNT() as pageviewsFROM eventsWHEREsite_id = {siteId:Int32}- AND ${geSqlParam(parameter)} IS NOT NULL- AND ${geSqlParam(parameter)} <> ''+ AND ${getSqlParam(parameter)} IS NOT NULL+ AND ${getSqlParam(parameter)} <> ''${filterStatement}${timeStatement}// AND type = 'pageview'diff --git a/server/src/api/analytics/utils.ts b/server/src/api/analytics/utils.tsindex 37e17431b..064e561f4 100644--- a/server/src/api/analytics/utils.ts+++ b/server/src/api/analytics/utils.ts@@ -85,7 +85,19 @@ const filterTypeToOperator = (type: FilterType) => {}};-export const geSqlParam = (parameter: FilterParameter) => {+export const getSqlParam = (parameter: FilterParameter) => {+ // Handle URL parameters through the url_parameters map+ if (parameter.startsWith("utm_") || parameter.startsWith("url_param:")) {+ // For explicit url_param: prefix (e.g., url_param:campaign_id)+ if (parameter.startsWith("url_param:")) {+ const paramName = parameter.substring("url_param:".length);+ return `url_parameters['${paramName}']`;+ }++ const utm = parameter; // e.g., utm_source, utm_medium, etc.+ return `url_parameters['${utm}']`;+ }+if (parameter === "referrer") {return "domainWithoutWWW(referrer)";}@@ -197,14 +209,14 @@ export function getFilterStatement(filters: string) {}if (filter.value.length === 1) {- return `${geSqlParam(filter.parameter)} ${filterTypeToOperator(+ return `${getSqlParam(filter.parameter)} ${filterTypeToOperator(filter.type)} ${SqlString.escape(x + filter.value[0] + x)}`;}const valuesWithOperator = filter.value.map((value) =>- `${geSqlParam(filter.parameter)} ${filterTypeToOperator(+ `${getSqlParam(filter.parameter)} ${filterTypeToOperator(filter.type)} ${SqlString.escape(x + value + x)}`);diff --git a/server/src/db/clickhouse/clickhouse.ts b/server/src/db/clickhouse/clickhouse.tsindex 99b27e9c4..715ad011e 100644--- a/server/src/db/clickhouse/clickhouse.ts+++ b/server/src/db/clickhouse/clickhouse.ts@@ -17,14 +17,10 @@ export const initializeClickhouse = async () => {user_id String,hostname String,pathname String,- querystring String,+ querystring String, /* URL parameters stored in raw format */+ url_parameters Map(String, String), /* Structured storage for all URL parameters */page_title String,referrer String,- utm_source LowCardinality(String),- utm_medium LowCardinality(String),- utm_campaign String,- utm_term String,- utm_content String,channel String,browser LowCardinality(String),browser_version LowCardinality(String),diff --git a/server/src/tracker/pageviewQueue.ts b/server/src/tracker/pageviewQueue.tsindex 24c139be5..ec5dd54e2 100644--- a/server/src/tracker/pageviewQueue.ts+++ b/server/src/tracker/pageviewQueue.ts@@ -1,10 +1,10 @@import { DateTime } from "luxon";import clickhouse from "../db/clickhouse/clickhouse.js";+import { getLocation } from "../db/geolocation/geolocation.js";import { TrackingPayload } from "../types.js";import { getDeviceType } from "../utils.js";import { getChannel } from "./getChannel.js";-import { clearSelfReferrer, getUTMParams } from "./trackingUtils.js";-import { getLocation } from "../db/geolocation/geolocation.js";+import { clearSelfReferrer, getAllUrlParams } from "./trackingUtils.js";type TotalPayload = TrackingPayload & {userId: string;@@ -81,8 +81,8 @@ class PageviewQueue {// Check if referrer is from the same domain and clear it if solet referrer = clearSelfReferrer(pv.referrer || "", pv.hostname || "");- // Get UTM parameters- const utmParams = getUTMParams(pv.querystring || "");+ // Get all URL parameters for the url_parameters map+ const allUrlParams = getAllUrlParams(pv.querystring || "");return {site_id: pv.site_id,@@ -113,11 +113,7 @@ class PageviewQueue {type: pv.type || "pageview",event_name: pv.event_name || "",props: getParsedProperties(pv.properties),- utm_source: utmParams["utm_source"] || "",- utm_medium: utmParams["utm_medium"] || "",- utm_campaign: utmParams["utm_campaign"] || "",- utm_term: utmParams["utm_term"] || "",- utm_content: utmParams["utm_content"] || "",+ url_parameters: allUrlParams,};});diff --git a/server/src/tracker/trackingUtils.ts b/server/src/tracker/trackingUtils.tsindex 86b6f85ba..f86bbd90f 100644--- a/server/src/tracker/trackingUtils.ts+++ b/server/src/tracker/trackingUtils.ts@@ -51,6 +51,30 @@ export function getUTMParams(querystring: string): Record<string, string> {return params;}+// Parse all URL parameters from querystring+export function getAllUrlParams(querystring: string): Record<string, string> {+ const params: Record<string, string> = {};++ if (!querystring) return params;++ // If querystring starts with ?, remove it+ const cleanQuerystring = querystring.startsWith("?")+ ? querystring.substring(1)+ : querystring;++ try {+ const searchParams = new URLSearchParams(cleanQuerystring);+ // Extract all parameters+ for (const [key, value] of searchParams.entries()) {+ params[key.toLowerCase()] = value;+ }+ } catch (e) {+ console.error("Error parsing query string for URL parameters:", e);+ }++ return params;+}+// Clear referrer if it's from the same domainexport function clearSelfReferrer(referrer: string, hostname: string): string {if (!referrer || !hostname) return referrer;