Skip to content

🐸 Rybbit - open-source and privacy-friendly alternative to Google Analytics that is 10x more intuitive.

Refactor analytics to utilize structured URL parameters

Bill Yang611d8ac088b3
diff --git a/server/src/api/analytics/getSessions.ts b/server/src/api/analytics/getSessions.ts
index 99175c445..ad239127d 100644
--- a/server/src/api/analytics/getSessions.ts
+++ b/server/src/api/analytics/getSessions.ts
@@ -79,11 +79,12 @@ export async function getSessions(
argMax(screen_height, timestamp) AS screen_height,
argMin(referrer, timestamp) AS referrer,
argMin(channel, timestamp) AS channel,
- argMin(utm_source, timestamp) AS utm_source,
- argMin(utm_medium, timestamp) AS utm_medium,
- argMin(utm_campaign, timestamp) AS utm_campaign,
- argMin(utm_term, timestamp) AS utm_term,
- argMin(utm_content, timestamp) AS utm_content,
+ /* UTM parameters from url_parameters map */
+ argMin(url_parameters, timestamp)['utm_source'] AS utm_source,
+ argMin(url_parameters, timestamp)['utm_medium'] AS utm_medium,
+ argMin(url_parameters, timestamp)['utm_campaign'] AS utm_campaign,
+ argMin(url_parameters, timestamp)['utm_term'] AS utm_term,
+ argMin(url_parameters, timestamp)['utm_content'] AS utm_content,
MAX(timestamp) AS session_end,
MIN(timestamp) AS session_start,
dateDiff('second', MIN(timestamp), MAX(timestamp)) AS session_duration,
diff --git a/server/src/api/analytics/getSingleCol.ts b/server/src/api/analytics/getSingleCol.ts
index 7095280ea..0505ba470 100644
--- a/server/src/api/analytics/getSingleCol.ts
+++ b/server/src/api/analytics/getSingleCol.ts
@@ -1,14 +1,13 @@
import { FastifyReply, FastifyRequest } from "fastify";
import clickhouse from "../../db/clickhouse/clickhouse.js";
+import { getUserHasAccessToSitePublic } from "../../lib/auth-utils.js";
+import { FilterParameter } from "./types.js";
import {
- geSqlParam,
getFilterStatement,
+ getSqlParam,
getTimeStatement,
processResults,
} from "./utils.js";
-import { getUserHasAccessToSitePublic } from "../../lib/auth-utils.js";
-import { FilterParameter } from "./types.js";
-import SqlString from "sqlstring";
interface GenericRequest {
Params: {
@@ -209,14 +208,14 @@ const getQuery = (request: FastifyRequest<GenericRequest>) => {
return `
WITH PageStats AS (
SELECT
- ${geSqlParam(parameter)} as value,
+ ${getSqlParam(parameter)} as value,
COUNT(distinct(session_id)) as unique_sessions,
COUNT() as pageviews
FROM events
WHERE
site_id = {siteId:Int32}
- AND ${geSqlParam(parameter)} IS NOT NULL
- AND ${geSqlParam(parameter)} <> ''
+ AND ${getSqlParam(parameter)} IS NOT NULL
+ AND ${getSqlParam(parameter)} <> ''
${filterStatement}
${timeStatement}
// AND type = 'pageview'
diff --git a/server/src/api/analytics/utils.ts b/server/src/api/analytics/utils.ts
index 37e17431b..064e561f4 100644
--- a/server/src/api/analytics/utils.ts
+++ b/server/src/api/analytics/utils.ts
@@ -85,7 +85,19 @@ const filterTypeToOperator = (type: FilterType) => {
}
};
-export const geSqlParam = (parameter: FilterParameter) => {
+export const getSqlParam = (parameter: FilterParameter) => {
+ // Handle URL parameters through the url_parameters map
+ if (parameter.startsWith("utm_") || parameter.startsWith("url_param:")) {
+ // For explicit url_param: prefix (e.g., url_param:campaign_id)
+ if (parameter.startsWith("url_param:")) {
+ const paramName = parameter.substring("url_param:".length);
+ return `url_parameters['${paramName}']`;
+ }
+
+ const utm = parameter; // e.g., utm_source, utm_medium, etc.
+ return `url_parameters['${utm}']`;
+ }
+
if (parameter === "referrer") {
return "domainWithoutWWW(referrer)";
}
@@ -197,14 +209,14 @@ export function getFilterStatement(filters: string) {
}
if (filter.value.length === 1) {
- return `${geSqlParam(filter.parameter)} ${filterTypeToOperator(
+ return `${getSqlParam(filter.parameter)} ${filterTypeToOperator(
filter.type
)} ${SqlString.escape(x + filter.value[0] + x)}`;
}
const valuesWithOperator = filter.value.map(
(value) =>
- `${geSqlParam(filter.parameter)} ${filterTypeToOperator(
+ `${getSqlParam(filter.parameter)} ${filterTypeToOperator(
filter.type
)} ${SqlString.escape(x + value + x)}`
);
diff --git a/server/src/db/clickhouse/clickhouse.ts b/server/src/db/clickhouse/clickhouse.ts
index 99b27e9c4..715ad011e 100644
--- a/server/src/db/clickhouse/clickhouse.ts
+++ b/server/src/db/clickhouse/clickhouse.ts
@@ -17,14 +17,10 @@ export const initializeClickhouse = async () => {
user_id String,
hostname String,
pathname String,
- querystring String,
+ querystring String, /* URL parameters stored in raw format */
+ url_parameters Map(String, String), /* Structured storage for all URL parameters */
page_title String,
referrer String,
- utm_source LowCardinality(String),
- utm_medium LowCardinality(String),
- utm_campaign String,
- utm_term String,
- utm_content String,
channel String,
browser LowCardinality(String),
browser_version LowCardinality(String),
diff --git a/server/src/tracker/pageviewQueue.ts b/server/src/tracker/pageviewQueue.ts
index 24c139be5..ec5dd54e2 100644
--- a/server/src/tracker/pageviewQueue.ts
+++ b/server/src/tracker/pageviewQueue.ts
@@ -1,10 +1,10 @@
import { DateTime } from "luxon";
import clickhouse from "../db/clickhouse/clickhouse.js";
+import { getLocation } from "../db/geolocation/geolocation.js";
import { TrackingPayload } from "../types.js";
import { getDeviceType } from "../utils.js";
import { getChannel } from "./getChannel.js";
-import { clearSelfReferrer, getUTMParams } from "./trackingUtils.js";
-import { getLocation } from "../db/geolocation/geolocation.js";
+import { clearSelfReferrer, getAllUrlParams } from "./trackingUtils.js";
type TotalPayload = TrackingPayload & {
userId: string;
@@ -81,8 +81,8 @@ class PageviewQueue {
// Check if referrer is from the same domain and clear it if so
let referrer = clearSelfReferrer(pv.referrer || "", pv.hostname || "");
- // Get UTM parameters
- const utmParams = getUTMParams(pv.querystring || "");
+ // Get all URL parameters for the url_parameters map
+ const allUrlParams = getAllUrlParams(pv.querystring || "");
return {
site_id: pv.site_id,
@@ -113,11 +113,7 @@ class PageviewQueue {
type: pv.type || "pageview",
event_name: pv.event_name || "",
props: getParsedProperties(pv.properties),
- utm_source: utmParams["utm_source"] || "",
- utm_medium: utmParams["utm_medium"] || "",
- utm_campaign: utmParams["utm_campaign"] || "",
- utm_term: utmParams["utm_term"] || "",
- utm_content: utmParams["utm_content"] || "",
+ url_parameters: allUrlParams,
};
});
diff --git a/server/src/tracker/trackingUtils.ts b/server/src/tracker/trackingUtils.ts
index 86b6f85ba..f86bbd90f 100644
--- a/server/src/tracker/trackingUtils.ts
+++ b/server/src/tracker/trackingUtils.ts
@@ -51,6 +51,30 @@ export function getUTMParams(querystring: string): Record<string, string> {
return params;
}
+// Parse all URL parameters from querystring
+export function getAllUrlParams(querystring: string): Record<string, string> {
+ const params: Record<string, string> = {};
+
+ if (!querystring) return params;
+
+ // If querystring starts with ?, remove it
+ const cleanQuerystring = querystring.startsWith("?")
+ ? querystring.substring(1)
+ : querystring;
+
+ try {
+ const searchParams = new URLSearchParams(cleanQuerystring);
+ // Extract all parameters
+ for (const [key, value] of searchParams.entries()) {
+ params[key.toLowerCase()] = value;
+ }
+ } catch (e) {
+ console.error("Error parsing query string for URL parameters:", e);
+ }
+
+ return params;
+}
+
// Clear referrer if it's from the same domain
export function clearSelfReferrer(referrer: string, hostname: string): string {
if (!referrer || !hostname) return referrer;