Skip to content

🐸 Rybbit - open-source and privacy-friendly alternative to Google Analytics that is 10x more intuitive.

Refactor analytics data structure to replace 'iso_3166_2' with 'region' (#92)

Bill Yanga915489fdaf7
diff --git a/client/src/api/analytics/userInfo.ts b/client/src/api/analytics/userInfo.ts
index bed61a960..5281730cd 100644
--- a/client/src/api/analytics/userInfo.ts
+++ b/client/src/api/analytics/userInfo.ts
@@ -6,7 +6,7 @@ export type UserInfo = {
duration: number;
sessions: number;
country: string;
- iso_3166_2: string;
+ region: string;
language: string;
device_type: string;
browser: string;
diff --git a/client/src/api/analytics/userSessions.ts b/client/src/api/analytics/userSessions.ts
index c579aec18..0cea97c5c 100644
--- a/client/src/api/analytics/userSessions.ts
+++ b/client/src/api/analytics/userSessions.ts
@@ -50,7 +50,7 @@ export type GetSessionsResponse = {
user_id: string;
country: string;
city: string;
- iso_3166_2: string;
+ region: string;
language: string;
device_type: string;
browser: string;
@@ -104,7 +104,7 @@ export interface SessionDetails {
session_id: string;
user_id: string;
country: string;
- iso_3166_2: string;
+ region: string;
language: string;
device_type: string;
browser: string;
diff --git a/client/src/api/analytics/users.ts b/client/src/api/analytics/users.ts
index 076968483..9fb86e5a3 100644
--- a/client/src/api/analytics/users.ts
+++ b/client/src/api/analytics/users.ts
@@ -12,7 +12,7 @@ import { APIResponse } from "../types";
export type UsersResponse = {
user_id: string;
country: string;
- iso_3166_2: string;
+ region: string;
city: string;
language: string;
browser: string;
diff --git a/client/src/app/[site]/components/shared/Filters/ValueSelect.tsx b/client/src/app/[site]/components/shared/Filters/ValueSelect.tsx
index f8743274b..0ceaed996 100644
--- a/client/src/app/[site]/components/shared/Filters/ValueSelect.tsx
+++ b/client/src/app/[site]/components/shared/Filters/ValueSelect.tsx
@@ -39,7 +39,7 @@ export function ValueSelect({
label: getCountryName(val),
};
}
- if (parameter === "iso_3166_2") {
+ if (parameter === "region") {
return {
value: val,
label: getRegionName(val),
@@ -64,7 +64,7 @@ export function ValueSelect({
if (parameter === "country") {
return { value: val, label: getCountryName(val) };
}
- if (parameter === "iso_3166_2") {
+ if (parameter === "region") {
return { value: val, label: getRegionName(val) };
}
return { value: val, label: val };
diff --git a/client/src/app/[site]/components/shared/Filters/const.tsx b/client/src/app/[site]/components/shared/Filters/const.tsx
index b66e7793d..582914698 100644
--- a/client/src/app/[site]/components/shared/Filters/const.tsx
+++ b/client/src/app/[site]/components/shared/Filters/const.tsx
@@ -1,32 +1,23 @@
-import React from "react";
-import { FilterParameter } from "../../../../../lib/store";
import {
- Globe,
- Map,
+ AppWindow,
+ Brain,
FileText,
- Search,
- ArrowUpRight,
- Radio,
- Flag,
- MapPin,
- Building,
- Smartphone,
- Laptop,
- Chrome,
+ FolderInput,
+ Globe,
Languages,
LogIn,
LogOut,
- Maximize,
- Zap,
MapPinHouse,
MapPinned,
+ Maximize,
+ MousePointerClick,
+ Radio,
+ Search,
Share2,
TabletSmartphone,
- MousePointerClick,
- FolderInput,
- AppWindow,
- Brain,
} from "lucide-react";
+import React from "react";
+import { FilterParameter } from "../../../../../lib/store";
export const FilterOptions: {
label: string;
@@ -80,7 +71,7 @@ export const FilterOptions: {
},
{
label: "Region",
- value: "iso_3166_2",
+ value: "region",
icon: <MapPinned className="h-4 w-4" />,
},
{
diff --git a/client/src/app/[site]/components/shared/Filters/utils.ts b/client/src/app/[site]/components/shared/Filters/utils.ts
index 693b5ea06..3d998794d 100644
--- a/client/src/app/[site]/components/shared/Filters/utils.ts
+++ b/client/src/app/[site]/components/shared/Filters/utils.ts
@@ -25,8 +25,6 @@ export function getParameterNameLabel(parameter: FilterParameter) {
return "City";
case "region":
return "Region";
- case "iso_3166_2":
- return "Region";
case "entry_page":
return "Entry Page";
case "exit_page":
@@ -57,13 +55,13 @@ export const filterTypeToLabel = (type: FilterType) => {
export function getParameterValueLabel(
filter: Filter,
- getRegionName: (iso_3166_2: string) => string | undefined
+ getRegionName: (region: string) => string | undefined
) {
const formatValue = (value: string) => {
if (filter.parameter === "country") {
return getCountryName(value);
}
- if (filter.parameter === "iso_3166_2") {
+ if (filter.parameter === "region") {
return getRegionName(value);
}
return value;
diff --git a/client/src/app/[site]/main/components/sections/Countries.tsx b/client/src/app/[site]/main/components/sections/Countries.tsx
index 9c575982e..993ca919f 100644
--- a/client/src/app/[site]/main/components/sections/Countries.tsx
+++ b/client/src/app/[site]/main/components/sections/Countries.tsx
@@ -88,7 +88,7 @@ export function Countries({ isRealtime = false }: { isRealtime?: boolean }) {
</TabsContent>
<TabsContent value="regions">
<ComponentToUse
- filterParameter="iso_3166_2"
+ filterParameter="region"
title="Regions"
getValue={(e) => e.value}
getKey={(e) => e.value}
diff --git a/client/src/app/[site]/main/components/sections/Map.tsx b/client/src/app/[site]/main/components/sections/Map.tsx
index f633038dd..650225a44 100644
--- a/client/src/app/[site]/main/components/sections/Map.tsx
+++ b/client/src/app/[site]/main/components/sections/Map.tsx
@@ -1,12 +1,5 @@
"use client";
-import {
- Card,
- CardContent,
- CardHeader,
- CardLoader,
- CardTitle,
-} from "@/components/ui/card";
import * as CountryFlags from "country-flag-icons/react/3x2";
import { scaleLinear } from "d3-scale";
import React, { useEffect, useMemo, useState } from "react";
@@ -36,7 +29,7 @@ export function MapComponent() {
});
const { data: subdivisionData, isLoading: isSubdivisionLoading } =
useSingleCol({
- parameter: "iso_3166_2",
+ parameter: "region",
});
const [dataVersion, setDataVersion] = useState<number>(0);
@@ -152,7 +145,7 @@ export function MapComponent() {
// Set filter based on the current map view
const filterParameter: FilterParameter = isCountryView
? "country"
- : "iso_3166_2";
+ : "region";
addFilter({
parameter: filterParameter,
diff --git a/client/src/app/[site]/user/[userId]/page.tsx b/client/src/app/[site]/user/[userId]/page.tsx
index 37ee1f42f..5ecf382df 100644
--- a/client/src/app/[site]/user/[userId]/page.tsx
+++ b/client/src/app/[site]/user/[userId]/page.tsx
@@ -70,7 +70,7 @@ export default function UserPage() {
</div>
<span className=" text-neutral-100">Region:</span>
<div className="text-neutral-300">
- {data?.iso_3166_2 ? getRegionName(data.iso_3166_2) : "N/A"}
+ {data?.region ? getRegionName(data.region) : "N/A"}
</div>
</div>
<div className="grid grid-cols-[110px_1fr] gap-2">
diff --git a/client/src/components/Sessions/SessionDetails.tsx b/client/src/components/Sessions/SessionDetails.tsx
index c9365cfb2..70b39528c 100644
--- a/client/src/components/Sessions/SessionDetails.tsx
+++ b/client/src/components/Sessions/SessionDetails.tsx
@@ -428,8 +428,8 @@ export function SessionDetails({ session, userId }: SessionDetailsProps) {
<div className="flex items-center gap-1 text-gray-400">
<CountryFlag country={sessionDetails.country} />
<span>{getCountryName(sessionDetails.country)}</span>
- {sessionDetails.iso_3166_2 && (
- <span>({sessionDetails.iso_3166_2})</span>
+ {sessionDetails.region && (
+ <span>({sessionDetails.region})</span>
)}
</div>
</div>
diff --git a/client/src/lib/geo.ts b/client/src/lib/geo.ts
index e1c3f1a0d..20fb6aa8a 100644
--- a/client/src/lib/geo.ts
+++ b/client/src/lib/geo.ts
@@ -55,9 +55,9 @@ export const useGetRegionName = () => {
const { data: subdivisions } = useSubdivisions();
return {
- getRegionName: (iso_3166_2: string) => {
+ getRegionName: (region: string) => {
return subdivisions?.features.find(
- (feature) => feature.properties.iso_3166_2 === iso_3166_2
+ (feature) => feature.properties.iso_3166_2 === region
)?.properties.name;
},
};
diff --git a/client/src/lib/store.ts b/client/src/lib/store.ts
index 37a728c0f..d058e60b4 100644
--- a/client/src/lib/store.ts
+++ b/client/src/lib/store.ts
@@ -35,7 +35,6 @@ export type FilterParameter =
| "pathname"
| "page_title"
| "querystring"
- | "iso_3166_2"
| "event_name"
| "channel"
// derivative parameters
@@ -52,7 +51,6 @@ export const SESSION_PAGE_FILTERS: FilterParameter[] = [
"city",
"device_type",
"referrer",
- "iso_3166_2",
"event_name",
"channel",
"entry_page",
@@ -68,7 +66,6 @@ export const USER_PAGE_FILTERS: FilterParameter[] = [
"city",
"device_type",
"referrer",
- "iso_3166_2",
];
export type Filter = {
diff --git a/server/src/api/analytics/getSession.ts b/server/src/api/analytics/getSession.ts
index b4c799a81..8e72d9217 100644
--- a/server/src/api/analytics/getSession.ts
+++ b/server/src/api/analytics/getSession.ts
@@ -7,7 +7,7 @@ export interface SessionDetails {
session_id: string;
user_id: string;
country: string;
- iso_3166_2: string;
+ region: string;
language: string;
device_type: string;
browser: string;
@@ -80,7 +80,7 @@ SELECT
session_id,
user_id,
country,
- iso_3166_2,
+ region,
language,
device_type,
browser,
diff --git a/server/src/api/analytics/getSessions.ts b/server/src/api/analytics/getSessions.ts
index c1acd3a14..e3514b29d 100644
--- a/server/src/api/analytics/getSessions.ts
+++ b/server/src/api/analytics/getSessions.ts
@@ -11,7 +11,7 @@ export type GetSessionsResponse = {
session_id: string;
user_id: string;
country: string;
- iso_3166_2: string;
+ region: string;
city: string;
language: string;
device_type: string;
@@ -64,7 +64,7 @@ export async function getSessions(
session_id,
user_id,
argMax(country, timestamp) AS country,
- argMax(iso_3166_2, timestamp) AS iso_3166_2,
+ argMax(region, timestamp) AS region,
argMax(city, timestamp) AS city,
argMax(language, timestamp) AS language,
argMax(device_type, timestamp) AS device_type,
diff --git a/server/src/api/analytics/getUserInfo.ts b/server/src/api/analytics/getUserInfo.ts
index 76ab07869..413928d4e 100644
--- a/server/src/api/analytics/getUserInfo.ts
+++ b/server/src/api/analytics/getUserInfo.ts
@@ -6,7 +6,7 @@ interface UserPageviewData {
sessions: number;
duration: number;
country: string;
- iso_3166_2: string;
+ region: string;
language: string;
device_type: string;
browser: string;
@@ -40,7 +40,7 @@ export async function getUserInfo(
session_id,
user_id,
argMax(country, timestamp) AS country,
- argMax(iso_3166_2, timestamp) AS iso_3166_2,
+ argMax(region, timestamp) AS region,
argMax(language, timestamp) AS language,
argMax(device_type, timestamp) AS device_type,
argMax(browser, timestamp) AS browser,
@@ -72,7 +72,7 @@ export async function getUserInfo(
COUNT(DISTINCT session_id) AS sessions,
ROUND(avg(session_duration)) AS duration,
any(country) as country,
- any(iso_3166_2) AS iso_3166_2,
+ any(region) AS region,
any(language) AS language,
any(device_type) AS device_type,
any(browser) AS browser,
diff --git a/server/src/api/analytics/getUserSessions.ts b/server/src/api/analytics/getUserSessions.ts
index 22f92ee79..e74a3495a 100644
--- a/server/src/api/analytics/getUserSessions.ts
+++ b/server/src/api/analytics/getUserSessions.ts
@@ -79,7 +79,7 @@ SELECT
operating_system,
device_type,
country,
- iso_3166_2
+ region
FROM events
WHERE
site_id = ${site}
diff --git a/server/src/api/analytics/getUsers.ts b/server/src/api/analytics/getUsers.ts
index b9df01ecd..6bbe65c58 100644
--- a/server/src/api/analytics/getUsers.ts
+++ b/server/src/api/analytics/getUsers.ts
@@ -10,7 +10,7 @@ import { getUserHasAccessToSitePublic } from "../../lib/auth-utils.js";
export type GetUsersResponse = {
user_id: string;
country: string;
- iso_3166_2: string;
+ region: string;
city: string;
language: string;
browser: string;
@@ -86,7 +86,7 @@ WITH AggregatedUsers AS (
SELECT
user_id,
argMax(country, timestamp) AS country,
- argMax(iso_3166_2, timestamp) AS iso_3166_2,
+ argMax(region, timestamp) AS region,
argMax(city, timestamp) AS city,
argMax(language, timestamp) AS language,
argMax(browser, timestamp) AS browser,
diff --git a/server/src/api/analytics/types.ts b/server/src/api/analytics/types.ts
index d9ca13e82..577e9fae0 100644
--- a/server/src/api/analytics/types.ts
+++ b/server/src/api/analytics/types.ts
@@ -12,7 +12,6 @@ export type FilterParameter =
| "pathname"
| "page_title"
| "querystring"
- | "iso_3166_2"
| "event_name"
| "channel"
// derivative parameters
diff --git a/server/src/db/clickhouse/clickhouse.ts b/server/src/db/clickhouse/clickhouse.ts
index dbc8c3ff5..8129406ba 100644
--- a/server/src/db/clickhouse/clickhouse.ts
+++ b/server/src/db/clickhouse/clickhouse.ts
@@ -32,7 +32,7 @@ export const initializeClickhouse = async () => {
operating_system_version LowCardinality(String),
language LowCardinality(String),
country LowCardinality(FixedString(2)),
- iso_3166_2 LowCardinality(String),
+ region LowCardinality(String),
city String,
latitude Float64,
longitude Float64,
@@ -49,114 +49,74 @@ export const initializeClickhouse = async () => {
`,
});
- // // Create sessions table
- // await clickhouse.exec({
- // query: `
- // CREATE TABLE IF NOT EXISTS sessions (
- // site_id UInt16,
- // start_time DateTime,
- // end_time DateTime,
- // session_id String,
- // user_id String,
- // hostname String,
- // pathname String,
- // querystring String,
- // page_title String,
- // referrer String,
- // browser LowCardinality(String),
- // browser_version LowCardinality(String),
- // operating_system LowCardinality(String),
- // operating_system_version LowCardinality(String),
- // language LowCardinality(String),
- // country LowCardinality(FixedString(2)),
- // iso_3166_2 LowCardinality(String),
- // screen_width UInt16,
- // screen_height UInt16,
- // device_type LowCardinality(String),
- // entry_page String,
- // exit_page String,
- // pageviews UInt32,
- // events UInt32,
- // utm_source String,
- // utm_medium String,
- // utm_campaign String,
- // utm_term String,
- // utm_content String
- // )
- // ENGINE = MergeTree()
- // PARTITION BY toYYYYMM(start_time)
- // ORDER BY (site_id, start_time)
- // `,
- // });
-
- // await clickhouse.exec({
- // query: `
- // CREATE TABLE IF NOT EXISTS sessions
- // (
- // site_id UInt16,
- // session_id String,
- // session_start DateTime,
- // session_end DateTime,
- // user_id String,
- // pageviews UInt32,
- // entry_page String,
- // exit_page String,
+ await clickhouse.exec({
+ query: `
+ CREATE TABLE IF NOT EXISTS sessions
+ (
+ site_id UInt16,
+ session_id String,
+ session_start DateTime,
+ session_end DateTime,
+ user_id String,
+ pageviews UInt32,
+ entry_page String,
+ exit_page String,
- // hostname String,
- // referrer String,
- // browser LowCardinality(String),
- // browser_version LowCardinality(String),
- // operating_system LowCardinality(String),
- // operating_system_version LowCardinality(String),
- // language LowCardinality(String),
- // country LowCardinality(FixedString(2)),
- // iso_3166_2 LowCardinality(String),
- // screen_width UInt16,
- // screen_height UInt16,
- // device_type LowCardinality(String),
- // -- Version column for ReplacingMergeTree
- // version UInt64
- // )
- // ENGINE = ReplacingMergeTree(version)
- // PARTITION BY toYYYYMM(session_start)
- // ORDER BY (site_id, session_id);
- // `,
- // });
- // await clickhouse.exec({
- // query: `
- // CREATE MATERIALIZED VIEW IF NOT EXISTS sessions_mv
- // TO sessions
- // AS
- // SELECT
- // site_id,
- // session_id,
- // min(timestamp) AS session_start,
- // max(timestamp) AS session_end,
- // any(user_id) AS user_id,
- // countIf(type = 'pageview') AS pageviews,
- // argMinIf(pathname, timestamp, type = 'pageview') AS entry_page,
- // argMaxIf(pathname, timestamp, type = 'pageview') AS exit_page,
+ hostname String,
+ referrer String,
+ browser LowCardinality(String),
+ browser_version LowCardinality(String),
+ operating_system LowCardinality(String),
+ operating_system_version LowCardinality(String),
+ language LowCardinality(String),
+ country LowCardinality(FixedString(2)),
+ region LowCardinality(String),
+ screen_width UInt16,
+ screen_height UInt16,
+ device_type LowCardinality(String),
+ -- Version column for ReplacingMergeTree
+ version UInt64
+ )
+ ENGINE = ReplacingMergeTree(version)
+ PARTITION BY toYYYYMM(session_start)
+ ORDER BY (site_id, session_id);
+ `,
+ });
+ await clickhouse.exec({
+ query: `
+ CREATE MATERIALIZED VIEW IF NOT EXISTS sessions_mv
+ TO sessions
+ AS
+ SELECT
+ site_id,
+ session_id,
+ min(timestamp) AS session_start,
+ max(timestamp) AS session_end,
+ any(user_id) AS user_id,
+ countIf(type = 'pageview') AS pageviews,
+ argMinIf(pathname, timestamp, type = 'pageview') AS entry_page,
+ argMaxIf(pathname, timestamp, type = 'pageview') AS exit_page,
- // any(hostname) AS hostname,
- // any(referrer) AS referrer,
- // any(browser) AS browser,
- // any(browser_version) AS browser_version,
- // any(operating_system) AS operating_system,
- // any(operating_system_version) AS operating_system_version,
- // any(language) AS language,
- // any(country) AS country,
- // any(iso_3166_2) AS iso_3166_2,
- // any(screen_width) AS screen_width,
- // any(screen_height) AS screen_height,
- // any(device_type) AS device_type,
- // -- Use the largest timestamp as the 'version'
- // max(toUInt64(timestamp)) AS version
- // FROM events
- // GROUP BY
- // site_id,
- // session_id;
- // `,
- // });
+ any(hostname) AS hostname,
+ any(referrer) AS referrer,
+ any(browser) AS browser,
+ any(browser_version) AS browser_version,
+ any(operating_system) AS operating_system,
+ any(operating_system_version) AS operating_system_version,
+ any(language) AS language,
+ any(country) AS country,
+ any(region) AS region,
+ any(screen_width) AS screen_width,
+ any(screen_height) AS screen_height,
+ any(device_type) AS device_type,
+ -- Use the largest timestamp as the 'version'
+ max(toUInt64(timestamp)) AS version
+ FROM events
+ GROUP BY
+ site_id,
+ session_id;
+ `,
+ });
};
export default clickhouse;
diff --git a/server/src/tracker/pageviewQueue.ts b/server/src/tracker/pageviewQueue.ts
index 0d3cc57c4..fecf0e4af 100644
--- a/server/src/tracker/pageviewQueue.ts
+++ b/server/src/tracker/pageviewQueue.ts
@@ -96,8 +96,7 @@ class PageviewQueue {
screen_height: pv.screenHeight || 0,
device_type: getDeviceType(pv.screenWidth, pv.screenHeight, pv.ua),
country: countryCode,
- iso_3166_2:
- countryCode && regionCode ? countryCode + "-" + regionCode : "",
+ region: countryCode && regionCode ? countryCode + "-" + regionCode : "",
city: city || "",
lat: latitude || 0,
lon: longitude || 0,