🐸 Rybbit - open-source and privacy-friendly alternative to Google Analytics that is 10x more intuitive.
Refactor analytics data structure to replace 'iso_3166_2' with 'region' (#92)
Bill Yanga915489fdaf7
diff --git a/client/src/api/analytics/userInfo.ts b/client/src/api/analytics/userInfo.tsindex bed61a960..5281730cd 100644--- a/client/src/api/analytics/userInfo.ts+++ b/client/src/api/analytics/userInfo.ts@@ -6,7 +6,7 @@ export type UserInfo = {duration: number;sessions: number;country: string;- iso_3166_2: string;+ region: string;language: string;device_type: string;browser: string;diff --git a/client/src/api/analytics/userSessions.ts b/client/src/api/analytics/userSessions.tsindex c579aec18..0cea97c5c 100644--- a/client/src/api/analytics/userSessions.ts+++ b/client/src/api/analytics/userSessions.ts@@ -50,7 +50,7 @@ export type GetSessionsResponse = {user_id: string;country: string;city: string;- iso_3166_2: string;+ region: string;language: string;device_type: string;browser: string;@@ -104,7 +104,7 @@ export interface SessionDetails {session_id: string;user_id: string;country: string;- iso_3166_2: string;+ region: string;language: string;device_type: string;browser: string;diff --git a/client/src/api/analytics/users.ts b/client/src/api/analytics/users.tsindex 076968483..9fb86e5a3 100644--- a/client/src/api/analytics/users.ts+++ b/client/src/api/analytics/users.ts@@ -12,7 +12,7 @@ import { APIResponse } from "../types";export type UsersResponse = {user_id: string;country: string;- iso_3166_2: string;+ region: string;city: string;language: string;browser: string;diff --git a/client/src/app/[site]/components/shared/Filters/ValueSelect.tsx b/client/src/app/[site]/components/shared/Filters/ValueSelect.tsxindex f8743274b..0ceaed996 100644--- a/client/src/app/[site]/components/shared/Filters/ValueSelect.tsx+++ b/client/src/app/[site]/components/shared/Filters/ValueSelect.tsx@@ -39,7 +39,7 @@ export function ValueSelect({label: getCountryName(val),};}- if (parameter === "iso_3166_2") {+ if (parameter === "region") {return {value: val,label: getRegionName(val),@@ -64,7 +64,7 @@ export function ValueSelect({if (parameter === "country") {return { value: val, label: getCountryName(val) };}- if (parameter === "iso_3166_2") {+ if (parameter === "region") {return { value: val, label: getRegionName(val) };}return { value: val, label: val };diff --git a/client/src/app/[site]/components/shared/Filters/const.tsx b/client/src/app/[site]/components/shared/Filters/const.tsxindex b66e7793d..582914698 100644--- a/client/src/app/[site]/components/shared/Filters/const.tsx+++ b/client/src/app/[site]/components/shared/Filters/const.tsx@@ -1,32 +1,23 @@-import React from "react";-import { FilterParameter } from "../../../../../lib/store";import {- Globe,- Map,+ AppWindow,+ Brain,FileText,- Search,- ArrowUpRight,- Radio,- Flag,- MapPin,- Building,- Smartphone,- Laptop,- Chrome,+ FolderInput,+ Globe,Languages,LogIn,LogOut,- Maximize,- Zap,MapPinHouse,MapPinned,+ Maximize,+ MousePointerClick,+ Radio,+ Search,Share2,TabletSmartphone,- MousePointerClick,- FolderInput,- AppWindow,- Brain,} from "lucide-react";+import React from "react";+import { FilterParameter } from "../../../../../lib/store";export const FilterOptions: {label: string;@@ -80,7 +71,7 @@ export const FilterOptions: {},{label: "Region",- value: "iso_3166_2",+ value: "region",icon: <MapPinned className="h-4 w-4" />,},{diff --git a/client/src/app/[site]/components/shared/Filters/utils.ts b/client/src/app/[site]/components/shared/Filters/utils.tsindex 693b5ea06..3d998794d 100644--- a/client/src/app/[site]/components/shared/Filters/utils.ts+++ b/client/src/app/[site]/components/shared/Filters/utils.ts@@ -25,8 +25,6 @@ export function getParameterNameLabel(parameter: FilterParameter) {return "City";case "region":return "Region";- case "iso_3166_2":- return "Region";case "entry_page":return "Entry Page";case "exit_page":@@ -57,13 +55,13 @@ export const filterTypeToLabel = (type: FilterType) => {export function getParameterValueLabel(filter: Filter,- getRegionName: (iso_3166_2: string) => string | undefined+ getRegionName: (region: string) => string | undefined) {const formatValue = (value: string) => {if (filter.parameter === "country") {return getCountryName(value);}- if (filter.parameter === "iso_3166_2") {+ if (filter.parameter === "region") {return getRegionName(value);}return value;diff --git a/client/src/app/[site]/main/components/sections/Countries.tsx b/client/src/app/[site]/main/components/sections/Countries.tsxindex 9c575982e..993ca919f 100644--- a/client/src/app/[site]/main/components/sections/Countries.tsx+++ b/client/src/app/[site]/main/components/sections/Countries.tsx@@ -88,7 +88,7 @@ export function Countries({ isRealtime = false }: { isRealtime?: boolean }) {</TabsContent><TabsContent value="regions"><ComponentToUse- filterParameter="iso_3166_2"+ filterParameter="region"title="Regions"getValue={(e) => e.value}getKey={(e) => e.value}diff --git a/client/src/app/[site]/main/components/sections/Map.tsx b/client/src/app/[site]/main/components/sections/Map.tsxindex f633038dd..650225a44 100644--- a/client/src/app/[site]/main/components/sections/Map.tsx+++ b/client/src/app/[site]/main/components/sections/Map.tsx@@ -1,12 +1,5 @@"use client";-import {- Card,- CardContent,- CardHeader,- CardLoader,- CardTitle,-} from "@/components/ui/card";import * as CountryFlags from "country-flag-icons/react/3x2";import { scaleLinear } from "d3-scale";import React, { useEffect, useMemo, useState } from "react";@@ -36,7 +29,7 @@ export function MapComponent() {});const { data: subdivisionData, isLoading: isSubdivisionLoading } =useSingleCol({- parameter: "iso_3166_2",+ parameter: "region",});const [dataVersion, setDataVersion] = useState<number>(0);@@ -152,7 +145,7 @@ export function MapComponent() {// Set filter based on the current map viewconst filterParameter: FilterParameter = isCountryView? "country"- : "iso_3166_2";+ : "region";addFilter({parameter: filterParameter,diff --git a/client/src/app/[site]/user/[userId]/page.tsx b/client/src/app/[site]/user/[userId]/page.tsxindex 37ee1f42f..5ecf382df 100644--- a/client/src/app/[site]/user/[userId]/page.tsx+++ b/client/src/app/[site]/user/[userId]/page.tsx@@ -70,7 +70,7 @@ export default function UserPage() {</div><span className=" text-neutral-100">Region:</span><div className="text-neutral-300">- {data?.iso_3166_2 ? getRegionName(data.iso_3166_2) : "N/A"}+ {data?.region ? getRegionName(data.region) : "N/A"}</div></div><div className="grid grid-cols-[110px_1fr] gap-2">diff --git a/client/src/components/Sessions/SessionDetails.tsx b/client/src/components/Sessions/SessionDetails.tsxindex c9365cfb2..70b39528c 100644--- a/client/src/components/Sessions/SessionDetails.tsx+++ b/client/src/components/Sessions/SessionDetails.tsx@@ -428,8 +428,8 @@ export function SessionDetails({ session, userId }: SessionDetailsProps) {<div className="flex items-center gap-1 text-gray-400"><CountryFlag country={sessionDetails.country} /><span>{getCountryName(sessionDetails.country)}</span>- {sessionDetails.iso_3166_2 && (- <span>({sessionDetails.iso_3166_2})</span>+ {sessionDetails.region && (+ <span>({sessionDetails.region})</span>)}</div></div>diff --git a/client/src/lib/geo.ts b/client/src/lib/geo.tsindex e1c3f1a0d..20fb6aa8a 100644--- a/client/src/lib/geo.ts+++ b/client/src/lib/geo.ts@@ -55,9 +55,9 @@ export const useGetRegionName = () => {const { data: subdivisions } = useSubdivisions();return {- getRegionName: (iso_3166_2: string) => {+ getRegionName: (region: string) => {return subdivisions?.features.find(- (feature) => feature.properties.iso_3166_2 === iso_3166_2+ (feature) => feature.properties.iso_3166_2 === region)?.properties.name;},};diff --git a/client/src/lib/store.ts b/client/src/lib/store.tsindex 37a728c0f..d058e60b4 100644--- a/client/src/lib/store.ts+++ b/client/src/lib/store.ts@@ -35,7 +35,6 @@ export type FilterParameter =| "pathname"| "page_title"| "querystring"- | "iso_3166_2"| "event_name"| "channel"// derivative parameters@@ -52,7 +51,6 @@ export const SESSION_PAGE_FILTERS: FilterParameter[] = ["city","device_type","referrer",- "iso_3166_2","event_name","channel","entry_page",@@ -68,7 +66,6 @@ export const USER_PAGE_FILTERS: FilterParameter[] = ["city","device_type","referrer",- "iso_3166_2",];export type Filter = {diff --git a/server/src/api/analytics/getSession.ts b/server/src/api/analytics/getSession.tsindex b4c799a81..8e72d9217 100644--- a/server/src/api/analytics/getSession.ts+++ b/server/src/api/analytics/getSession.ts@@ -7,7 +7,7 @@ export interface SessionDetails {session_id: string;user_id: string;country: string;- iso_3166_2: string;+ region: string;language: string;device_type: string;browser: string;@@ -80,7 +80,7 @@ SELECTsession_id,user_id,country,- iso_3166_2,+ region,language,device_type,browser,diff --git a/server/src/api/analytics/getSessions.ts b/server/src/api/analytics/getSessions.tsindex c1acd3a14..e3514b29d 100644--- a/server/src/api/analytics/getSessions.ts+++ b/server/src/api/analytics/getSessions.ts@@ -11,7 +11,7 @@ export type GetSessionsResponse = {session_id: string;user_id: string;country: string;- iso_3166_2: string;+ region: string;city: string;language: string;device_type: string;@@ -64,7 +64,7 @@ export async function getSessions(session_id,user_id,argMax(country, timestamp) AS country,- argMax(iso_3166_2, timestamp) AS iso_3166_2,+ argMax(region, timestamp) AS region,argMax(city, timestamp) AS city,argMax(language, timestamp) AS language,argMax(device_type, timestamp) AS device_type,diff --git a/server/src/api/analytics/getUserInfo.ts b/server/src/api/analytics/getUserInfo.tsindex 76ab07869..413928d4e 100644--- a/server/src/api/analytics/getUserInfo.ts+++ b/server/src/api/analytics/getUserInfo.ts@@ -6,7 +6,7 @@ interface UserPageviewData {sessions: number;duration: number;country: string;- iso_3166_2: string;+ region: string;language: string;device_type: string;browser: string;@@ -40,7 +40,7 @@ export async function getUserInfo(session_id,user_id,argMax(country, timestamp) AS country,- argMax(iso_3166_2, timestamp) AS iso_3166_2,+ argMax(region, timestamp) AS region,argMax(language, timestamp) AS language,argMax(device_type, timestamp) AS device_type,argMax(browser, timestamp) AS browser,@@ -72,7 +72,7 @@ export async function getUserInfo(COUNT(DISTINCT session_id) AS sessions,ROUND(avg(session_duration)) AS duration,any(country) as country,- any(iso_3166_2) AS iso_3166_2,+ any(region) AS region,any(language) AS language,any(device_type) AS device_type,any(browser) AS browser,diff --git a/server/src/api/analytics/getUserSessions.ts b/server/src/api/analytics/getUserSessions.tsindex 22f92ee79..e74a3495a 100644--- a/server/src/api/analytics/getUserSessions.ts+++ b/server/src/api/analytics/getUserSessions.ts@@ -79,7 +79,7 @@ SELECToperating_system,device_type,country,- iso_3166_2+ regionFROM eventsWHEREsite_id = ${site}diff --git a/server/src/api/analytics/getUsers.ts b/server/src/api/analytics/getUsers.tsindex b9df01ecd..6bbe65c58 100644--- a/server/src/api/analytics/getUsers.ts+++ b/server/src/api/analytics/getUsers.ts@@ -10,7 +10,7 @@ import { getUserHasAccessToSitePublic } from "../../lib/auth-utils.js";export type GetUsersResponse = {user_id: string;country: string;- iso_3166_2: string;+ region: string;city: string;language: string;browser: string;@@ -86,7 +86,7 @@ WITH AggregatedUsers AS (SELECTuser_id,argMax(country, timestamp) AS country,- argMax(iso_3166_2, timestamp) AS iso_3166_2,+ argMax(region, timestamp) AS region,argMax(city, timestamp) AS city,argMax(language, timestamp) AS language,argMax(browser, timestamp) AS browser,diff --git a/server/src/api/analytics/types.ts b/server/src/api/analytics/types.tsindex d9ca13e82..577e9fae0 100644--- a/server/src/api/analytics/types.ts+++ b/server/src/api/analytics/types.ts@@ -12,7 +12,6 @@ export type FilterParameter =| "pathname"| "page_title"| "querystring"- | "iso_3166_2"| "event_name"| "channel"// derivative parametersdiff --git a/server/src/db/clickhouse/clickhouse.ts b/server/src/db/clickhouse/clickhouse.tsindex dbc8c3ff5..8129406ba 100644--- a/server/src/db/clickhouse/clickhouse.ts+++ b/server/src/db/clickhouse/clickhouse.ts@@ -32,7 +32,7 @@ export const initializeClickhouse = async () => {operating_system_version LowCardinality(String),language LowCardinality(String),country LowCardinality(FixedString(2)),- iso_3166_2 LowCardinality(String),+ region LowCardinality(String),city String,latitude Float64,longitude Float64,@@ -49,114 +49,74 @@ export const initializeClickhouse = async () => {`,});- // // Create sessions table- // await clickhouse.exec({- // query: `- // CREATE TABLE IF NOT EXISTS sessions (- // site_id UInt16,- // start_time DateTime,- // end_time DateTime,- // session_id String,- // user_id String,- // hostname String,- // pathname String,- // querystring String,- // page_title String,- // referrer String,- // browser LowCardinality(String),- // browser_version LowCardinality(String),- // operating_system LowCardinality(String),- // operating_system_version LowCardinality(String),- // language LowCardinality(String),- // country LowCardinality(FixedString(2)),- // iso_3166_2 LowCardinality(String),- // screen_width UInt16,- // screen_height UInt16,- // device_type LowCardinality(String),- // entry_page String,- // exit_page String,- // pageviews UInt32,- // events UInt32,- // utm_source String,- // utm_medium String,- // utm_campaign String,- // utm_term String,- // utm_content String- // )- // ENGINE = MergeTree()- // PARTITION BY toYYYYMM(start_time)- // ORDER BY (site_id, start_time)- // `,- // });-- // await clickhouse.exec({- // query: `- // CREATE TABLE IF NOT EXISTS sessions- // (- // site_id UInt16,- // session_id String,- // session_start DateTime,- // session_end DateTime,- // user_id String,- // pageviews UInt32,- // entry_page String,- // exit_page String,+ await clickhouse.exec({+ query: `+ CREATE TABLE IF NOT EXISTS sessions+ (+ site_id UInt16,+ session_id String,+ session_start DateTime,+ session_end DateTime,+ user_id String,+ pageviews UInt32,+ entry_page String,+ exit_page String,- // hostname String,- // referrer String,- // browser LowCardinality(String),- // browser_version LowCardinality(String),- // operating_system LowCardinality(String),- // operating_system_version LowCardinality(String),- // language LowCardinality(String),- // country LowCardinality(FixedString(2)),- // iso_3166_2 LowCardinality(String),- // screen_width UInt16,- // screen_height UInt16,- // device_type LowCardinality(String),- // -- Version column for ReplacingMergeTree- // version UInt64- // )- // ENGINE = ReplacingMergeTree(version)- // PARTITION BY toYYYYMM(session_start)- // ORDER BY (site_id, session_id);- // `,- // });- // await clickhouse.exec({- // query: `- // CREATE MATERIALIZED VIEW IF NOT EXISTS sessions_mv- // TO sessions- // AS- // SELECT- // site_id,- // session_id,- // min(timestamp) AS session_start,- // max(timestamp) AS session_end,- // any(user_id) AS user_id,- // countIf(type = 'pageview') AS pageviews,- // argMinIf(pathname, timestamp, type = 'pageview') AS entry_page,- // argMaxIf(pathname, timestamp, type = 'pageview') AS exit_page,+ hostname String,+ referrer String,+ browser LowCardinality(String),+ browser_version LowCardinality(String),+ operating_system LowCardinality(String),+ operating_system_version LowCardinality(String),+ language LowCardinality(String),+ country LowCardinality(FixedString(2)),+ region LowCardinality(String),+ screen_width UInt16,+ screen_height UInt16,+ device_type LowCardinality(String),+ -- Version column for ReplacingMergeTree+ version UInt64+ )+ ENGINE = ReplacingMergeTree(version)+ PARTITION BY toYYYYMM(session_start)+ ORDER BY (site_id, session_id);+ `,+ });+ await clickhouse.exec({+ query: `+ CREATE MATERIALIZED VIEW IF NOT EXISTS sessions_mv+ TO sessions+ AS+ SELECT+ site_id,+ session_id,+ min(timestamp) AS session_start,+ max(timestamp) AS session_end,+ any(user_id) AS user_id,+ countIf(type = 'pageview') AS pageviews,+ argMinIf(pathname, timestamp, type = 'pageview') AS entry_page,+ argMaxIf(pathname, timestamp, type = 'pageview') AS exit_page,- // any(hostname) AS hostname,- // any(referrer) AS referrer,- // any(browser) AS browser,- // any(browser_version) AS browser_version,- // any(operating_system) AS operating_system,- // any(operating_system_version) AS operating_system_version,- // any(language) AS language,- // any(country) AS country,- // any(iso_3166_2) AS iso_3166_2,- // any(screen_width) AS screen_width,- // any(screen_height) AS screen_height,- // any(device_type) AS device_type,- // -- Use the largest timestamp as the 'version'- // max(toUInt64(timestamp)) AS version- // FROM events- // GROUP BY- // site_id,- // session_id;- // `,- // });+ any(hostname) AS hostname,+ any(referrer) AS referrer,+ any(browser) AS browser,+ any(browser_version) AS browser_version,+ any(operating_system) AS operating_system,+ any(operating_system_version) AS operating_system_version,+ any(language) AS language,+ any(country) AS country,+ any(region) AS region,+ any(screen_width) AS screen_width,+ any(screen_height) AS screen_height,+ any(device_type) AS device_type,+ -- Use the largest timestamp as the 'version'+ max(toUInt64(timestamp)) AS version+ FROM events+ GROUP BY+ site_id,+ session_id;+ `,+ });};export default clickhouse;diff --git a/server/src/tracker/pageviewQueue.ts b/server/src/tracker/pageviewQueue.tsindex 0d3cc57c4..fecf0e4af 100644--- a/server/src/tracker/pageviewQueue.ts+++ b/server/src/tracker/pageviewQueue.ts@@ -96,8 +96,7 @@ class PageviewQueue {screen_height: pv.screenHeight || 0,device_type: getDeviceType(pv.screenWidth, pv.screenHeight, pv.ua),country: countryCode,- iso_3166_2:- countryCode && regionCode ? countryCode + "-" + regionCode : "",+ region: countryCode && regionCode ? countryCode + "-" + regionCode : "",city: city || "",lat: latitude || 0,lon: longitude || 0,