Skip to content

🐸 Rybbit - open-source and privacy-friendly alternative to Google Analytics that is 10x more intuitive.

Update RetentionChart margin and add channel field to Clickhouse schema

Bill Yangd7bd778bd662
diff --git a/client/src/app/[site]/retention/RetentionChart.tsx b/client/src/app/[site]/retention/RetentionChart.tsx
index d74be483d..d7433c846 100644
--- a/client/src/app/[site]/retention/RetentionChart.tsx
+++ b/client/src/app/[site]/retention/RetentionChart.tsx
@@ -138,7 +138,7 @@ export function RetentionChart({ data, isLoading, mode }: RetentionChartProps) {
<ResponsiveLine
data={chartData}
theme={nivoTheme}
- margin={{ top: 20, right: 100, bottom: 30, left: 40 }}
+ margin={{ top: 20, right: 90, bottom: 30, left: 40 }}
xScale={{
type: "linear",
min: 0,
diff --git a/server/src/db/clickhouse/clickhouse.ts b/server/src/db/clickhouse/clickhouse.ts
index 424ebaedf..f142ea6e5 100644
--- a/server/src/db/clickhouse/clickhouse.ts
+++ b/server/src/db/clickhouse/clickhouse.ts
@@ -21,6 +21,7 @@ export const initializeClickhouse = async () => {
querystring String,
page_title String,
referrer String,
+ channel String,
browser LowCardinality(String),
browser_version LowCardinality(String),
operating_system LowCardinality(String),
diff --git a/server/src/tracker/const.ts b/server/src/tracker/const.ts
new file mode 100644
index 000000000..ab0dc6cdf
--- /dev/null
+++ b/server/src/tracker/const.ts
@@ -0,0 +1,623 @@
+export const searchDomains = [
+ // Google and variants
+ "google.com",
+ "google.",
+
+ // Bing
+ "bing.com",
+ "bing.",
+
+ // Yahoo
+ "yahoo.com",
+ "yahoo.",
+ "search.yahoo.",
+
+ // Other global search engines
+ "duckduckgo.com",
+ "duck.com",
+ "baidu.com",
+ "baidu.",
+ "yandex.com",
+ "yandex.",
+ "ya.ru",
+ "qwant.com",
+ "search.",
+ "ecosia.org",
+ "brave.com",
+ "startpage.com",
+ "searchencrypt.com",
+ "searx.",
+ "swisscows.com",
+ "mojeek.com",
+ "gibiru.com",
+ "metager.org",
+ "search.aol.com",
+ "lycos.com",
+ "wolframalpha.com",
+ "ask.com",
+ "dogpile.com",
+ "webcrawler.com",
+
+ // Regional search engines
+ "naver.com",
+ "daum.net",
+ "seznam.cz",
+ "coccoc.com",
+ "yam.com",
+ "so.com",
+ "sogou.com",
+ "goo.ne.jp",
+ "rambler.ru",
+];
+
+export const socialDomains = [
+ // Major social networks and their variants
+ "facebook.com",
+ "fb.com",
+ "fb.me",
+ "messenger.com",
+ "m.facebook.com",
+ "instagram.com",
+ "instagram.",
+ "ig.me",
+ "twitter.com",
+ "t.co",
+ "x.com",
+ "linkedin.com",
+ "lnkd.in",
+ "tiktok.com",
+ "tiktok.",
+ "vm.tiktok.com",
+ "pinterest.com",
+ "pinterest.",
+ "pin.it",
+ "reddit.com",
+ "redd.it",
+ "old.reddit.com",
+
+ // Other global social platforms
+ "snapchat.com",
+ "snap.com",
+ "youtube.com",
+ "youtu.be",
+ "discord.com",
+ "discord.gg",
+ "whatsapp.com",
+ "wa.me",
+ "telegram.org",
+ "t.me",
+ "medium.com",
+ "tumblr.com",
+ "tmblr.co",
+ "quora.com",
+ "threads.net",
+ "mastodon.social",
+ "mastodon.",
+ "slack.com",
+ "nextdoor.com",
+ "clubhouse.com",
+ "twitch.tv",
+ "news.ycombinator.com",
+ "hn.algolia.com",
+
+ // Regional social platforms
+ "wechat.com",
+ "weixin.qq.com",
+ "vk.com",
+ "qq.com",
+ "weibo.com",
+ "weibo.cn",
+ "line.me",
+];
+
+export const videoDomains = [
+ // Major video platforms
+ "youtube.com",
+ "youtu.be",
+ "vimeo.com",
+ "twitch.tv",
+ "dailymotion.com",
+
+ // Streaming services
+ "netflix.com",
+ "disneyplus.com",
+ "hulu.com",
+ "hbomax.com",
+ "peacocktv.com",
+ "primevideo.com",
+ "amazon.com/prime-video",
+ "paramountplus.com",
+ "discoveryplus.com",
+ "crunchyroll.com",
+ "funimation.com",
+ "curiositystream.com",
+ "mubi.com",
+
+ // Video sharing
+ "tiktok.com",
+ "vm.tiktok.com",
+ "vevo.com",
+ "streamable.com",
+ "bitchute.com",
+ "rumble.com",
+ "odysee.com",
+ "lbry.tv",
+ "tiktok.com",
+ "reels.instagram.com",
+
+ // Educational
+ "ted.com",
+ "khanacademy.org",
+ "skillshare.com",
+ "udemy.com",
+ "masterclass.com",
+ "coursera.org",
+ "pluralsight.com",
+ "lynda.com",
+ "linkedin.com/learning",
+
+ // Live streaming
+ "facebook.com/gaming",
+ "youtube.com/live",
+ "kick.com",
+ "caffeine.tv",
+
+ // Regional platforms
+ "bilibili.com",
+ "niconico.jp",
+ "vlive.tv",
+ "youku.com",
+ "tudou.com",
+ "iqiyi.com",
+];
+
+export const shoppingDomains = [
+ // Major marketplaces
+ "amazon.com",
+ "amazon.",
+ "ebay.com",
+ "ebay.",
+ "etsy.com",
+ "walmart.com",
+ "target.com",
+ "aliexpress.com",
+ "wish.com",
+ "shopify.com",
+ "shop.app",
+
+ // Fashion and apparel
+ "asos.com",
+ "zara.com",
+ "forever21.com",
+ "hm.com",
+ "gap.com",
+ "macys.com",
+ "nordstrom.com",
+ "net-a-porter.com",
+ "farfetch.com",
+ "fashionnova.com",
+
+ // Electronics
+ "bestbuy.com",
+ "newegg.com",
+ "bhphotovideo.com",
+ "apple.com/shop",
+ "samsung.com/shop",
+
+ // Home and furniture
+ "wayfair.com",
+ "ikea.com",
+ "homedepot.com",
+ "lowes.com",
+ "overstock.com",
+
+ // Regional marketplaces
+ "taobao.com",
+ "jd.com",
+ "rakuten.com",
+ "flipkart.com",
+ "lazada.com",
+ "mercadolibre.com",
+ "ozon.ru",
+ "allegro.pl",
+ "coupang.com",
+ "gmarket.co.kr",
+
+ // Grocery and food
+ "instacart.com",
+ "freshdirect.com",
+ "ocado.com",
+ "groceries.",
+ "doordash.com",
+ "ubereats.com",
+ "grubhub.com",
+ "deliveroo.com",
+];
+
+// Define sources types
+export const searchSources = [
+ // Major search engines
+ "google",
+ "bing",
+ "yahoo",
+ "duckduckgo",
+ "baidu",
+ "yandex",
+ "qwant",
+
+ // Other search engines
+ "ecosia",
+ "brave",
+ "startpage",
+ "searchencrypt",
+ "swisscows",
+ "mojeek",
+ "gibiru",
+ "metager",
+ "wolframalpha",
+ "ask",
+ "dogpile",
+ "webcrawler",
+ "aol",
+
+ // Regional search engines
+ "naver",
+ "daum",
+ "seznam",
+ "coccoc",
+ "yam",
+ "so",
+ "sogou",
+ "goo",
+ "rambler",
+];
+export const socialSources = [
+ // Major social platforms
+ "facebook",
+ "twitter",
+ "linkedin",
+ "instagram",
+ "tiktok",
+ "pinterest",
+ "reddit",
+
+ // Other platforms
+ "snapchat",
+ "youtube",
+ "discord",
+ "whatsapp",
+ "telegram",
+ "medium",
+ "tumblr",
+ "quora",
+ "threads",
+ "mastodon",
+ "slack",
+ "nextdoor",
+ "clubhouse",
+ "twitch",
+ "hacker news",
+ "hackernews",
+ "ycombinator",
+
+ // Shortened variants
+ "fb",
+ "ig",
+ "pin",
+ "li",
+ "tw",
+ "yt",
+
+ // Regional platforms
+ "wechat",
+ "weixin",
+ "vk",
+ "qq",
+ "weibo",
+ "line",
+ "kakao",
+];
+export const videoSources = [
+ // Major video platforms
+ "youtube",
+ "vimeo",
+ "twitch",
+ "dailymotion",
+ "tiktok",
+
+ // Streaming services
+ "disneyplus",
+ "netflix",
+ "hulu",
+ "hbomax",
+ "peacock",
+ "prime video",
+ "paramount+",
+ "discovery+",
+ "crunchyroll",
+ "funimation",
+ "curiositystream",
+ "mubi",
+
+ // Video sharing
+ "vevo",
+ "streamable",
+ "bitchute",
+ "rumble",
+ "odysee",
+ "lbry",
+ "reels",
+
+ // Educational
+ "ted",
+ "khanacademy",
+ "skillshare",
+ "udemy",
+ "masterclass",
+ "coursera",
+ "pluralsight",
+ "lynda",
+
+ // Live streaming
+ "kick",
+ "caffeine",
+
+ // Regional platforms
+ "bilibili",
+ "niconico",
+ "vlive",
+ "youku",
+ "tudou",
+ "iqiyi",
+];
+export const shoppingSources = [
+ // Major marketplaces
+ "amazon",
+ "ebay",
+ "etsy",
+ "shopify",
+ "walmart",
+ "target",
+ "aliexpress",
+ "wish",
+ "shop",
+ "store",
+ "shopping",
+
+ // Fashion and apparel
+ "asos",
+ "zara",
+ "forever21",
+ "hm",
+ "gap",
+ "macys",
+ "nordstrom",
+ "net-a-porter",
+ "farfetch",
+ "fashionnova",
+
+ // Electronics
+ "bestbuy",
+ "newegg",
+ "bhphotovideo",
+ "apple",
+ "samsung",
+
+ // Home and furniture
+ "wayfair",
+ "ikea",
+ "homedepot",
+ "lowes",
+ "overstock",
+
+ // Regional marketplaces
+ "taobao",
+ "jd",
+ "rakuten",
+ "flipkart",
+ "lazada",
+ "mercadolibre",
+ "ozon",
+ "allegro",
+ "coupang",
+ "gmarket",
+
+ // Grocery and food
+ "instacart",
+ "freshdirect",
+ "ocado",
+ "groceries",
+ "doordash",
+ "ubereats",
+ "grubhub",
+ "deliveroo",
+];
+export const emailSources = [
+ "email",
+ "e_mail",
+ "e-mail",
+ "mail",
+ "newsletter",
+ "mailchimp",
+ "campaign-archive",
+ "sendgrid",
+ "mailgun",
+ "constantcontact",
+ "klaviyo",
+ "hubspot",
+ "marketo",
+ "brevo",
+ "sendinblue",
+ "getresponse",
+ "activecampaign",
+ "mailerlite",
+ "convertkit",
+ "drip",
+ "gmail",
+ "yahoo",
+ "outlook",
+ "hotmail",
+ "list",
+ "blast",
+ "campaign",
+];
+export const smsSources = [
+ "sms",
+ "text",
+ "twilio",
+ "message",
+ "whatsapp",
+ "viber",
+ "line",
+ "imessage",
+];
+
+// Medium types
+export const socialMediums = [
+ "sm",
+ "social-media",
+ "social-network",
+ "social",
+ "community",
+ "forum",
+ "organic-social",
+ "feed",
+ "share",
+ "repost",
+ "tweet",
+ "post",
+ "update",
+ "engagement",
+ "ugc",
+ "user-generated",
+ "social_post",
+];
+
+export const videoMediums = [
+ "video",
+ "youtube",
+ "vimeo",
+ "streaming",
+ "live",
+ "tv",
+ "ott",
+ "broadcast",
+ "clip",
+ "trailer",
+ "episode",
+ "series",
+ "documentary",
+ "film",
+ "movie",
+ "animation",
+ "video_ad",
+ "pre-roll",
+ "mid-roll",
+ "post-roll",
+];
+
+export const displayMediums = [
+ "display",
+ "interstitial",
+ "banner",
+ "ad",
+ "advert",
+ "advertisement",
+ "rich-media",
+ "popup",
+ "popunder",
+ "overlay",
+ "expandable",
+ "floating",
+ "skin",
+ "wallpaper",
+ "native",
+ "programmatic",
+ "dsp",
+ "retargeting",
+ "remarketing",
+ "impression",
+];
+
+export const affiliateMediums = [
+ "affiliate",
+ "aff",
+ "partner",
+ "partnership",
+ "commission",
+ "rev-share",
+ "performance",
+ "cpa",
+ "lead-gen",
+ "lead-generation",
+];
+
+export const referralMediums = [
+ "referral",
+ "link",
+ "app",
+ "invite",
+ "recommendation",
+ "advocate",
+ "refer-a-friend",
+ "share",
+ "web",
+ "embed",
+ "backlink",
+ "qr",
+ "qr-code",
+ "deeplink",
+ "vanity-url",
+];
+
+export const emailMediums = [
+ "email",
+ "e_mail",
+ "e-mail",
+ "mail",
+ "newsletter",
+ "digest",
+ "bulletin",
+ "marketing_email",
+ "promotional",
+ "transactional",
+ "notification_email",
+ "alert",
+ "confirmation",
+ "update",
+ "news",
+];
+
+export const pushMediums = [
+ "push",
+ "notification",
+ "mobile",
+ "app-notification",
+ "web-notification",
+ "browser-notification",
+ "alert",
+ "toast",
+ "prompt",
+ "pwa",
+ "install",
+];
+
+export const audioMediums = [
+ "audio",
+ "podcast",
+ "radio",
+ "broadcast",
+ "streaming",
+ "music",
+ "playlist",
+ "episode",
+ "show",
+ "airplay",
+ "song",
+ "track",
+ "voice",
+ "audio_ad",
+ "spot",
+ "jingle",
+ "commercial",
+];
diff --git a/server/src/tracker/getChannel.ts b/server/src/tracker/getChannel.ts
new file mode 100644
index 000000000..747e3c01f
--- /dev/null
+++ b/server/src/tracker/getChannel.ts
@@ -0,0 +1,160 @@
+import {
+ affiliateMediums,
+ audioMediums,
+ displayMediums,
+ emailMediums,
+ emailSources,
+ pushMediums,
+ referralMediums,
+ searchDomains,
+ searchSources,
+ shoppingDomains,
+ shoppingSources,
+ smsSources,
+ socialDomains,
+ socialMediums,
+ socialSources,
+ videoDomains,
+ videoMediums,
+ videoSources,
+} from "./const.js";
+
+// UTM and URL parameter parsing utilities
+function getUTMParams(querystring: string): Record<string, string> {
+ const params: Record<string, string> = {};
+
+ if (!querystring) return params;
+
+ try {
+ const searchParams = new URLSearchParams(querystring);
+
+ // Extract UTM parameters
+ for (const [key, value] of searchParams.entries()) {
+ if (key.startsWith("utm_") || key === "gclid" || key === "gad_source") {
+ params[key.toLowerCase()] = value.toLowerCase();
+ }
+ }
+ } catch (e) {
+ console.error("Error parsing query string:", e);
+ }
+
+ return params;
+}
+
+function getDomainFromReferrer(referrer: string): string {
+ if (!referrer) return "$direct";
+
+ try {
+ const url = new URL(referrer);
+ return url.hostname;
+ } catch (e) {
+ // Handle invalid URLs
+ return "$direct";
+ }
+}
+
+export function getChannel(referrer: string, querystring: string): string {
+ const utmParams = getUTMParams(querystring);
+ const referringDomain = getDomainFromReferrer(referrer);
+
+ // UTM parameters
+ const utmSource = utmParams["utm_source"] || "";
+ const utmMedium = utmParams["utm_medium"] || "";
+ const utmCampaign = utmParams["utm_campaign"] || "";
+ const gclid = utmParams["gclid"] || "";
+ const gadSource = utmParams["gad_source"] || "";
+
+ // Check if traffic is paid
+ const isPaid =
+ ["cpc", "cpm", "cpv", "cpa", "ppc", "retargeting"].includes(utmMedium) ||
+ utmMedium.startsWith("paid") ||
+ gclid !== "" ||
+ gadSource !== "";
+
+ // Check domain type - improved to handle subdomains better
+ const isDomainSearch = searchDomains.some(
+ (domain) =>
+ referringDomain.includes(domain) ||
+ /^(?:www\.)?google\.[a-z]{2,3}$/.test(referringDomain)
+ );
+ const isDomainSocial = socialDomains.some((domain) =>
+ referringDomain.includes(domain)
+ );
+ const isDomainVideo = videoDomains.some((domain) =>
+ referringDomain.includes(domain)
+ );
+ const isDomainShopping = shoppingDomains.some((domain) =>
+ referringDomain.includes(domain)
+ );
+
+ // Check source type
+ const isSourceSearch = searchSources.includes(utmSource);
+ const isSourceSocial = socialSources.includes(utmSource);
+ const isSourceVideo = videoSources.includes(utmSource);
+ const isSourceShopping = shoppingSources.includes(utmSource);
+ const isSourceEmail = emailSources.includes(utmSource);
+ const isSourceSMS = smsSources.includes(utmSource);
+
+ // Apply channel detection logic (in order of precedence)
+
+ // Cross Network
+ if (utmCampaign === "cross-network") return "Cross-Network";
+
+ // Paid channels
+ if (isPaid) {
+ if (isSourceSearch || isDomainSearch) return "Paid Search";
+ if (isSourceSocial || isDomainSocial) return "Paid Social";
+ if (isSourceVideo || isDomainVideo) return "Paid Video";
+ if (isSourceShopping || isDomainShopping) return "Paid Shopping";
+ if (socialMediums.includes(utmMedium)) return "Paid Social";
+ if (videoMediums.includes(utmMedium)) return "Paid Video";
+ if (displayMediums.includes(utmMedium)) return "Display";
+ if (gadSource === "1") return "Paid Search";
+ if (/video/.test(utmCampaign)) return "Paid Video";
+ return "Paid Unknown";
+ }
+
+ // Direct
+ if (
+ referringDomain === "$direct" &&
+ !utmMedium &&
+ (!utmSource || utmSource === "direct" || utmSource === "(direct)")
+ ) {
+ return "Direct";
+ }
+
+ // Organic channels - with referrer domain check prioritized for more accurate detection
+ // This is the key part that handles organic search from Google clicks without UTM params
+ if (isDomainSearch) return "Organic Search";
+ if (isDomainSocial) return "Organic Social";
+ if (isDomainVideo) return "Organic Video";
+ if (isDomainShopping) return "Organic Shopping";
+
+ // Then check UTM parameters if referrer didn't match
+ if (isSourceSearch) return "Organic Search";
+ if (isSourceSocial) return "Organic Social";
+ if (isSourceVideo) return "Organic Video";
+ if (isSourceShopping) return "Organic Shopping";
+ if (isSourceEmail || emailMediums.includes(utmMedium)) return "Email";
+ if (isSourceSMS) return "SMS";
+
+ // Medium-based detection
+ if (socialMediums.includes(utmMedium)) return "Organic Social";
+ if (videoMediums.includes(utmMedium)) return "Organic Video";
+ if (affiliateMediums.includes(utmMedium)) return "Affiliate";
+ if (referralMediums.includes(utmMedium)) return "Referral";
+ if (displayMediums.includes(utmMedium)) return "Display";
+ if (audioMediums.includes(utmMedium)) return "Audio";
+ if (pushMediums.includes(utmMedium) || utmMedium.endsWith("push"))
+ return "Push";
+
+ // Campaign-based detection
+ if (/video/.test(utmCampaign)) return "Organic Video";
+ if (/shop|shopping/.test(utmCampaign)) return "Organic Shopping";
+
+ // If referring domain exists but we couldn't categorize it
+ if (referringDomain && referringDomain !== "$direct") return "Referral";
+
+ // Default fallback
+ return "Unknown";
+}
diff --git a/server/src/tracker/pageviewQueue.ts b/server/src/tracker/pageviewQueue.ts
index 69434a881..be297ab19 100644
--- a/server/src/tracker/pageviewQueue.ts
+++ b/server/src/tracker/pageviewQueue.ts
@@ -2,6 +2,7 @@ import { DateTime } from "luxon";
import clickhouse from "../db/clickhouse/clickhouse.js";
import { TrackingPayload } from "../types.js";
import { getDeviceType } from "../utils.js";
+import { getChannel } from "./getChannel.js";
type TotalPayload = TrackingPayload & {
userId: string;
@@ -78,6 +79,7 @@ class PageviewQueue {
querystring: pv.querystring || "",
page_title: pv.page_title || "",
referrer: pv.referrer || "",
+ channel: getChannel(pv.referrer, pv.querystring),
browser: pv.ua.browser.name || "",
browser_version: pv.ua.browser.major || "",
operating_system: pv.ua.os.name || "",