🐸 Rybbit - open-source and privacy-friendly alternative to Google Analytics that is 10x more intuitive.
Update RetentionChart margin and add channel field to Clickhouse schema
Bill Yangd7bd778bd662
diff --git a/client/src/app/[site]/retention/RetentionChart.tsx b/client/src/app/[site]/retention/RetentionChart.tsxindex d74be483d..d7433c846 100644--- a/client/src/app/[site]/retention/RetentionChart.tsx+++ b/client/src/app/[site]/retention/RetentionChart.tsx@@ -138,7 +138,7 @@ export function RetentionChart({ data, isLoading, mode }: RetentionChartProps) {<ResponsiveLinedata={chartData}theme={nivoTheme}- margin={{ top: 20, right: 100, bottom: 30, left: 40 }}+ margin={{ top: 20, right: 90, bottom: 30, left: 40 }}xScale={{type: "linear",min: 0,diff --git a/server/src/db/clickhouse/clickhouse.ts b/server/src/db/clickhouse/clickhouse.tsindex 424ebaedf..f142ea6e5 100644--- a/server/src/db/clickhouse/clickhouse.ts+++ b/server/src/db/clickhouse/clickhouse.ts@@ -21,6 +21,7 @@ export const initializeClickhouse = async () => {querystring String,page_title String,referrer String,+ channel String,browser LowCardinality(String),browser_version LowCardinality(String),operating_system LowCardinality(String),diff --git a/server/src/tracker/const.ts b/server/src/tracker/const.tsnew file mode 100644index 000000000..ab0dc6cdf--- /dev/null+++ b/server/src/tracker/const.ts@@ -0,0 +1,623 @@+export const searchDomains = [+ // Google and variants+ "google.com",+ "google.",++ // Bing+ "bing.com",+ "bing.",++ // Yahoo+ "yahoo.com",+ "yahoo.",+ "search.yahoo.",++ // Other global search engines+ "duckduckgo.com",+ "duck.com",+ "baidu.com",+ "baidu.",+ "yandex.com",+ "yandex.",+ "ya.ru",+ "qwant.com",+ "search.",+ "ecosia.org",+ "brave.com",+ "startpage.com",+ "searchencrypt.com",+ "searx.",+ "swisscows.com",+ "mojeek.com",+ "gibiru.com",+ "metager.org",+ "search.aol.com",+ "lycos.com",+ "wolframalpha.com",+ "ask.com",+ "dogpile.com",+ "webcrawler.com",++ // Regional search engines+ "naver.com",+ "daum.net",+ "seznam.cz",+ "coccoc.com",+ "yam.com",+ "so.com",+ "sogou.com",+ "goo.ne.jp",+ "rambler.ru",+];++export const socialDomains = [+ // Major social networks and their variants+ "facebook.com",+ "fb.com",+ "fb.me",+ "messenger.com",+ "m.facebook.com",+ "instagram.com",+ "instagram.",+ "ig.me",+ "twitter.com",+ "t.co",+ "x.com",+ "linkedin.com",+ "lnkd.in",+ "tiktok.com",+ "tiktok.",+ "vm.tiktok.com",+ "pinterest.com",+ "pinterest.",+ "pin.it",+ "reddit.com",+ "redd.it",+ "old.reddit.com",++ // Other global social platforms+ "snapchat.com",+ "snap.com",+ "youtube.com",+ "youtu.be",+ "discord.com",+ "discord.gg",+ "whatsapp.com",+ "wa.me",+ "telegram.org",+ "t.me",+ "medium.com",+ "tumblr.com",+ "tmblr.co",+ "quora.com",+ "threads.net",+ "mastodon.social",+ "mastodon.",+ "slack.com",+ "nextdoor.com",+ "clubhouse.com",+ "twitch.tv",+ "news.ycombinator.com",+ "hn.algolia.com",++ // Regional social platforms+ "wechat.com",+ "weixin.qq.com",+ "vk.com",+ "qq.com",+ "weibo.com",+ "weibo.cn",+ "line.me",+];++export const videoDomains = [+ // Major video platforms+ "youtube.com",+ "youtu.be",+ "vimeo.com",+ "twitch.tv",+ "dailymotion.com",++ // Streaming services+ "netflix.com",+ "disneyplus.com",+ "hulu.com",+ "hbomax.com",+ "peacocktv.com",+ "primevideo.com",+ "amazon.com/prime-video",+ "paramountplus.com",+ "discoveryplus.com",+ "crunchyroll.com",+ "funimation.com",+ "curiositystream.com",+ "mubi.com",++ // Video sharing+ "tiktok.com",+ "vm.tiktok.com",+ "vevo.com",+ "streamable.com",+ "bitchute.com",+ "rumble.com",+ "odysee.com",+ "lbry.tv",+ "tiktok.com",+ "reels.instagram.com",++ // Educational+ "ted.com",+ "khanacademy.org",+ "skillshare.com",+ "udemy.com",+ "masterclass.com",+ "coursera.org",+ "pluralsight.com",+ "lynda.com",+ "linkedin.com/learning",++ // Live streaming+ "facebook.com/gaming",+ "youtube.com/live",+ "kick.com",+ "caffeine.tv",++ // Regional platforms+ "bilibili.com",+ "niconico.jp",+ "vlive.tv",+ "youku.com",+ "tudou.com",+ "iqiyi.com",+];++export const shoppingDomains = [+ // Major marketplaces+ "amazon.com",+ "amazon.",+ "ebay.com",+ "ebay.",+ "etsy.com",+ "walmart.com",+ "target.com",+ "aliexpress.com",+ "wish.com",+ "shopify.com",+ "shop.app",++ // Fashion and apparel+ "asos.com",+ "zara.com",+ "forever21.com",+ "hm.com",+ "gap.com",+ "macys.com",+ "nordstrom.com",+ "net-a-porter.com",+ "farfetch.com",+ "fashionnova.com",++ // Electronics+ "bestbuy.com",+ "newegg.com",+ "bhphotovideo.com",+ "apple.com/shop",+ "samsung.com/shop",++ // Home and furniture+ "wayfair.com",+ "ikea.com",+ "homedepot.com",+ "lowes.com",+ "overstock.com",++ // Regional marketplaces+ "taobao.com",+ "jd.com",+ "rakuten.com",+ "flipkart.com",+ "lazada.com",+ "mercadolibre.com",+ "ozon.ru",+ "allegro.pl",+ "coupang.com",+ "gmarket.co.kr",++ // Grocery and food+ "instacart.com",+ "freshdirect.com",+ "ocado.com",+ "groceries.",+ "doordash.com",+ "ubereats.com",+ "grubhub.com",+ "deliveroo.com",+];++// Define sources types+export const searchSources = [+ // Major search engines+ "google",+ "bing",+ "yahoo",+ "duckduckgo",+ "baidu",+ "yandex",+ "qwant",++ // Other search engines+ "ecosia",+ "brave",+ "startpage",+ "searchencrypt",+ "swisscows",+ "mojeek",+ "gibiru",+ "metager",+ "wolframalpha",+ "ask",+ "dogpile",+ "webcrawler",+ "aol",++ // Regional search engines+ "naver",+ "daum",+ "seznam",+ "coccoc",+ "yam",+ "so",+ "sogou",+ "goo",+ "rambler",+];+export const socialSources = [+ // Major social platforms+ "facebook",+ "twitter",+ "linkedin",+ "instagram",+ "tiktok",+ "pinterest",+ "reddit",++ // Other platforms+ "snapchat",+ "youtube",+ "discord",+ "whatsapp",+ "telegram",+ "medium",+ "tumblr",+ "quora",+ "threads",+ "mastodon",+ "slack",+ "nextdoor",+ "clubhouse",+ "twitch",+ "hacker news",+ "hackernews",+ "ycombinator",++ // Shortened variants+ "fb",+ "ig",+ "pin",+ "li",+ "tw",+ "yt",++ // Regional platforms+ "wechat",+ "weixin",+ "vk",+ "qq",+ "weibo",+ "line",+ "kakao",+];+export const videoSources = [+ // Major video platforms+ "youtube",+ "vimeo",+ "twitch",+ "dailymotion",+ "tiktok",++ // Streaming services+ "disneyplus",+ "netflix",+ "hulu",+ "hbomax",+ "peacock",+ "prime video",+ "paramount+",+ "discovery+",+ "crunchyroll",+ "funimation",+ "curiositystream",+ "mubi",++ // Video sharing+ "vevo",+ "streamable",+ "bitchute",+ "rumble",+ "odysee",+ "lbry",+ "reels",++ // Educational+ "ted",+ "khanacademy",+ "skillshare",+ "udemy",+ "masterclass",+ "coursera",+ "pluralsight",+ "lynda",++ // Live streaming+ "kick",+ "caffeine",++ // Regional platforms+ "bilibili",+ "niconico",+ "vlive",+ "youku",+ "tudou",+ "iqiyi",+];+export const shoppingSources = [+ // Major marketplaces+ "amazon",+ "ebay",+ "etsy",+ "shopify",+ "walmart",+ "target",+ "aliexpress",+ "wish",+ "shop",+ "store",+ "shopping",++ // Fashion and apparel+ "asos",+ "zara",+ "forever21",+ "hm",+ "gap",+ "macys",+ "nordstrom",+ "net-a-porter",+ "farfetch",+ "fashionnova",++ // Electronics+ "bestbuy",+ "newegg",+ "bhphotovideo",+ "apple",+ "samsung",++ // Home and furniture+ "wayfair",+ "ikea",+ "homedepot",+ "lowes",+ "overstock",++ // Regional marketplaces+ "taobao",+ "jd",+ "rakuten",+ "flipkart",+ "lazada",+ "mercadolibre",+ "ozon",+ "allegro",+ "coupang",+ "gmarket",++ // Grocery and food+ "instacart",+ "freshdirect",+ "ocado",+ "groceries",+ "doordash",+ "ubereats",+ "grubhub",+ "deliveroo",+];+export const emailSources = [+ "email",+ "e_mail",+ "e-mail",+ "mail",+ "newsletter",+ "mailchimp",+ "campaign-archive",+ "sendgrid",+ "mailgun",+ "constantcontact",+ "klaviyo",+ "hubspot",+ "marketo",+ "brevo",+ "sendinblue",+ "getresponse",+ "activecampaign",+ "mailerlite",+ "convertkit",+ "drip",+ "gmail",+ "yahoo",+ "outlook",+ "hotmail",+ "list",+ "blast",+ "campaign",+];+export const smsSources = [+ "sms",+ "text",+ "twilio",+ "message",+ "whatsapp",+ "viber",+ "line",+ "imessage",+];++// Medium types+export const socialMediums = [+ "sm",+ "social-media",+ "social-network",+ "social",+ "community",+ "forum",+ "organic-social",+ "feed",+ "share",+ "repost",+ "tweet",+ "post",+ "update",+ "engagement",+ "ugc",+ "user-generated",+ "social_post",+];++export const videoMediums = [+ "video",+ "youtube",+ "vimeo",+ "streaming",+ "live",+ "tv",+ "ott",+ "broadcast",+ "clip",+ "trailer",+ "episode",+ "series",+ "documentary",+ "film",+ "movie",+ "animation",+ "video_ad",+ "pre-roll",+ "mid-roll",+ "post-roll",+];++export const displayMediums = [+ "display",+ "interstitial",+ "banner",+ "ad",+ "advert",+ "advertisement",+ "rich-media",+ "popup",+ "popunder",+ "overlay",+ "expandable",+ "floating",+ "skin",+ "wallpaper",+ "native",+ "programmatic",+ "dsp",+ "retargeting",+ "remarketing",+ "impression",+];++export const affiliateMediums = [+ "affiliate",+ "aff",+ "partner",+ "partnership",+ "commission",+ "rev-share",+ "performance",+ "cpa",+ "lead-gen",+ "lead-generation",+];++export const referralMediums = [+ "referral",+ "link",+ "app",+ "invite",+ "recommendation",+ "advocate",+ "refer-a-friend",+ "share",+ "web",+ "embed",+ "backlink",+ "qr",+ "qr-code",+ "deeplink",+ "vanity-url",+];++export const emailMediums = [+ "email",+ "e_mail",+ "e-mail",+ "mail",+ "newsletter",+ "digest",+ "bulletin",+ "marketing_email",+ "promotional",+ "transactional",+ "notification_email",+ "alert",+ "confirmation",+ "update",+ "news",+];++export const pushMediums = [+ "push",+ "notification",+ "mobile",+ "app-notification",+ "web-notification",+ "browser-notification",+ "alert",+ "toast",+ "prompt",+ "pwa",+ "install",+];++export const audioMediums = [+ "audio",+ "podcast",+ "radio",+ "broadcast",+ "streaming",+ "music",+ "playlist",+ "episode",+ "show",+ "airplay",+ "song",+ "track",+ "voice",+ "audio_ad",+ "spot",+ "jingle",+ "commercial",+];diff --git a/server/src/tracker/getChannel.ts b/server/src/tracker/getChannel.tsnew file mode 100644index 000000000..747e3c01f--- /dev/null+++ b/server/src/tracker/getChannel.ts@@ -0,0 +1,160 @@+import {+ affiliateMediums,+ audioMediums,+ displayMediums,+ emailMediums,+ emailSources,+ pushMediums,+ referralMediums,+ searchDomains,+ searchSources,+ shoppingDomains,+ shoppingSources,+ smsSources,+ socialDomains,+ socialMediums,+ socialSources,+ videoDomains,+ videoMediums,+ videoSources,+} from "./const.js";++// UTM and URL parameter parsing utilities+function getUTMParams(querystring: string): Record<string, string> {+ const params: Record<string, string> = {};++ if (!querystring) return params;++ try {+ const searchParams = new URLSearchParams(querystring);++ // Extract UTM parameters+ for (const [key, value] of searchParams.entries()) {+ if (key.startsWith("utm_") || key === "gclid" || key === "gad_source") {+ params[key.toLowerCase()] = value.toLowerCase();+ }+ }+ } catch (e) {+ console.error("Error parsing query string:", e);+ }++ return params;+}++function getDomainFromReferrer(referrer: string): string {+ if (!referrer) return "$direct";++ try {+ const url = new URL(referrer);+ return url.hostname;+ } catch (e) {+ // Handle invalid URLs+ return "$direct";+ }+}++export function getChannel(referrer: string, querystring: string): string {+ const utmParams = getUTMParams(querystring);+ const referringDomain = getDomainFromReferrer(referrer);++ // UTM parameters+ const utmSource = utmParams["utm_source"] || "";+ const utmMedium = utmParams["utm_medium"] || "";+ const utmCampaign = utmParams["utm_campaign"] || "";+ const gclid = utmParams["gclid"] || "";+ const gadSource = utmParams["gad_source"] || "";++ // Check if traffic is paid+ const isPaid =+ ["cpc", "cpm", "cpv", "cpa", "ppc", "retargeting"].includes(utmMedium) ||+ utmMedium.startsWith("paid") ||+ gclid !== "" ||+ gadSource !== "";++ // Check domain type - improved to handle subdomains better+ const isDomainSearch = searchDomains.some(+ (domain) =>+ referringDomain.includes(domain) ||+ /^(?:www\.)?google\.[a-z]{2,3}$/.test(referringDomain)+ );+ const isDomainSocial = socialDomains.some((domain) =>+ referringDomain.includes(domain)+ );+ const isDomainVideo = videoDomains.some((domain) =>+ referringDomain.includes(domain)+ );+ const isDomainShopping = shoppingDomains.some((domain) =>+ referringDomain.includes(domain)+ );++ // Check source type+ const isSourceSearch = searchSources.includes(utmSource);+ const isSourceSocial = socialSources.includes(utmSource);+ const isSourceVideo = videoSources.includes(utmSource);+ const isSourceShopping = shoppingSources.includes(utmSource);+ const isSourceEmail = emailSources.includes(utmSource);+ const isSourceSMS = smsSources.includes(utmSource);++ // Apply channel detection logic (in order of precedence)++ // Cross Network+ if (utmCampaign === "cross-network") return "Cross-Network";++ // Paid channels+ if (isPaid) {+ if (isSourceSearch || isDomainSearch) return "Paid Search";+ if (isSourceSocial || isDomainSocial) return "Paid Social";+ if (isSourceVideo || isDomainVideo) return "Paid Video";+ if (isSourceShopping || isDomainShopping) return "Paid Shopping";+ if (socialMediums.includes(utmMedium)) return "Paid Social";+ if (videoMediums.includes(utmMedium)) return "Paid Video";+ if (displayMediums.includes(utmMedium)) return "Display";+ if (gadSource === "1") return "Paid Search";+ if (/video/.test(utmCampaign)) return "Paid Video";+ return "Paid Unknown";+ }++ // Direct+ if (+ referringDomain === "$direct" &&+ !utmMedium &&+ (!utmSource || utmSource === "direct" || utmSource === "(direct)")+ ) {+ return "Direct";+ }++ // Organic channels - with referrer domain check prioritized for more accurate detection+ // This is the key part that handles organic search from Google clicks without UTM params+ if (isDomainSearch) return "Organic Search";+ if (isDomainSocial) return "Organic Social";+ if (isDomainVideo) return "Organic Video";+ if (isDomainShopping) return "Organic Shopping";++ // Then check UTM parameters if referrer didn't match+ if (isSourceSearch) return "Organic Search";+ if (isSourceSocial) return "Organic Social";+ if (isSourceVideo) return "Organic Video";+ if (isSourceShopping) return "Organic Shopping";+ if (isSourceEmail || emailMediums.includes(utmMedium)) return "Email";+ if (isSourceSMS) return "SMS";++ // Medium-based detection+ if (socialMediums.includes(utmMedium)) return "Organic Social";+ if (videoMediums.includes(utmMedium)) return "Organic Video";+ if (affiliateMediums.includes(utmMedium)) return "Affiliate";+ if (referralMediums.includes(utmMedium)) return "Referral";+ if (displayMediums.includes(utmMedium)) return "Display";+ if (audioMediums.includes(utmMedium)) return "Audio";+ if (pushMediums.includes(utmMedium) || utmMedium.endsWith("push"))+ return "Push";++ // Campaign-based detection+ if (/video/.test(utmCampaign)) return "Organic Video";+ if (/shop|shopping/.test(utmCampaign)) return "Organic Shopping";++ // If referring domain exists but we couldn't categorize it+ if (referringDomain && referringDomain !== "$direct") return "Referral";++ // Default fallback+ return "Unknown";+}diff --git a/server/src/tracker/pageviewQueue.ts b/server/src/tracker/pageviewQueue.tsindex 69434a881..be297ab19 100644--- a/server/src/tracker/pageviewQueue.ts+++ b/server/src/tracker/pageviewQueue.ts@@ -2,6 +2,7 @@ import { DateTime } from "luxon";import clickhouse from "../db/clickhouse/clickhouse.js";import { TrackingPayload } from "../types.js";import { getDeviceType } from "../utils.js";+import { getChannel } from "./getChannel.js";type TotalPayload = TrackingPayload & {userId: string;@@ -78,6 +79,7 @@ class PageviewQueue {querystring: pv.querystring || "",page_title: pv.page_title || "",referrer: pv.referrer || "",+ channel: getChannel(pv.referrer, pv.querystring),browser: pv.ua.browser.name || "",browser_version: pv.ua.browser.major || "",operating_system: pv.ua.os.name || "",