🐸 Rybbit - open-source and privacy-friendly alternative to Google Analytics that is 10x more intuitive.
Do data transformation on server
Edward Cuif71fc65d7b5f
diff --git a/client/src/api/admin/import.ts b/client/src/api/admin/import.tsindex 003073cd5..0be9d43a3 100644--- a/client/src/api/admin/import.ts+++ b/client/src/api/admin/import.ts@@ -1,7 +1,6 @@import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query";import { authedFetch } from "@/api/utils";import { APIResponse } from "@/api/types";-import type { ImportQuotaInfo } from "@/lib/import/types";interface GetSiteImportsResponse {importId: string;@@ -94,11 +93,3 @@ export function useDeleteSiteImport(site: number) {retry: false,});}--export function useGetImportQuota(site: number) {- return useQuery({- queryKey: ["import-quota", site],- queryFn: async () => await authedFetch<APIResponse<ImportQuotaInfo>>(`/import-quota/${site}`),- staleTime: 60000,- });-}diff --git a/client/src/lib/import/csv-worker-manager.ts b/client/src/lib/import/csv-worker-manager.tsindex 236e66214..36d6d501c 100644--- a/client/src/lib/import/csv-worker-manager.ts+++ b/client/src/lib/import/csv-worker-manager.ts@@ -1,14 +1,8 @@// Worker manager for coordinating CSV import process// Manages worker lifecycle, batch uploads, and error handling+// Server handles transformation and quota checking-import type {- WorkerMessageToWorker,- WorkerMessageToMain,- ImportProgress,- FailedBatch,- RybbitEvent,- ImportQuotaInfo,-} from "./types";+import type { WorkerMessageToWorker, WorkerMessageToMain, ImportProgress, FailedBatch, UmamiEvent } from "./types";const MAX_CONCURRENT_UPLOADS = 3; // Maximum number of parallel batch uploadsconst RETRY_ATTEMPTS = 3; // Number of retry attempts for failed batches@@ -32,7 +26,7 @@ export class CSVWorkerManager {};private onProgress: ProgressCallback | null = null;private onComplete: CompleteCallback | null = null;- private uploadQueue: Array<{ events: RybbitEvent[]; chunkIndex: number }> = [];+ private uploadQueue: Array<{ events: UmamiEvent[]; chunkIndex: number }> = [];private activeUploads = 0;private completedUploads = new Set<number>();private failedBatches: FailedBatch[] = [];@@ -50,7 +44,6 @@ export class CSVWorkerManager {siteId: number,importId: string,platform: "umami",- quotaInfo: ImportQuotaInfo,startDate?: string,endDate?: string): void {@@ -85,7 +78,7 @@ export class CSVWorkerManager {};// Set up error handler- this.worker.onerror = (error) => {+ this.worker.onerror = error => {this.progress.status = "failed";this.progress.errors.push({message: `Worker error: ${error.message}`,@@ -96,7 +89,7 @@ export class CSVWorkerManager {}};- // Start parsing+ // Start parsing (no quota info - server handles that)const message: WorkerMessageToWorker = {type: "PARSE_START",file,@@ -105,9 +98,6 @@ export class CSVWorkerManager {platform,startDate,endDate,- monthlyLimit: quotaInfo.monthlyLimit,- historicalWindowMonths: quotaInfo.historicalWindowMonths,- monthlyUsage: quotaInfo.monthlyUsage,};this.worker.postMessage(message);@@ -146,7 +136,7 @@ export class CSVWorkerManager {this.progress.skippedRows = message.totalSkipped;// Add detailed errors- message.errorDetails.forEach((error) => {+ message.errorDetails.forEach(error => {this.progress.errors.push({message: `Row ${error.row}: ${error.message}`,});@@ -185,7 +175,7 @@ export class CSVWorkerManager {}}- private async uploadBatch(events: RybbitEvent[], chunkIndex: number, retryCount = 0): Promise<void> {+ private async uploadBatch(events: UmamiEvent[], chunkIndex: number, retryCount = 0): Promise<void> {try {const response = await fetch(`/api/batch-import-events/${this.siteId}`, {method: "POST",@@ -267,10 +257,7 @@ export class CSVWorkerManager {this.progress.status = "completed";this.notifyProgress();if (this.onComplete) {- this.onComplete(- true,- `Import completed successfully: ${this.progress.importedEvents} events imported`- );+ this.onComplete(true, `Import completed successfully: ${this.progress.importedEvents} events imported`);}} else {this.progress.status = "completed";diff --git a/client/src/lib/import/mappers/tracker-const.ts b/client/src/lib/import/mappers/tracker-const.tsdeleted file mode 100644index 17a85707c..000000000--- a/client/src/lib/import/mappers/tracker-const.ts+++ /dev/null@@ -1,992 +0,0 @@-// Duplicated from server/src/services/tracker/const.ts for client-side import processing--const searchDomains = [- "google.com",- "google.",- "bing.com",- "bing.",- "yahoo.com",- "yahoo.",- "search.yahoo.",- "duckduckgo.com",- "duck.com",- "baidu.com",- "baidu.",- "yandex.com",- "yandex.",- "ya.ru",- "qwant.com",- "search.",- "ecosia.org",- "brave.com",- "startpage.com",- "searchencrypt.com",- "searx.",- "swisscows.com",- "mojeek.com",- "gibiru.com",- "metager.org",- "search.aol.com",- "lycos.com",- "wolframalpha.com",- "ask.com",- "dogpile.com",- "webcrawler.com",- "naver.com",- "daum.net",- "seznam.cz",- "coccoc.com",- "yam.com",- "so.com",- "sogou.com",- "goo.ne.jp",- "rambler.ru",- "perplexity.ai",- "you.com",- "phind.com",- "andi.com",- "neeva.com",- "kagi.com",-];--const socialDomains = [- "facebook.com",- "fb.com",- "fb.me",- "messenger.com",- "m.facebook.com",- "instagram.com",- "instagram.",- "ig.me",- "twitter.com",- "t.co",- "x.com",- "linkedin.com",- "lnkd.in",- "tiktok.com",- "tiktok.",- "vm.tiktok.com",- "pinterest.com",- "pinterest.",- "pin.it",- "reddit.com",- "redd.it",- "old.reddit.com",- "snapchat.com",- "snap.com",- "youtube.com",- "youtu.be",- "discord.com",- "discord.gg",- "whatsapp.com",- "wa.me",- "telegram.org",- "t.me",- "medium.com",- "tumblr.com",- "tmblr.co",- "quora.com",- "threads.net",- "mastodon.social",- "mastodon.",- "slack.com",- "nextdoor.com",- "clubhouse.com",- "twitch.tv",- "news.ycombinator.com",- "hn.algolia.com",- "bluesky.app",- "bsky.app",- "post.news",- "truth.social",- "truthsocial.com",- "gettr.com",- "parler.com",- "gab.com",- "minds.com",- "diaspora.",- "lemmy.",- "kbin.social",- "bereal.com",- "vsco.co",- "flickr.com",- "flic.kr",- "500px.com",- "deviantart.com",- "behance.net",- "dribbble.com",- "xing.com",- "viadeo.com",- "meetup.com",- "wechat.com",- "weixin.qq.com",- "vk.com",- "qq.com",- "weibo.com",- "weibo.cn",- "line.me",- "kakaotalk.com",- "viber.com",- "ok.ru",- "odnoklassniki.ru",-];--const videoDomains = [- "youtube.com",- "youtu.be",- "vimeo.com",- "twitch.tv",- "dailymotion.com",- "netflix.com",- "disneyplus.com",- "hulu.com",- "hbomax.com",- "max.com",- "peacocktv.com",- "primevideo.com",- "amazon.com/prime-video",- "paramountplus.com",- "discoveryplus.com",- "crunchyroll.com",- "funimation.com",- "curiositystream.com",- "mubi.com",- "appletv.com",- "tv.apple.com",- "tiktok.com",- "vm.tiktok.com",- "vevo.com",- "streamable.com",- "bitchute.com",- "rumble.com",- "odysee.com",- "lbry.tv",- "reels.instagram.com",- "ted.com",- "khanacademy.org",- "skillshare.com",- "udemy.com",- "masterclass.com",- "coursera.org",- "pluralsight.com",- "lynda.com",- "linkedin.com/learning",- "facebook.com/gaming",- "youtube.com/live",- "kick.com",- "caffeine.tv",- "mixer.com",- "bilibili.com",- "niconico.jp",- "vlive.tv",- "youku.com",- "tudou.com",- "iqiyi.com",- "vine.co",- "byte.co",- "triller.co",- "likee.video",-];--const shoppingDomains = [- "amazon.com",- "amazon.",- "ebay.com",- "ebay.",- "etsy.com",- "walmart.com",- "target.com",- "aliexpress.com",- "wish.com",- "shopify.com",- "shop.app",- "asos.com",- "zara.com",- "forever21.com",- "hm.com",- "gap.com",- "macys.com",- "nordstrom.com",- "net-a-porter.com",- "farfetch.com",- "fashionnova.com",- "shein.com",- "romwe.com",- "zaful.com",- "boohoo.com",- "prettylittlething.com",- "bestbuy.com",- "newegg.com",- "bhphotovideo.com",- "apple.com/shop",- "samsung.com/shop",- "microcenter.com",- "tigerdirect.com",- "wayfair.com",- "ikea.com",- "homedepot.com",- "lowes.com",- "overstock.com",- "cb2.com",- "crateandbarrel.com",- "westelm.com",- "taobao.com",- "jd.com",- "rakuten.com",- "flipkart.com",- "lazada.com",- "mercadolibre.com",- "ozon.ru",- "allegro.pl",- "coupang.com",- "gmarket.co.kr",- "shopee.com",- "tokopedia.com",- "bukalapak.com",- "instacart.com",- "freshdirect.com",- "ocado.com",- "groceries.",- "doordash.com",- "ubereats.com",- "grubhub.com",- "deliveroo.com",- "postmates.com",- "seamless.com",- "foodpanda.com",-];--const searchSources = [- "google",- "bing",- "yahoo",- "duckduckgo",- "baidu",- "yandex",- "qwant",- "ecosia",- "brave",- "startpage",- "searchencrypt",- "swisscows",- "mojeek",- "gibiru",- "metager",- "wolframalpha",- "ask",- "dogpile",- "webcrawler",- "aol",- "naver",- "daum",- "seznam",- "coccoc",- "yam",- "so",- "sogou",- "goo",- "rambler",- "perplexity",- "you",- "phind",- "andi",- "neeva",- "kagi",-];--const socialSources = [- "facebook",- "twitter",- "linkedin",- "instagram",- "tiktok",- "pinterest",- "reddit",- "snapchat",- "youtube",- "discord",- "whatsapp",- "telegram",- "medium",- "tumblr",- "quora",- "threads",- "mastodon",- "slack",- "nextdoor",- "clubhouse",- "twitch",- "hacker news",- "hackernews",- "ycombinator",- "fb",- "ig",- "pin",- "li",- "tw",- "yt",- "bluesky",- "bsky",- "post",- "truth social",- "gettr",- "parler",- "gab",- "minds",- "diaspora",- "lemmy",- "kbin",- "bereal",- "vsco",- "flickr",- "500px",- "deviantart",- "behance",- "dribbble",- "xing",- "viadeo",- "meetup",- "wechat",- "weixin",- "vk",- "qq",- "weibo",- "line",- "kakao",- "kakaotalk",- "viber",- "ok",- "odnoklassniki",-];--const videoSources = [- "youtube",- "vimeo",- "twitch",- "dailymotion",- "tiktok",- "disneyplus",- "netflix",- "hulu",- "hbomax",- "max",- "peacock",- "prime video",- "paramount+",- "discovery+",- "crunchyroll",- "funimation",- "curiositystream",- "mubi",- "apple tv",- "appletv",- "vevo",- "streamable",- "bitchute",- "rumble",- "odysee",- "lbry",- "reels",- "ted",- "khanacademy",- "skillshare",- "udemy",- "masterclass",- "coursera",- "pluralsight",- "lynda",- "kick",- "caffeine",- "mixer",- "bilibili",- "niconico",- "vlive",- "youku",- "tudou",- "iqiyi",- "vine",- "byte",- "triller",- "likee",-];--const shoppingSources = [- "amazon",- "ebay",- "etsy",- "shopify",- "walmart",- "target",- "aliexpress",- "wish",- "shop",- "store",- "shopping",- "asos",- "zara",- "forever21",- "hm",- "gap",- "macys",- "nordstrom",- "net-a-porter",- "farfetch",- "fashionnova",- "shein",- "romwe",- "zaful",- "boohoo",- "prettylittlething",- "bestbuy",- "newegg",- "bhphotovideo",- "apple",- "samsung",- "microcenter",- "tigerdirect",- "wayfair",- "ikea",- "homedepot",- "lowes",- "overstock",- "cb2",- "crateandbarrel",- "westelm",- "taobao",- "jd",- "rakuten",- "flipkart",- "lazada",- "mercadolibre",- "ozon",- "allegro",- "coupang",- "gmarket",- "shopee",- "tokopedia",- "bukalapak",- "instacart",- "freshdirect",- "ocado",- "groceries",- "doordash",- "ubereats",- "grubhub",- "deliveroo",- "postmates",- "seamless",- "foodpanda",-];--const emailSources = [- "email",- "e_mail",- "e-mail",- "mail",- "newsletter",- "mailchimp",- "campaign-archive",- "sendgrid",- "mailgun",- "constantcontact",- "klaviyo",- "hubspot",- "marketo",- "brevo",- "sendinblue",- "getresponse",- "activecampaign",- "mailerlite",- "convertkit",- "drip",- "gmail",- "yahoo",- "outlook",- "hotmail",- "list",- "blast",- "campaign",-];--const smsSources = [- "sms",- "text",- "twilio",- "message",- "whatsapp",- "viber",- "line",- "imessage",-];--const socialMediums = [- "sm",- "social-media",- "social-network",- "social",- "community",- "forum",- "organic-social",- "feed",- "share",- "repost",- "tweet",- "post",- "update",- "engagement",- "ugc",- "user-generated",- "social_post",-];--const videoMediums = [- "video",- "youtube",- "vimeo",- "streaming",- "live",- "tv",- "ott",- "broadcast",- "clip",- "trailer",- "episode",- "series",- "documentary",- "film",- "movie",- "animation",- "video_ad",- "pre-roll",- "mid-roll",- "post-roll",-];--const displayMediums = [- "display",- "interstitial",- "banner",- "ad",- "advert",- "advertisement",- "rich-media",- "popup",- "popunder",- "overlay",- "expandable",- "floating",- "skin",- "wallpaper",- "native",- "programmatic",- "dsp",- "retargeting",- "remarketing",- "impression",-];--const affiliateMediums = [- "affiliate",- "aff",- "partner",- "partnership",- "commission",- "rev-share",- "performance",- "cpa",- "lead-gen",- "lead-generation",-];--const referralMediums = [- "referral",- "link",- "app",- "invite",- "recommendation",- "advocate",- "refer-a-friend",- "share",- "web",- "embed",- "backlink",- "qr",- "qr-code",- "deeplink",- "vanity-url",-];--const emailMediums = [- "email",- "e_mail",- "e-mail",- "mail",- "newsletter",- "digest",- "bulletin",- "marketing_email",- "promotional",- "transactional",- "notification_email",- "alert",- "confirmation",- "update",- "news",-];--const pushMediums = [- "push",- "notification",- "mobile",- "app-notification",- "web-notification",- "browser-notification",- "alert",- "toast",- "prompt",- "pwa",- "install",-];--const audioMediums = [- "audio",- "podcast",- "radio",- "broadcast",- "streaming",- "music",- "playlist",- "episode",- "show",- "airplay",- "song",- "track",- "voice",- "audio_ad",- "spot",- "jingle",- "commercial",-];--const influencerMediums = [- "influencer",- "creator",- "sponsored",- "collaboration",- "brand-ambassador",- "micro-influencer",- "macro-influencer",- "nano-influencer",- "ugc-creator",- "content-creator",- "partnership",- "endorsement",-];--const cpcMediums = [- "cpc",- "ppc",- "paid-search",- "search-ads",- "google-ads",- "bing-ads",- "yahoo-ads",- "adwords",- "sem",- "paid-click",-];--const cpmMediums = [- "cpm",- "display-ads",- "banner-ads",- "impression",- "programmatic",- "rtb",- "dsp",- "demand-side",- "supply-side",- "ad-exchange",-];--const contentMediums = [- "content",- "blog",- "article",- "guest-post",- "editorial",- "pr",- "press-release",- "news",- "publication",- "magazine",- "journal",- "whitepaper",- "case-study",- "resource",-];--const eventMediums = [- "event",- "conference",- "webinar",- "workshop",- "seminar",- "meetup",- "trade-show",- "expo",- "summit",- "networking",- "live-event",- "virtual-event",-];--export const socialAppIds = [- "com.facebook",- "com.facebook.katana",- "com.facebook.facebook",- "com.facebook.messenger",- "com.facebook.orca",- "com.instagram",- "com.instagram.android",- "com.burbn.instagram",- "com.twitter",- "com.twitter.android",- "com.atebits.tweetie2",- "com.zhiliaoapp.musically",- "com.tiktok",- "com.snapchat",- "com.snapchat.android",- "com.toyopagroup.picaboo",- "com.linkedin",- "com.linkedin.android",- "com.linkedin.LinkedIn",- "com.pinterest",- "com.reddit",- "com.reddit.frontpage",- "com.reddit.reddit",- "com.discord",- "com.hammerandchisel.discord",- "org.telegram",- "org.telegram.messenger",- "ph.telegra.Telegraph",- "com.whatsapp",- "com.instagram.barcelona",- "com.threads",- "xyz.blueskyweb.app",- "org.joinmastodon.android",- "com.bereal.ft",- "com.vsco.cam",- "com.slack",- "im.vector.app",- "com.nextdoor",- "com.clubhouse.app",-];--export const videoAppIds = [- "com.google.android.youtube",- "com.google.ios.youtube",- "com.google.ios.youtubekids",- "com.google.android.apps.youtube.kids",- "com.google.ios.youtubeunplugged",- "com.google.android.youtube.tv",- "com.netflix",- "com.disney.disneyplus",- "com.hulu",- "com.hbo.hbonow",- "com.hbo.hbomax",- "com.peacocktv",- "com.amazon.avod",- "com.amazon.amazonvideo",- "com.cbs.app",- "com.paramountplus",- "tv.twitch",- "tv.twitch.android.app",- "com.vimeo",- "com.dailymotion",-];--export const searchAppIds = [- "com.google.android.googlequicksearchbox",- "com.google.android.websearch",- "com.microsoft.bing",- "com.yahoo.mobile.client.android.search",- "com.yahoo.search",- "com.duckduckgo.mobile.android",- "com.duckduckgo.mobile.ios",- "com.ecosia.android",- "com.brave.browser",- "org.mozilla.firefox",- "com.microsoft.emmx",-];--export const emailAppIds = [- "com.google.android.gm",- "com.google.android.gm.lite",- "com.google.Gmail",- "com.microsoft.office.outlook",- "com.microsoft.outlooklite",- "com.microsoft.Office.Outlook",- "com.yahoo.mobile.client.android.mail",- "com.yahoo.Aerogram",- "com.apple.mobilemail",- "ch.protonmail.android",- "ch.protonmail.protonmail",- "com.superhuman.mail",- "com.superhuman.Superhuman",- "com.samsung.android.email.provider",- "me.bluemail.mail",- "com.easilydo.mail",- "org.kman.AquaMail",- "com.aol.mobile.aolapp",- "ru.mail.mailapp",- "ru.mail.mail",- "ru.yandex.mail",- "com.pingapp.app",- "com.readdle.smartemail",-];--export const shoppingAppIds = [- "com.amazon.mShop",- "com.amazon.shopping",- "com.ebay.mobile",- "com.walmart.android",- "com.target.ui",- "com.etsy.android",- "com.shopify.mobile",- "com.contextlogic.wish",- "com.alibaba.aliexpresshd",- "com.zzkko",- "com.wayfair.wayfair",- "com.newegg.app",- "com.bestbuy.android",- "com.ikea.app",- "com.homedepot",- "com.lowes.android",- "com.overstock",-];--export const newsAppIds = [- "com.cnn.mobile.android.phone",- "com.foxnews.android",- "com.nytimes.android",- "com.washingtonpost.rainbow",- "com.wsj.reader",- "com.usatoday.android.news",- "com.bbc.news",- "com.reuters.android",- "com.ap.mobile",- "com.nbcuni.nbc",- "com.aol.mobile.techcrunch",- "com.theverge.verge",- "com.wired.android",- "com.arstechnica.app",- "com.engadget.android",- "com.google.android.apps.magazines",- "flipboard.app",- "com.apple.news",- "com.microsoft.amp.apps.bingnews",-];--export const productivityAppIds = [- "com.microsoft.office.word",- "com.microsoft.office.excel",- "com.microsoft.office.powerpoint",- "com.microsoft.teams",- "com.google.android.apps.docs",- "com.google.android.apps.docs.editors.sheets",- "com.google.android.apps.docs.editors.slides",- "com.google.android.apps.meetings",- "com.evernote",- "us.zoom.videomeetings",- "com.notion.id",- "md.obsidian",- "com.dropbox.android",-];--export function isMobileAppId(source: string): boolean {- return /^[a-z0-9_]+(\.([a-z0-9_]+))+$/.test(source);-}--export function getSourceType(source: string): string {- const lowerSource = source.toLowerCase();-- if (searchDomains.some((domain) => lowerSource.includes(domain))) return "search";- if (socialDomains.some((domain) => lowerSource.includes(domain))) return "social";- if (videoDomains.some((domain) => lowerSource.includes(domain))) return "video";- if (shoppingDomains.some((domain) => lowerSource.includes(domain))) return "shopping";-- if (searchSources.includes(lowerSource)) return "search";- if (socialSources.includes(lowerSource)) return "social";- if (videoSources.includes(lowerSource)) return "video";- if (shoppingSources.includes(lowerSource)) return "shopping";- if (emailSources.includes(lowerSource)) return "email";- if (smsSources.includes(lowerSource)) return "sms";-- if (isMobileAppId(source)) {- if (socialAppIds.includes(source)) return "social";- if (videoAppIds.includes(source)) return "video";- if (searchAppIds.includes(source)) return "search";- if (emailAppIds.includes(source)) return "email";- if (shoppingAppIds.includes(source)) return "shopping";- if (newsAppIds.includes(source)) return "news";- if (productivityAppIds.includes(source)) return "productivity";- return "mobile-app";- }-- return "direct";-}--export function getMediumType(medium: string): string {- const lowerMedium = medium.toLowerCase();-- if (socialMediums.includes(lowerMedium)) return "social";- if (videoMediums.includes(lowerMedium)) return "video";- if (displayMediums.includes(lowerMedium)) return "display";- if (affiliateMediums.includes(lowerMedium)) return "affiliate";- if (referralMediums.includes(lowerMedium)) return "referral";- if (emailMediums.includes(lowerMedium)) return "email";- if (pushMediums.includes(lowerMedium)) return "push";- if (audioMediums.includes(lowerMedium)) return "audio";- if (influencerMediums.includes(lowerMedium)) return "influencer";- if (cpcMediums.includes(lowerMedium)) return "cpc";- if (cpmMediums.includes(lowerMedium)) return "cpm";- if (contentMediums.includes(lowerMedium)) return "content";- if (eventMediums.includes(lowerMedium)) return "event";-- return "organic";-}--export function isPaidTraffic(medium: string, source: string): boolean {- const lowerMedium = medium.toLowerCase();- const lowerSource = source.toLowerCase();-- const paidMediums = [- ...cpcMediums,- ...cpmMediums,- ...displayMediums,- "paid",- "ad",- "ads",- "advertising",- "sponsored",- "promotion",- ];-- const paidSources = [- "google ads",- "googleads",- "bing ads",- "facebook ads",- "instagram ads",- "twitter ads",- "linkedin ads",- "tiktok ads",- "youtube ads",- "pinterest ads",- ];-- return (- paidMediums.some((pm) => lowerMedium.includes(pm)) || paidSources.some((ps) => lowerSource.includes(ps))- );-}diff --git a/client/src/lib/import/mappers/tracker-utils.ts b/client/src/lib/import/mappers/tracker-utils.tsdeleted file mode 100644index b1a11a12c..000000000--- a/client/src/lib/import/mappers/tracker-utils.ts+++ /dev/null@@ -1,262 +0,0 @@-// Duplicated from server/src/services/tracker/utils.ts and getChannel.ts for client-side import processing--import {- emailAppIds,- getMediumType,- getSourceType,- isMobileAppId,- isPaidTraffic,- newsAppIds,- productivityAppIds,- searchAppIds,- shoppingAppIds,- socialAppIds,- videoAppIds,-} from "./tracker-const";--// UTM and URL parameter parsing utilities-export function getUTMParams(querystring: string): Record<string, string> {- const params: Record<string, string> = {};-- if (!querystring) return params;-- try {- const searchParams = new URLSearchParams(querystring);-- // Extract UTM parameters- for (const [key, value] of searchParams.entries()) {- if (key.startsWith("utm_") || key === "gclid" || key === "gad_source") {- params[key.toLowerCase()] = value.toLowerCase();- }- }- } catch (e) {- console.error("Error parsing query string:", e);- }-- return params;-}--// Parse all URL parameters from querystring-export function getAllUrlParams(querystring: string): Record<string, string> {- const params: Record<string, string> = {};-- if (!querystring) return params;-- // If querystring starts with ?, remove it- const cleanQuerystring = querystring.startsWith("?") ? querystring.substring(1) : querystring;-- try {- const searchParams = new URLSearchParams(cleanQuerystring);- // Extract all parameters- for (const [key, value] of searchParams.entries()) {- params[key.toLowerCase()] = value;- }- } catch (e) {- console.error("Error parsing query string for URL parameters:", e);- }-- return params;-}--// Clear referrer if it's from the same domain-export function clearSelfReferrer(referrer: string, hostname: string): string {- if (!referrer || !hostname) return referrer;-- try {- const referrerUrl = new URL(referrer);- if (referrerUrl.hostname === hostname) {- // Internal navigation, clear the referrer- return "";- }- } catch (e) {- // Invalid URL, return original referrer- }-- return referrer;-}--// Channel detection logic--function getMobileAppCategory(appId: string): { type: string; isPaid: boolean } | null {- const appIdLower = appId.toLowerCase();-- if (socialAppIds.some((id) => appIdLower.includes(id))) {- return { type: "Organic Social", isPaid: false };- }-- if (videoAppIds.some((id) => appIdLower.includes(id))) {- return { type: "Organic Video", isPaid: false };- }-- if (searchAppIds.some((id) => appIdLower.includes(id))) {- return { type: "Organic Search", isPaid: false };- }-- if (emailAppIds.some((id) => appIdLower.includes(id))) {- return { type: "Email", isPaid: false };- }-- if (shoppingAppIds.some((id) => appIdLower.includes(id))) {- return { type: "Organic Shopping", isPaid: false };- }-- if (newsAppIds.some((id) => appIdLower.includes(id))) {- return { type: "News", isPaid: false };- }-- if (productivityAppIds.some((id) => appIdLower.includes(id))) {- return { type: "Productivity", isPaid: false };- }-- return null;-}--function getDomainFromReferrer(referrer: string): string {- if (!referrer) return "$direct";-- try {- const url = new URL(referrer);- return url.hostname;- } catch (e) {- return "$direct";- }-}--function isSelfReferral(referringDomain: string, hostname: string): boolean {- if (!referringDomain || !hostname) return false;-- const stripWww = (domain: string) => domain.replace(/^www\./, "");- const refDomain = stripWww(referringDomain);- const currentDomain = stripWww(hostname);-- return refDomain === currentDomain || refDomain.endsWith("." + currentDomain);-}--export function getChannel(referrer: string, querystring: string, hostname?: string): string {- const utmParams = getUTMParams(querystring);- const referringDomain = getDomainFromReferrer(referrer);-- const selfReferral = hostname ? isSelfReferral(referringDomain, hostname) : false;-- const utmSource = utmParams["utm_source"] || "";- const utmMedium = utmParams["utm_medium"] || "";- const utmCampaign = utmParams["utm_campaign"] || "";- const gclid = utmParams["gclid"] || "";- const gadSource = utmParams["gad_source"] || "";-- if (utmSource && isMobileAppId(utmSource)) {- const appCategory = getMobileAppCategory(utmSource);- if (appCategory) {- return appCategory.isPaid ? "Paid " + appCategory.type.split(" ")[1] : appCategory.type;- }- }-- if (!referrer && !utmSource && !utmMedium && !utmCampaign && !gclid && !gadSource) {- return selfReferral ? "Internal" : "Direct";- }-- const sourceType = getSourceType(utmSource || referringDomain);- const mediumType = getMediumType(utmMedium);- const isPaid = isPaidTraffic(utmMedium, utmSource) || gclid !== "" || gadSource !== "";-- // Cross Network- if (utmCampaign === "cross-network") return "Cross-Network";-- // Direct traffic- if (- (referringDomain === "$direct" || (!referrer && !selfReferral)) &&- !utmMedium &&- (!utmSource || utmSource === "direct" || utmSource === "(direct)")- ) {- return "Direct";- }-- // Paid channels- if (isPaid) {- switch (sourceType) {- case "search":- return "Paid Search";- case "social":- return "Paid Social";- case "video":- return "Paid Video";- case "shopping":- return "Paid Shopping";- default:- switch (mediumType) {- case "social":- return "Paid Social";- case "video":- return "Paid Video";- case "display":- case "cpm":- return "Display";- case "cpc":- return "Paid Search";- case "influencer":- return "Paid Influencer";- case "audio":- return "Paid Audio";- default:- return "Paid Unknown";- }- }- }-- // Organic channels- switch (sourceType) {- case "search":- return "Organic Search";- case "social":- return "Organic Social";- case "video":- return "Organic Video";- case "shopping":- return "Organic Shopping";- case "email":- return "Email";- case "sms":- return "SMS";- case "news":- return "News";- case "productivity":- return "Productivity";- }-- // Medium-based detection- switch (mediumType) {- case "social":- return "Organic Social";- case "video":- return "Organic Video";- case "affiliate":- return "Affiliate";- case "referral":- return "Referral";- case "display":- return "Display";- case "audio":- return "Audio";- case "push":- return "Push";- case "influencer":- return "Influencer";- case "content":- return "Content";- case "event":- return "Event";- case "email":- return "Email";- }-- // Campaign-based detection- if (/video/.test(utmCampaign)) return "Organic Video";- if (/shop|shopping/.test(utmCampaign)) return "Organic Shopping";- if (/influencer|creator|sponsored/.test(utmCampaign)) return "Influencer";- if (/event|conference|webinar/.test(utmCampaign)) return "Event";- if (/social|facebook|twitter|instagram|linkedin/.test(utmCampaign)) return "Organic Social";-- if (referringDomain && referringDomain !== "$direct" && !selfReferral) return "Referral";-- return "Unknown";-}diff --git a/client/src/lib/import/mappers/umami.ts b/client/src/lib/import/mappers/umami.tsdeleted file mode 100644index 5b8c98de3..000000000--- a/client/src/lib/import/mappers/umami.ts+++ /dev/null@@ -1,207 +0,0 @@-// Duplicated from server/src/services/import/mappings/umami.ts for client-side import processing--import { z } from "zod";-import { UmamiEvent, RybbitEvent } from "../types";-import { getAllUrlParams, clearSelfReferrer, getChannel } from "./tracker-utils";--export const umamiHeaders = [- undefined,- "session_id",- undefined,- undefined,- "hostname",- "browser",- "os",- "device",- "screen",- "language",- "country",- "region",- "city",- "url_path",- "url_query",- undefined,- undefined,- undefined,- undefined,- undefined,- "referrer_path",- "referrer_query",- "referrer_domain",- "page_title",- undefined,- undefined,- undefined,- undefined,- undefined,- undefined,- "event_type",- "event_name",- undefined,- "distinct_id",- "created_at",- undefined,-];--export class UmamiImportMapper {- private static readonly browserMap: Record<string, string> = {- chrome: "Chrome",- opera: "Opera",- crios: "Mobile Chrome",- firefox: "Firefox",- facebook: "Facebook",- safari: "Safari",- ios: "Mobile Safari",- "ios-webview": "Mobile Safari",- "edge-chromium": "Edge",- samsung: "Samsung Internet",- yandexbrowser: "Yandex",- "edge-ios": "Edge",- "chromium-webview": "Chrome WebView",- fxios: "Mobile Firefox",- edge: "Edge",- };-- private static readonly browserSchema = z- .string()- .max(30)- .transform((browser) => {- const key = browser.toLowerCase();- return UmamiImportMapper.browserMap[key] ?? browser;- });-- private static readonly osMap: Record<string, string> = {- "windows 10": "Windows",- "windows 7": "Windows",- "windows server 2003": "Windows",- "mac os": "macOS",- ios: "iOS",- "android os": "Android",- linux: "Linux",- "chrome os": "Chrome OS",- };-- private static readonly osSchema = z- .string()- .max(25)- .transform((os) => {- const key = os.toLowerCase();- return UmamiImportMapper.osMap[key] ?? os;- });-- private static readonly deviceMap: Record<string, "Desktop" | "Mobile"> = {- laptop: "Desktop",- desktop: "Desktop",- mobile: "Mobile",- tablet: "Mobile",- };-- private static readonly deviceSchema = z- .string()- .max(20)- .transform((device) => {- const key = device.toLowerCase();- return UmamiImportMapper.deviceMap[key] ?? device;- });-- private static readonly umamiEventSchema = z.object({- session_id: z.string().uuid(),-- hostname: z.string().max(253),- browser: UmamiImportMapper.browserSchema,- os: UmamiImportMapper.osSchema,- device: UmamiImportMapper.deviceSchema,- screen: z- .string()- .regex(/^\d{1,5}x\d{1,5}$/)- .or(z.literal("")),- language: z.string().max(35),- country: z- .string()- .regex(/^[A-Z]{2}$/)- .or(z.literal("")),- region: z- .string()- .regex(/^[A-Z]{2}-[A-Z0-9]{1,3}$/)- .or(z.literal("")),- city: z.string().max(60),-- url_path: z.string().max(2048),- url_query: z- .string()- .max(2048)- .transform((querystring) => (querystring ? `?${querystring}` : "")),- referrer_path: z.string().max(2048),- referrer_query: z- .string()- .max(2048)- .transform((querystring) => (querystring ? `?${querystring}` : "")),- referrer_domain: z- .string()- .max(253)- .transform((url) => (url ? `https://${url}` : "")),- page_title: z.string().max(512),-- event_type: z.enum(["1", "2"]),- event_name: z.string().max(256),- distinct_id: z.string().max(64),- created_at: z.string().regex(/^\d{4}-(0[1-9]|1[0-2])-(0[1-9]|[12]\d|3[01]) ([01]\d|2[0-3]):([0-5]\d):([0-5]\d)$/),- });-- static transform(- events: UmamiEvent[],- site: string,- importId: string,- onError?: (error: { row: number; message: string }) => void- ): RybbitEvent[] {- return events.reduce<RybbitEvent[]>((acc, event, index) => {- const parsed = UmamiImportMapper.umamiEventSchema.safeParse(event);- if (!parsed.success) {- if (onError) {- onError({- row: index + 1,- message: `Validation failed: ${parsed.error.errors.map((e) => e.message).join(", ")}`,- });- }- return acc;- }-- const data = parsed.data;- const referrer = data.referrer_domain + data.referrer_path + data.referrer_query;- const [screenWidth, screenHeight] = data.screen ? data.screen.split("x") : ["0", "0"];-- acc.push({- site_id: Number(site),- timestamp: data.created_at,- session_id: data.session_id,- user_id: data.distinct_id,- hostname: data.hostname,- pathname: data.url_path,- querystring: data.url_query,- url_parameters: getAllUrlParams(data.url_query),- page_title: data.page_title,- referrer: clearSelfReferrer(referrer, data.hostname.replace(/^www\./, "")),- channel: getChannel(referrer, data.url_query, data.hostname),- browser: data.browser,- browser_version: "",- operating_system: data.os,- operating_system_version: event.os === "Windows 10" ? "10" : event.os === "Windows 7" ? "7" : "",- language: data.language,- country: data.country,- region: data.region,- city: data.city,- lat: 0,- lon: 0,- screen_width: parseInt(screenWidth, 10),- screen_height: parseInt(screenHeight, 10),- device_type: data.device,- type: data.event_type === "1" ? "pageview" : "custom_event",- event_name: data.event_type === "1" ? "" : data.event_name,- props: {},- import_id: importId,- });-- return acc;- }, []);- }-}diff --git a/client/src/lib/import/quota-checker.ts b/client/src/lib/import/quota-checker.tsdeleted file mode 100644index 90f852abb..000000000--- a/client/src/lib/import/quota-checker.ts+++ /dev/null@@ -1,83 +0,0 @@-// Client-side quota checking for import validation-// Based on server/src/services/import/importQuotaChecker.ts--import { DateTime } from "luxon";-import { ImportQuotaInfo } from "./types";--export class ClientQuotaChecker {- private monthlyUsage: Map<string, number>;- private readonly monthlyLimit: number;- private readonly historicalWindowMonths: number;- private readonly oldestAllowedMonth: string;-- constructor(quotaInfo: ImportQuotaInfo) {- this.monthlyUsage = new Map(Object.entries(quotaInfo.monthlyUsage));- this.monthlyLimit = quotaInfo.monthlyLimit;- this.historicalWindowMonths = quotaInfo.historicalWindowMonths;-- const oldestAllowedDate = DateTime.utc().minus({ months: quotaInfo.historicalWindowMonths }).startOf("month");- this.oldestAllowedMonth = oldestAllowedDate.toFormat("yyyyMM");- }-- canImportEvent(timestamp: string): boolean {- if (this.monthlyLimit === Infinity) {- return true;- }-- const dt = DateTime.fromFormat(timestamp, "yyyy-MM-dd HH:mm:ss", { zone: "utc" });- if (!dt.isValid) {- console.warn("Invalid timestamp format:", timestamp);- return false;- }-- const month = dt.toFormat("yyyyMM");-- // Check if event is within historical window- if (month < this.oldestAllowedMonth) {- return false;- }-- const used = this.monthlyUsage.get(month) || 0;- if (used >= this.monthlyLimit) {- return false;- }-- // Update usage tracker- this.monthlyUsage.set(month, used + 1);- return true;- }-- getSummary(): {- totalMonthsInWindow: number;- monthsAtCapacity: number;- monthsWithSpace: number;- oldestAllowedMonth: string;- } {- if (this.monthlyLimit === Infinity) {- return {- totalMonthsInWindow: this.historicalWindowMonths,- monthsAtCapacity: 0,- monthsWithSpace: this.historicalWindowMonths,- oldestAllowedMonth: this.oldestAllowedMonth,- };- }-- let monthsAtCapacity = 0;- for (const usage of this.monthlyUsage.values()) {- if (usage >= this.monthlyLimit) {- monthsAtCapacity++;- }- }-- return {- totalMonthsInWindow: this.historicalWindowMonths,- monthsAtCapacity,- monthsWithSpace: this.historicalWindowMonths - monthsAtCapacity,- oldestAllowedMonth: this.oldestAllowedMonth,- };- }-- getMonthlyUsage(): Record<string, number> {- return Object.fromEntries(this.monthlyUsage);- }-}diff --git a/client/src/lib/import/types.ts b/client/src/lib/import/types.tsindex bbc65dd01..292e8ddd6 100644--- a/client/src/lib/import/types.ts+++ b/client/src/lib/import/types.ts@@ -1,35 +1,5 @@// Types for client-side CSV import system--export interface RybbitEvent {- site_id: number;- timestamp: string;- session_id: string;- user_id: string;- hostname: string;- pathname: string;- querystring: string;- url_parameters: Record<string, string>;- page_title: string;- referrer: string;- channel: string;- browser: string;- browser_version: string;- operating_system: string;- operating_system_version: string;- language: string;- country: string;- region: string;- city: string;- lat: number;- lon: number;- screen_width: number;- screen_height: number;- device_type: string;- type: string;- event_name: string;- props: Record<string, unknown>;- import_id: string;-}+// Client parses CSV, server transforms and validatesexport interface UmamiEvent {session_id: string;@@ -64,9 +34,6 @@ export type WorkerMessageToWorker =platform: "umami";startDate?: string;endDate?: string;- monthlyLimit: number;- historicalWindowMonths: number;- monthlyUsage: Record<string, number>;}| {type: "CANCEL";@@ -81,7 +48,7 @@ export type WorkerMessageToMain =}| {type: "CHUNK_READY";- events: RybbitEvent[];+ events: UmamiEvent[]; // Raw CSV rows, not transformedchunkIndex: number;}| {@@ -97,18 +64,9 @@ export type WorkerMessageToMain =error?: unknown;};-// Quota information from server-export interface ImportQuotaInfo {- monthlyLimit: number;- historicalWindowMonths: number;- monthlyUsage: Record<string, number>; // { "202401": 5000, "202402": 7500, ... }- currentMonthUsage: number;- organizationId: string;-}--// Batch import request+// Batch import request (client sends raw rows to server)export interface BatchImportRequest {- events: RybbitEvent[];+ events: UmamiEvent[]; // Raw Umami CSV rowsimportId: string;batchIndex: number;totalBatches: number;@@ -138,7 +96,7 @@ export interface ImportProgress {// Failed batch info for retryexport interface FailedBatch {batchIndex: number;- events: RybbitEvent[];+ events: UmamiEvent[]; // Raw rows, not transformederror: string;retryCount: number;}diff --git a/client/src/workers/csv-import.worker.ts b/client/src/workers/csv-import.worker.tsindex 4f1e6f19b..55dd61c4b 100644--- a/client/src/workers/csv-import.worker.ts+++ b/client/src/workers/csv-import.worker.ts@@ -1,35 +1,68 @@-// Web Worker for client-side CSV parsing and transformation+// Web Worker for client-side CSV parsing only// This worker runs in a separate thread to avoid blocking the main UI thread+// Transformation happens on the serverimport Papa from "papaparse";import { DateTime } from "luxon";-import type { WorkerMessageToWorker, WorkerMessageToMain, UmamiEvent, RybbitEvent } from "@/lib/import/types";-import { UmamiImportMapper, umamiHeaders } from "@/lib/import/mappers/umami";-import { ClientQuotaChecker } from "@/lib/import/quota-checker";+import type { WorkerMessageToWorker, WorkerMessageToMain, UmamiEvent } from "@/lib/import/types";-const CHUNK_SIZE = 5000; // Number of events per batch sent to main thread+const CHUNK_SIZE = 5000; // Number of rows per batch sent to main threadconst PROGRESS_UPDATE_INTERVAL = 1000; // Update progress every 1000 rows-let quotaChecker: ClientQuotaChecker | null = null;-let currentBatch: RybbitEvent[] = [];+let currentBatch: UmamiEvent[] = [];let chunkIndex = 0;let totalParsed = 0;let totalSkipped = 0;let totalErrors = 0;let errorDetails: Array<{ row: number; message: string }> = [];-let siteId: number;-let importId: string;-let platform: "umami";let lastProgressUpdate = 0;-// Date range filter+// Date range filter (client-side optimization to reduce server load)let startDate: DateTime | null = null;let endDate: DateTime | null = null;+// Umami CSV header mapping+const umamiHeaders = [+ undefined,+ "session_id",+ undefined,+ undefined,+ "hostname",+ "browser",+ "os",+ "device",+ "screen",+ "language",+ "country",+ "region",+ "city",+ "url_path",+ "url_query",+ undefined,+ undefined,+ undefined,+ undefined,+ undefined,+ "referrer_path",+ "referrer_query",+ "referrer_domain",+ "page_title",+ undefined,+ undefined,+ undefined,+ undefined,+ undefined,+ undefined,+ "event_type",+ "event_name",+ undefined,+ "distinct_id",+ "created_at",+ undefined,+];+function createDateRangeFilter(startDateStr?: string, endDateStr?: string) {- startDate = startDateStr- ? DateTime.fromFormat(startDateStr, "yyyy-MM-dd", { zone: "utc" }).startOf("day")- : null;+ startDate = startDateStr ? DateTime.fromFormat(startDateStr, "yyyy-MM-dd", { zone: "utc" }).startOf("day") : null;endDate = endDateStr ? DateTime.fromFormat(endDateStr, "yyyy-MM-dd", { zone: "utc" }).endOf("day") : null;if (startDate && !startDate.isValid) {@@ -82,42 +115,29 @@ function sendProgress() {}function handleParsedRow(row: unknown, rowIndex: number) {- // Skip rows with missing or invalid datesconst umamiEvent = row as UmamiEvent;- if (!umamiEvent.created_at) {- totalSkipped++;- return;- }-- // Apply user-specified date range filter- if (!isDateInRange(umamiEvent.created_at)) {- totalSkipped++;- return;- }- // Check per-month quota- if (quotaChecker && !quotaChecker.canImportEvent(umamiEvent.created_at)) {+ // Skip rows with missing created_at (required field)+ if (!umamiEvent.created_at) {totalSkipped++;return;}- // Transform the event- const transformed = UmamiImportMapper.transform([umamiEvent], siteId.toString(), importId, (error) => {- totalErrors++;- if (errorDetails.length < 100) {- // Limit error collection- errorDetails.push({ row: rowIndex, message: error.message });+ // Apply optional date range filter (client-side optimization)+ if (startDate || endDate) {+ if (!isDateInRange(umamiEvent.created_at)) {+ totalSkipped++;+ return;}- });+ }- if (transformed.length > 0) {- currentBatch.push(transformed[0]);- totalParsed++;+ // Add to batch (no transformation, send raw row to server)+ currentBatch.push(umamiEvent);+ totalParsed++;- // Send batch when it reaches chunk size- if (currentBatch.length >= CHUNK_SIZE) {- sendChunk();- }+ // Send batch when it reaches chunk size+ if (currentBatch.length >= CHUNK_SIZE) {+ sendChunk();}// Send progress update periodically@@ -132,12 +152,14 @@ function parseCSV(file: File) {Papa.parse<UmamiEvent>(file, {header: true,- skipEmptyLines: true,+ dynamicTyping: true, // Auto-convert strings to numbers/booleans+ skipEmptyLines: "greedy", // Skip all empty lines (improved)+ delimiter: "", // Auto-detect delimiter (comma, tab, semicolon, etc.)transformHeader: (header, index) => {- // Use predefined Umami headers mapping+ // Map Umami CSV column positions to field namesreturn umamiHeaders[index] || header;},- step: (results) => {+ step: results => {if (results.data) {handleParsedRow(results.data, rowIndex);rowIndex++;@@ -145,9 +167,10 @@ function parseCSV(file: File) {if (results.errors && results.errors.length > 0) {totalErrors++;if (errorDetails.length < 100) {+ // Limit error collection to avoid memory issueserrorDetails.push({row: rowIndex,- message: results.errors.map((e) => e.message).join(", "),+ message: results.errors.map(e => e.message).join(", "),});}}@@ -159,20 +182,6 @@ function parseCSV(file: File) {// Send final progresssendProgress();- // Check if we skipped events due to quota- const quotaSummary = quotaChecker?.getSummary();- if (quotaSummary && quotaSummary.monthsAtCapacity > 0 && totalSkipped > 0) {- const quotaErrorMessage =- `${totalSkipped} events exceeded monthly quotas or fell outside the ${quotaSummary.totalMonthsInWindow}-month historical window. ` +- `${quotaSummary.monthsAtCapacity} of ${quotaSummary.totalMonthsInWindow} months are at full capacity. ` +- `Try importing newer data or upgrade your plan for higher monthly quotas.`;-- errorDetails.push({- row: 0,- message: quotaErrorMessage,- });- }-// Send completion messageconst message: WorkerMessageToMain = {type: "COMPLETE",@@ -183,7 +192,7 @@ function parseCSV(file: File) {};self.postMessage(message);},- error: (error) => {+ error: error => {const message: WorkerMessageToMain = {type: "ERROR",message: error.message,@@ -209,21 +218,7 @@ self.onmessage = (event: MessageEvent<WorkerMessageToWorker>) => {errorDetails = [];lastProgressUpdate = 0;- // Set up parameters- siteId = message.siteId;- importId = message.importId;- platform = message.platform;-- // Create quota checker- quotaChecker = new ClientQuotaChecker({- monthlyLimit: message.monthlyLimit,- historicalWindowMonths: message.historicalWindowMonths,- monthlyUsage: message.monthlyUsage,- currentMonthUsage: 0,- organizationId: "",- });-- // Set up date range filter+ // Set up date range filter (optional client-side optimization)createDateRangeFilter(message.startDate, message.endDate);// Start parsingdiff --git a/server/src/api/sites/batchImportEvents.ts b/server/src/api/sites/batchImportEvents.tsindex 0ee4cbfd1..264da7b41 100644--- a/server/src/api/sites/batchImportEvents.ts+++ b/server/src/api/sites/batchImportEvents.ts@@ -3,7 +3,11 @@ import { z } from "zod";import { getUserHasAdminAccessToSite } from "../../lib/auth-utils.js";import { clickhouse } from "../../db/clickhouse/clickhouse.js";import { updateImportProgress, updateImportStatus, getImportById } from "../../services/import/importStatusManager.js";-import { RybbitEvent } from "../../services/import/mappings/rybbit.js";+import { UmamiImportMapper, type UmamiEvent } from "../../services/import/mappings/umami.js";+import { ImportQuotaTracker } from "../../services/import/importQuotaChecker.js";+import { db } from "../../db/postgres/postgres.js";+import { sites } from "../../db/postgres/schema.js";+import { eq } from "drizzle-orm";import { createServiceLogger } from "../../lib/logger/logger.js";const logger = createServiceLogger("import:batch");@@ -14,7 +18,7 @@ const batchImportRequestSchema = zsite: z.string().min(1),}),body: z.object({- events: z.array(z.any()).min(1).max(10000), // Limit batch size to 10k events+ events: z.array(z.any()).min(1).max(10000), // Limit batch size to 10k events, raw Umami rowsimportId: z.string().uuid(),batchIndex: z.number().int().min(0),totalBatches: z.number().int().min(1),@@ -75,26 +79,96 @@ export async function batchImportEvents(request: FastifyRequest<BatchImportRequeawait updateImportStatus(importId, "processing");}+ // Get organization ID for quota checking+ const [siteRecord] = await db+ .select({ organizationId: sites.organizationId })+ .from(sites)+ .where(eq(sites.siteId, siteId))+ .limit(1);++ if (!siteRecord) {+ logger.error({ siteId }, "Site not found");+ return reply.status(404).send({ error: "Site not found" });+ }+try {- // Insert events into ClickHouse+ // Create quota tracker for server-side quota checking+ const quotaTracker = await ImportQuotaTracker.create(siteRecord.organizationId);++ // Filter events based on quota availability+ const eventsWithinQuota: UmamiEvent[] = [];+ let skippedDueToQuota = 0;++ for (const event of events as UmamiEvent[]) {+ if (!event.created_at) {+ continue; // Skip events without timestamp+ }++ if (quotaTracker.canImportEvent(event.created_at)) {+ eventsWithinQuota.push(event);+ } else {+ skippedDueToQuota++;+ }+ }++ // If all events were skipped due to quota, fail the batch+ if (eventsWithinQuota.length === 0 && events.length > 0) {+ const quotaSummary = quotaTracker.getSummary();+ const errorMessage =+ `All ${events.length} events in batch ${batchIndex} exceeded monthly quotas or fell outside the ${quotaSummary.totalMonthsInWindow}-month historical window. ` ++ `${quotaSummary.monthsAtCapacity} of ${quotaSummary.totalMonthsInWindow} months are at full capacity.`;++ logger.warn({ importId, batchIndex, skippedDueToQuota }, errorMessage);++ // Update import status to failed if this was the only batch or early in the import+ if (batchIndex === 0 || totalBatches === 1) {+ await updateImportStatus(importId, "failed", errorMessage);+ }++ return reply.status(400).send({+ success: false,+ error: "Quota exceeded",+ message: errorMessage,+ });+ }++ // Transform events using server-side mapper (single source of truth)+ const transformedEvents = UmamiImportMapper.transform(eventsWithinQuota, site, importId);++ if (transformedEvents.length === 0) {+ logger.warn({ importId, batchIndex }, "No valid events after transformation");+ return reply.send({+ success: true,+ importedCount: 0,+ message: `Batch ${batchIndex + 1}/${totalBatches}: No valid events`,+ });+ }++ // Insert transformed events into ClickHouseawait clickhouse.insert({table: "events",- values: events as RybbitEvent[],+ values: transformedEvents,format: "JSONEachRow",});logger.info(- { importId, batchIndex, eventCount: events.length, totalBatches },+ {+ importId,+ batchIndex,+ eventCount: transformedEvents.length,+ skippedDueToQuota,+ totalBatches,+ },"Batch inserted successfully");// Update progress- await updateImportProgress(importId, events.length);+ await updateImportProgress(importId, transformedEvents.length);return reply.send({success: true,- importedCount: events.length,- message: `Batch ${batchIndex + 1}/${totalBatches} imported successfully`,+ importedCount: transformedEvents.length,+ message: `Batch ${batchIndex + 1}/${totalBatches} imported successfully${skippedDueToQuota > 0 ? ` (${skippedDueToQuota} events skipped due to quota)` : ""}`,});} catch (insertError) {logger.error({ importId, batchIndex, error: insertError }, "Failed to insert batch");diff --git a/server/src/api/sites/getImportQuota.ts b/server/src/api/sites/getImportQuota.tsdeleted file mode 100644index 48339a18e..000000000--- a/server/src/api/sites/getImportQuota.ts+++ /dev/null@@ -1,75 +0,0 @@-import { FastifyReply, FastifyRequest } from "fastify";-import { z } from "zod";-import { getUserHasAdminAccessToSite } from "../../lib/auth-utils.js";-import { ImportQuotaTracker } from "../../services/import/importQuotaChecker.js";-import { db } from "../../db/postgres/postgres.js";-import { sites } from "../../db/postgres/schema.js";-import { eq } from "drizzle-orm";--const getImportQuotaRequestSchema = z- .object({- params: z.object({- site: z.string().min(1),- }),- })- .strict();--type GetImportQuotaRequest = {- Params: z.infer<typeof getImportQuotaRequestSchema.shape.params>;-};--export async function getImportQuota(request: FastifyRequest<GetImportQuotaRequest>, reply: FastifyReply) {- try {- const parsed = getImportQuotaRequestSchema.safeParse({- params: request.params,- });-- if (!parsed.success) {- return reply.status(400).send({ error: "Validation error" });- }-- const { site } = parsed.data.params;- const siteId = Number(site);-- const userHasAccess = await getUserHasAdminAccessToSite(request, site);- if (!userHasAccess) {- return reply.status(403).send({ error: "Forbidden" });- }-- // Get organization ID from site- const [siteRecord] = await db- .select({ organizationId: sites.organizationId })- .from(sites)- .where(eq(sites.siteId, siteId))- .limit(1);-- if (!siteRecord) {- return reply.status(404).send({ error: "Site not found" });- }-- // Create quota tracker to get quota information- const quotaTracker = await ImportQuotaTracker.create(siteRecord.organizationId);- const summary = quotaTracker.getSummary();-- // Get current month usage from the monthly usage map- const currentMonth = new Date().toISOString().slice(0, 7).replace("-", ""); // YYYYMM format- const monthlyUsageMap = await quotaTracker["queryMonthlyUsage"](- [siteId],- new Date(new Date().setMonth(new Date().getMonth() - summary.totalMonthsInWindow)).toISOString().split("T")[0]- );-- return reply.send({- data: {- monthlyLimit: quotaTracker["monthlyLimit"],- historicalWindowMonths: summary.totalMonthsInWindow,- monthlyUsage: Object.fromEntries(monthlyUsageMap),- currentMonthUsage: monthlyUsageMap.get(currentMonth) || 0,- organizationId: siteRecord.organizationId,- oldestAllowedMonth: summary.oldestAllowedMonth,- },- });- } catch (error) {- console.error("Error fetching import quota:", error);- return reply.status(500).send({ error: "Internal server error" });- }-}diff --git a/server/src/index.ts b/server/src/index.tsindex a45c36160..c1631ce89 100644--- a/server/src/index.ts+++ b/server/src/index.ts@@ -90,7 +90,6 @@ import { selectGSCProperty } from "./api/gsc/selectProperty.js";import { getSiteImports } from "./api/sites/getSiteImports.js";import { importSiteData } from "./api/sites/importSiteData.js";import { deleteSiteImport } from "./api/sites/deleteSiteImport.js";-import { getImportQuota } from "./api/sites/getImportQuota.js";import { batchImportEvents } from "./api/sites/batchImportEvents.js";import { getJobQueue } from "./services/import/queues/jobQueueFactory.js";@@ -367,8 +366,7 @@ server.post("/api/import-site-data/:site", {}, importSiteData);server.get("/api/get-site-imports/:site", getSiteImports);server.delete("/api/delete-site-import/:site/:importId", deleteSiteImport);-// Client-side import endpoints-server.get("/api/import-quota/:site", getImportQuota);+// Client-side import endpoint (parses CSV, server transforms)server.post("/api/batch-import-events/:site", batchImportEvents);// Administrative