Skip to content

🐸 Rybbit - open-source and privacy-friendly alternative to Google Analytics that is 10x more intuitive.

Do data transformation on server

Edward Cuif71fc65d7b5f
diff --git a/client/src/api/admin/import.ts b/client/src/api/admin/import.ts
index 003073cd5..0be9d43a3 100644
--- a/client/src/api/admin/import.ts
+++ b/client/src/api/admin/import.ts
@@ -1,7 +1,6 @@
import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query";
import { authedFetch } from "@/api/utils";
import { APIResponse } from "@/api/types";
-import type { ImportQuotaInfo } from "@/lib/import/types";
interface GetSiteImportsResponse {
importId: string;
@@ -94,11 +93,3 @@ export function useDeleteSiteImport(site: number) {
retry: false,
});
}
-
-export function useGetImportQuota(site: number) {
- return useQuery({
- queryKey: ["import-quota", site],
- queryFn: async () => await authedFetch<APIResponse<ImportQuotaInfo>>(`/import-quota/${site}`),
- staleTime: 60000,
- });
-}
diff --git a/client/src/lib/import/csv-worker-manager.ts b/client/src/lib/import/csv-worker-manager.ts
index 236e66214..36d6d501c 100644
--- a/client/src/lib/import/csv-worker-manager.ts
+++ b/client/src/lib/import/csv-worker-manager.ts
@@ -1,14 +1,8 @@
// Worker manager for coordinating CSV import process
// Manages worker lifecycle, batch uploads, and error handling
+// Server handles transformation and quota checking
-import type {
- WorkerMessageToWorker,
- WorkerMessageToMain,
- ImportProgress,
- FailedBatch,
- RybbitEvent,
- ImportQuotaInfo,
-} from "./types";
+import type { WorkerMessageToWorker, WorkerMessageToMain, ImportProgress, FailedBatch, UmamiEvent } from "./types";
const MAX_CONCURRENT_UPLOADS = 3; // Maximum number of parallel batch uploads
const RETRY_ATTEMPTS = 3; // Number of retry attempts for failed batches
@@ -32,7 +26,7 @@ export class CSVWorkerManager {
};
private onProgress: ProgressCallback | null = null;
private onComplete: CompleteCallback | null = null;
- private uploadQueue: Array<{ events: RybbitEvent[]; chunkIndex: number }> = [];
+ private uploadQueue: Array<{ events: UmamiEvent[]; chunkIndex: number }> = [];
private activeUploads = 0;
private completedUploads = new Set<number>();
private failedBatches: FailedBatch[] = [];
@@ -50,7 +44,6 @@ export class CSVWorkerManager {
siteId: number,
importId: string,
platform: "umami",
- quotaInfo: ImportQuotaInfo,
startDate?: string,
endDate?: string
): void {
@@ -85,7 +78,7 @@ export class CSVWorkerManager {
};
// Set up error handler
- this.worker.onerror = (error) => {
+ this.worker.onerror = error => {
this.progress.status = "failed";
this.progress.errors.push({
message: `Worker error: ${error.message}`,
@@ -96,7 +89,7 @@ export class CSVWorkerManager {
}
};
- // Start parsing
+ // Start parsing (no quota info - server handles that)
const message: WorkerMessageToWorker = {
type: "PARSE_START",
file,
@@ -105,9 +98,6 @@ export class CSVWorkerManager {
platform,
startDate,
endDate,
- monthlyLimit: quotaInfo.monthlyLimit,
- historicalWindowMonths: quotaInfo.historicalWindowMonths,
- monthlyUsage: quotaInfo.monthlyUsage,
};
this.worker.postMessage(message);
@@ -146,7 +136,7 @@ export class CSVWorkerManager {
this.progress.skippedRows = message.totalSkipped;
// Add detailed errors
- message.errorDetails.forEach((error) => {
+ message.errorDetails.forEach(error => {
this.progress.errors.push({
message: `Row ${error.row}: ${error.message}`,
});
@@ -185,7 +175,7 @@ export class CSVWorkerManager {
}
}
- private async uploadBatch(events: RybbitEvent[], chunkIndex: number, retryCount = 0): Promise<void> {
+ private async uploadBatch(events: UmamiEvent[], chunkIndex: number, retryCount = 0): Promise<void> {
try {
const response = await fetch(`/api/batch-import-events/${this.siteId}`, {
method: "POST",
@@ -267,10 +257,7 @@ export class CSVWorkerManager {
this.progress.status = "completed";
this.notifyProgress();
if (this.onComplete) {
- this.onComplete(
- true,
- `Import completed successfully: ${this.progress.importedEvents} events imported`
- );
+ this.onComplete(true, `Import completed successfully: ${this.progress.importedEvents} events imported`);
}
} else {
this.progress.status = "completed";
diff --git a/client/src/lib/import/mappers/tracker-const.ts b/client/src/lib/import/mappers/tracker-const.ts
deleted file mode 100644
index 17a85707c..000000000
--- a/client/src/lib/import/mappers/tracker-const.ts
+++ /dev/null
@@ -1,992 +0,0 @@
-// Duplicated from server/src/services/tracker/const.ts for client-side import processing
-
-const searchDomains = [
- "google.com",
- "google.",
- "bing.com",
- "bing.",
- "yahoo.com",
- "yahoo.",
- "search.yahoo.",
- "duckduckgo.com",
- "duck.com",
- "baidu.com",
- "baidu.",
- "yandex.com",
- "yandex.",
- "ya.ru",
- "qwant.com",
- "search.",
- "ecosia.org",
- "brave.com",
- "startpage.com",
- "searchencrypt.com",
- "searx.",
- "swisscows.com",
- "mojeek.com",
- "gibiru.com",
- "metager.org",
- "search.aol.com",
- "lycos.com",
- "wolframalpha.com",
- "ask.com",
- "dogpile.com",
- "webcrawler.com",
- "naver.com",
- "daum.net",
- "seznam.cz",
- "coccoc.com",
- "yam.com",
- "so.com",
- "sogou.com",
- "goo.ne.jp",
- "rambler.ru",
- "perplexity.ai",
- "you.com",
- "phind.com",
- "andi.com",
- "neeva.com",
- "kagi.com",
-];
-
-const socialDomains = [
- "facebook.com",
- "fb.com",
- "fb.me",
- "messenger.com",
- "m.facebook.com",
- "instagram.com",
- "instagram.",
- "ig.me",
- "twitter.com",
- "t.co",
- "x.com",
- "linkedin.com",
- "lnkd.in",
- "tiktok.com",
- "tiktok.",
- "vm.tiktok.com",
- "pinterest.com",
- "pinterest.",
- "pin.it",
- "reddit.com",
- "redd.it",
- "old.reddit.com",
- "snapchat.com",
- "snap.com",
- "youtube.com",
- "youtu.be",
- "discord.com",
- "discord.gg",
- "whatsapp.com",
- "wa.me",
- "telegram.org",
- "t.me",
- "medium.com",
- "tumblr.com",
- "tmblr.co",
- "quora.com",
- "threads.net",
- "mastodon.social",
- "mastodon.",
- "slack.com",
- "nextdoor.com",
- "clubhouse.com",
- "twitch.tv",
- "news.ycombinator.com",
- "hn.algolia.com",
- "bluesky.app",
- "bsky.app",
- "post.news",
- "truth.social",
- "truthsocial.com",
- "gettr.com",
- "parler.com",
- "gab.com",
- "minds.com",
- "diaspora.",
- "lemmy.",
- "kbin.social",
- "bereal.com",
- "vsco.co",
- "flickr.com",
- "flic.kr",
- "500px.com",
- "deviantart.com",
- "behance.net",
- "dribbble.com",
- "xing.com",
- "viadeo.com",
- "meetup.com",
- "wechat.com",
- "weixin.qq.com",
- "vk.com",
- "qq.com",
- "weibo.com",
- "weibo.cn",
- "line.me",
- "kakaotalk.com",
- "viber.com",
- "ok.ru",
- "odnoklassniki.ru",
-];
-
-const videoDomains = [
- "youtube.com",
- "youtu.be",
- "vimeo.com",
- "twitch.tv",
- "dailymotion.com",
- "netflix.com",
- "disneyplus.com",
- "hulu.com",
- "hbomax.com",
- "max.com",
- "peacocktv.com",
- "primevideo.com",
- "amazon.com/prime-video",
- "paramountplus.com",
- "discoveryplus.com",
- "crunchyroll.com",
- "funimation.com",
- "curiositystream.com",
- "mubi.com",
- "appletv.com",
- "tv.apple.com",
- "tiktok.com",
- "vm.tiktok.com",
- "vevo.com",
- "streamable.com",
- "bitchute.com",
- "rumble.com",
- "odysee.com",
- "lbry.tv",
- "reels.instagram.com",
- "ted.com",
- "khanacademy.org",
- "skillshare.com",
- "udemy.com",
- "masterclass.com",
- "coursera.org",
- "pluralsight.com",
- "lynda.com",
- "linkedin.com/learning",
- "facebook.com/gaming",
- "youtube.com/live",
- "kick.com",
- "caffeine.tv",
- "mixer.com",
- "bilibili.com",
- "niconico.jp",
- "vlive.tv",
- "youku.com",
- "tudou.com",
- "iqiyi.com",
- "vine.co",
- "byte.co",
- "triller.co",
- "likee.video",
-];
-
-const shoppingDomains = [
- "amazon.com",
- "amazon.",
- "ebay.com",
- "ebay.",
- "etsy.com",
- "walmart.com",
- "target.com",
- "aliexpress.com",
- "wish.com",
- "shopify.com",
- "shop.app",
- "asos.com",
- "zara.com",
- "forever21.com",
- "hm.com",
- "gap.com",
- "macys.com",
- "nordstrom.com",
- "net-a-porter.com",
- "farfetch.com",
- "fashionnova.com",
- "shein.com",
- "romwe.com",
- "zaful.com",
- "boohoo.com",
- "prettylittlething.com",
- "bestbuy.com",
- "newegg.com",
- "bhphotovideo.com",
- "apple.com/shop",
- "samsung.com/shop",
- "microcenter.com",
- "tigerdirect.com",
- "wayfair.com",
- "ikea.com",
- "homedepot.com",
- "lowes.com",
- "overstock.com",
- "cb2.com",
- "crateandbarrel.com",
- "westelm.com",
- "taobao.com",
- "jd.com",
- "rakuten.com",
- "flipkart.com",
- "lazada.com",
- "mercadolibre.com",
- "ozon.ru",
- "allegro.pl",
- "coupang.com",
- "gmarket.co.kr",
- "shopee.com",
- "tokopedia.com",
- "bukalapak.com",
- "instacart.com",
- "freshdirect.com",
- "ocado.com",
- "groceries.",
- "doordash.com",
- "ubereats.com",
- "grubhub.com",
- "deliveroo.com",
- "postmates.com",
- "seamless.com",
- "foodpanda.com",
-];
-
-const searchSources = [
- "google",
- "bing",
- "yahoo",
- "duckduckgo",
- "baidu",
- "yandex",
- "qwant",
- "ecosia",
- "brave",
- "startpage",
- "searchencrypt",
- "swisscows",
- "mojeek",
- "gibiru",
- "metager",
- "wolframalpha",
- "ask",
- "dogpile",
- "webcrawler",
- "aol",
- "naver",
- "daum",
- "seznam",
- "coccoc",
- "yam",
- "so",
- "sogou",
- "goo",
- "rambler",
- "perplexity",
- "you",
- "phind",
- "andi",
- "neeva",
- "kagi",
-];
-
-const socialSources = [
- "facebook",
- "twitter",
- "linkedin",
- "instagram",
- "tiktok",
- "pinterest",
- "reddit",
- "snapchat",
- "youtube",
- "discord",
- "whatsapp",
- "telegram",
- "medium",
- "tumblr",
- "quora",
- "threads",
- "mastodon",
- "slack",
- "nextdoor",
- "clubhouse",
- "twitch",
- "hacker news",
- "hackernews",
- "ycombinator",
- "fb",
- "ig",
- "pin",
- "li",
- "tw",
- "yt",
- "bluesky",
- "bsky",
- "post",
- "truth social",
- "gettr",
- "parler",
- "gab",
- "minds",
- "diaspora",
- "lemmy",
- "kbin",
- "bereal",
- "vsco",
- "flickr",
- "500px",
- "deviantart",
- "behance",
- "dribbble",
- "xing",
- "viadeo",
- "meetup",
- "wechat",
- "weixin",
- "vk",
- "qq",
- "weibo",
- "line",
- "kakao",
- "kakaotalk",
- "viber",
- "ok",
- "odnoklassniki",
-];
-
-const videoSources = [
- "youtube",
- "vimeo",
- "twitch",
- "dailymotion",
- "tiktok",
- "disneyplus",
- "netflix",
- "hulu",
- "hbomax",
- "max",
- "peacock",
- "prime video",
- "paramount+",
- "discovery+",
- "crunchyroll",
- "funimation",
- "curiositystream",
- "mubi",
- "apple tv",
- "appletv",
- "vevo",
- "streamable",
- "bitchute",
- "rumble",
- "odysee",
- "lbry",
- "reels",
- "ted",
- "khanacademy",
- "skillshare",
- "udemy",
- "masterclass",
- "coursera",
- "pluralsight",
- "lynda",
- "kick",
- "caffeine",
- "mixer",
- "bilibili",
- "niconico",
- "vlive",
- "youku",
- "tudou",
- "iqiyi",
- "vine",
- "byte",
- "triller",
- "likee",
-];
-
-const shoppingSources = [
- "amazon",
- "ebay",
- "etsy",
- "shopify",
- "walmart",
- "target",
- "aliexpress",
- "wish",
- "shop",
- "store",
- "shopping",
- "asos",
- "zara",
- "forever21",
- "hm",
- "gap",
- "macys",
- "nordstrom",
- "net-a-porter",
- "farfetch",
- "fashionnova",
- "shein",
- "romwe",
- "zaful",
- "boohoo",
- "prettylittlething",
- "bestbuy",
- "newegg",
- "bhphotovideo",
- "apple",
- "samsung",
- "microcenter",
- "tigerdirect",
- "wayfair",
- "ikea",
- "homedepot",
- "lowes",
- "overstock",
- "cb2",
- "crateandbarrel",
- "westelm",
- "taobao",
- "jd",
- "rakuten",
- "flipkart",
- "lazada",
- "mercadolibre",
- "ozon",
- "allegro",
- "coupang",
- "gmarket",
- "shopee",
- "tokopedia",
- "bukalapak",
- "instacart",
- "freshdirect",
- "ocado",
- "groceries",
- "doordash",
- "ubereats",
- "grubhub",
- "deliveroo",
- "postmates",
- "seamless",
- "foodpanda",
-];
-
-const emailSources = [
- "email",
- "e_mail",
- "e-mail",
- "mail",
- "newsletter",
- "mailchimp",
- "campaign-archive",
- "sendgrid",
- "mailgun",
- "constantcontact",
- "klaviyo",
- "hubspot",
- "marketo",
- "brevo",
- "sendinblue",
- "getresponse",
- "activecampaign",
- "mailerlite",
- "convertkit",
- "drip",
- "gmail",
- "yahoo",
- "outlook",
- "hotmail",
- "list",
- "blast",
- "campaign",
-];
-
-const smsSources = [
- "sms",
- "text",
- "twilio",
- "message",
- "whatsapp",
- "viber",
- "line",
- "imessage",
-];
-
-const socialMediums = [
- "sm",
- "social-media",
- "social-network",
- "social",
- "community",
- "forum",
- "organic-social",
- "feed",
- "share",
- "repost",
- "tweet",
- "post",
- "update",
- "engagement",
- "ugc",
- "user-generated",
- "social_post",
-];
-
-const videoMediums = [
- "video",
- "youtube",
- "vimeo",
- "streaming",
- "live",
- "tv",
- "ott",
- "broadcast",
- "clip",
- "trailer",
- "episode",
- "series",
- "documentary",
- "film",
- "movie",
- "animation",
- "video_ad",
- "pre-roll",
- "mid-roll",
- "post-roll",
-];
-
-const displayMediums = [
- "display",
- "interstitial",
- "banner",
- "ad",
- "advert",
- "advertisement",
- "rich-media",
- "popup",
- "popunder",
- "overlay",
- "expandable",
- "floating",
- "skin",
- "wallpaper",
- "native",
- "programmatic",
- "dsp",
- "retargeting",
- "remarketing",
- "impression",
-];
-
-const affiliateMediums = [
- "affiliate",
- "aff",
- "partner",
- "partnership",
- "commission",
- "rev-share",
- "performance",
- "cpa",
- "lead-gen",
- "lead-generation",
-];
-
-const referralMediums = [
- "referral",
- "link",
- "app",
- "invite",
- "recommendation",
- "advocate",
- "refer-a-friend",
- "share",
- "web",
- "embed",
- "backlink",
- "qr",
- "qr-code",
- "deeplink",
- "vanity-url",
-];
-
-const emailMediums = [
- "email",
- "e_mail",
- "e-mail",
- "mail",
- "newsletter",
- "digest",
- "bulletin",
- "marketing_email",
- "promotional",
- "transactional",
- "notification_email",
- "alert",
- "confirmation",
- "update",
- "news",
-];
-
-const pushMediums = [
- "push",
- "notification",
- "mobile",
- "app-notification",
- "web-notification",
- "browser-notification",
- "alert",
- "toast",
- "prompt",
- "pwa",
- "install",
-];
-
-const audioMediums = [
- "audio",
- "podcast",
- "radio",
- "broadcast",
- "streaming",
- "music",
- "playlist",
- "episode",
- "show",
- "airplay",
- "song",
- "track",
- "voice",
- "audio_ad",
- "spot",
- "jingle",
- "commercial",
-];
-
-const influencerMediums = [
- "influencer",
- "creator",
- "sponsored",
- "collaboration",
- "brand-ambassador",
- "micro-influencer",
- "macro-influencer",
- "nano-influencer",
- "ugc-creator",
- "content-creator",
- "partnership",
- "endorsement",
-];
-
-const cpcMediums = [
- "cpc",
- "ppc",
- "paid-search",
- "search-ads",
- "google-ads",
- "bing-ads",
- "yahoo-ads",
- "adwords",
- "sem",
- "paid-click",
-];
-
-const cpmMediums = [
- "cpm",
- "display-ads",
- "banner-ads",
- "impression",
- "programmatic",
- "rtb",
- "dsp",
- "demand-side",
- "supply-side",
- "ad-exchange",
-];
-
-const contentMediums = [
- "content",
- "blog",
- "article",
- "guest-post",
- "editorial",
- "pr",
- "press-release",
- "news",
- "publication",
- "magazine",
- "journal",
- "whitepaper",
- "case-study",
- "resource",
-];
-
-const eventMediums = [
- "event",
- "conference",
- "webinar",
- "workshop",
- "seminar",
- "meetup",
- "trade-show",
- "expo",
- "summit",
- "networking",
- "live-event",
- "virtual-event",
-];
-
-export const socialAppIds = [
- "com.facebook",
- "com.facebook.katana",
- "com.facebook.facebook",
- "com.facebook.messenger",
- "com.facebook.orca",
- "com.instagram",
- "com.instagram.android",
- "com.burbn.instagram",
- "com.twitter",
- "com.twitter.android",
- "com.atebits.tweetie2",
- "com.zhiliaoapp.musically",
- "com.tiktok",
- "com.snapchat",
- "com.snapchat.android",
- "com.toyopagroup.picaboo",
- "com.linkedin",
- "com.linkedin.android",
- "com.linkedin.LinkedIn",
- "com.pinterest",
- "com.reddit",
- "com.reddit.frontpage",
- "com.reddit.reddit",
- "com.discord",
- "com.hammerandchisel.discord",
- "org.telegram",
- "org.telegram.messenger",
- "ph.telegra.Telegraph",
- "com.whatsapp",
- "com.instagram.barcelona",
- "com.threads",
- "xyz.blueskyweb.app",
- "org.joinmastodon.android",
- "com.bereal.ft",
- "com.vsco.cam",
- "com.slack",
- "im.vector.app",
- "com.nextdoor",
- "com.clubhouse.app",
-];
-
-export const videoAppIds = [
- "com.google.android.youtube",
- "com.google.ios.youtube",
- "com.google.ios.youtubekids",
- "com.google.android.apps.youtube.kids",
- "com.google.ios.youtubeunplugged",
- "com.google.android.youtube.tv",
- "com.netflix",
- "com.disney.disneyplus",
- "com.hulu",
- "com.hbo.hbonow",
- "com.hbo.hbomax",
- "com.peacocktv",
- "com.amazon.avod",
- "com.amazon.amazonvideo",
- "com.cbs.app",
- "com.paramountplus",
- "tv.twitch",
- "tv.twitch.android.app",
- "com.vimeo",
- "com.dailymotion",
-];
-
-export const searchAppIds = [
- "com.google.android.googlequicksearchbox",
- "com.google.android.websearch",
- "com.microsoft.bing",
- "com.yahoo.mobile.client.android.search",
- "com.yahoo.search",
- "com.duckduckgo.mobile.android",
- "com.duckduckgo.mobile.ios",
- "com.ecosia.android",
- "com.brave.browser",
- "org.mozilla.firefox",
- "com.microsoft.emmx",
-];
-
-export const emailAppIds = [
- "com.google.android.gm",
- "com.google.android.gm.lite",
- "com.google.Gmail",
- "com.microsoft.office.outlook",
- "com.microsoft.outlooklite",
- "com.microsoft.Office.Outlook",
- "com.yahoo.mobile.client.android.mail",
- "com.yahoo.Aerogram",
- "com.apple.mobilemail",
- "ch.protonmail.android",
- "ch.protonmail.protonmail",
- "com.superhuman.mail",
- "com.superhuman.Superhuman",
- "com.samsung.android.email.provider",
- "me.bluemail.mail",
- "com.easilydo.mail",
- "org.kman.AquaMail",
- "com.aol.mobile.aolapp",
- "ru.mail.mailapp",
- "ru.mail.mail",
- "ru.yandex.mail",
- "com.pingapp.app",
- "com.readdle.smartemail",
-];
-
-export const shoppingAppIds = [
- "com.amazon.mShop",
- "com.amazon.shopping",
- "com.ebay.mobile",
- "com.walmart.android",
- "com.target.ui",
- "com.etsy.android",
- "com.shopify.mobile",
- "com.contextlogic.wish",
- "com.alibaba.aliexpresshd",
- "com.zzkko",
- "com.wayfair.wayfair",
- "com.newegg.app",
- "com.bestbuy.android",
- "com.ikea.app",
- "com.homedepot",
- "com.lowes.android",
- "com.overstock",
-];
-
-export const newsAppIds = [
- "com.cnn.mobile.android.phone",
- "com.foxnews.android",
- "com.nytimes.android",
- "com.washingtonpost.rainbow",
- "com.wsj.reader",
- "com.usatoday.android.news",
- "com.bbc.news",
- "com.reuters.android",
- "com.ap.mobile",
- "com.nbcuni.nbc",
- "com.aol.mobile.techcrunch",
- "com.theverge.verge",
- "com.wired.android",
- "com.arstechnica.app",
- "com.engadget.android",
- "com.google.android.apps.magazines",
- "flipboard.app",
- "com.apple.news",
- "com.microsoft.amp.apps.bingnews",
-];
-
-export const productivityAppIds = [
- "com.microsoft.office.word",
- "com.microsoft.office.excel",
- "com.microsoft.office.powerpoint",
- "com.microsoft.teams",
- "com.google.android.apps.docs",
- "com.google.android.apps.docs.editors.sheets",
- "com.google.android.apps.docs.editors.slides",
- "com.google.android.apps.meetings",
- "com.evernote",
- "us.zoom.videomeetings",
- "com.notion.id",
- "md.obsidian",
- "com.dropbox.android",
-];
-
-export function isMobileAppId(source: string): boolean {
- return /^[a-z0-9_]+(\.([a-z0-9_]+))+$/.test(source);
-}
-
-export function getSourceType(source: string): string {
- const lowerSource = source.toLowerCase();
-
- if (searchDomains.some((domain) => lowerSource.includes(domain))) return "search";
- if (socialDomains.some((domain) => lowerSource.includes(domain))) return "social";
- if (videoDomains.some((domain) => lowerSource.includes(domain))) return "video";
- if (shoppingDomains.some((domain) => lowerSource.includes(domain))) return "shopping";
-
- if (searchSources.includes(lowerSource)) return "search";
- if (socialSources.includes(lowerSource)) return "social";
- if (videoSources.includes(lowerSource)) return "video";
- if (shoppingSources.includes(lowerSource)) return "shopping";
- if (emailSources.includes(lowerSource)) return "email";
- if (smsSources.includes(lowerSource)) return "sms";
-
- if (isMobileAppId(source)) {
- if (socialAppIds.includes(source)) return "social";
- if (videoAppIds.includes(source)) return "video";
- if (searchAppIds.includes(source)) return "search";
- if (emailAppIds.includes(source)) return "email";
- if (shoppingAppIds.includes(source)) return "shopping";
- if (newsAppIds.includes(source)) return "news";
- if (productivityAppIds.includes(source)) return "productivity";
- return "mobile-app";
- }
-
- return "direct";
-}
-
-export function getMediumType(medium: string): string {
- const lowerMedium = medium.toLowerCase();
-
- if (socialMediums.includes(lowerMedium)) return "social";
- if (videoMediums.includes(lowerMedium)) return "video";
- if (displayMediums.includes(lowerMedium)) return "display";
- if (affiliateMediums.includes(lowerMedium)) return "affiliate";
- if (referralMediums.includes(lowerMedium)) return "referral";
- if (emailMediums.includes(lowerMedium)) return "email";
- if (pushMediums.includes(lowerMedium)) return "push";
- if (audioMediums.includes(lowerMedium)) return "audio";
- if (influencerMediums.includes(lowerMedium)) return "influencer";
- if (cpcMediums.includes(lowerMedium)) return "cpc";
- if (cpmMediums.includes(lowerMedium)) return "cpm";
- if (contentMediums.includes(lowerMedium)) return "content";
- if (eventMediums.includes(lowerMedium)) return "event";
-
- return "organic";
-}
-
-export function isPaidTraffic(medium: string, source: string): boolean {
- const lowerMedium = medium.toLowerCase();
- const lowerSource = source.toLowerCase();
-
- const paidMediums = [
- ...cpcMediums,
- ...cpmMediums,
- ...displayMediums,
- "paid",
- "ad",
- "ads",
- "advertising",
- "sponsored",
- "promotion",
- ];
-
- const paidSources = [
- "google ads",
- "googleads",
- "bing ads",
- "facebook ads",
- "instagram ads",
- "twitter ads",
- "linkedin ads",
- "tiktok ads",
- "youtube ads",
- "pinterest ads",
- ];
-
- return (
- paidMediums.some((pm) => lowerMedium.includes(pm)) || paidSources.some((ps) => lowerSource.includes(ps))
- );
-}
diff --git a/client/src/lib/import/mappers/tracker-utils.ts b/client/src/lib/import/mappers/tracker-utils.ts
deleted file mode 100644
index b1a11a12c..000000000
--- a/client/src/lib/import/mappers/tracker-utils.ts
+++ /dev/null
@@ -1,262 +0,0 @@
-// Duplicated from server/src/services/tracker/utils.ts and getChannel.ts for client-side import processing
-
-import {
- emailAppIds,
- getMediumType,
- getSourceType,
- isMobileAppId,
- isPaidTraffic,
- newsAppIds,
- productivityAppIds,
- searchAppIds,
- shoppingAppIds,
- socialAppIds,
- videoAppIds,
-} from "./tracker-const";
-
-// UTM and URL parameter parsing utilities
-export function getUTMParams(querystring: string): Record<string, string> {
- const params: Record<string, string> = {};
-
- if (!querystring) return params;
-
- try {
- const searchParams = new URLSearchParams(querystring);
-
- // Extract UTM parameters
- for (const [key, value] of searchParams.entries()) {
- if (key.startsWith("utm_") || key === "gclid" || key === "gad_source") {
- params[key.toLowerCase()] = value.toLowerCase();
- }
- }
- } catch (e) {
- console.error("Error parsing query string:", e);
- }
-
- return params;
-}
-
-// Parse all URL parameters from querystring
-export function getAllUrlParams(querystring: string): Record<string, string> {
- const params: Record<string, string> = {};
-
- if (!querystring) return params;
-
- // If querystring starts with ?, remove it
- const cleanQuerystring = querystring.startsWith("?") ? querystring.substring(1) : querystring;
-
- try {
- const searchParams = new URLSearchParams(cleanQuerystring);
- // Extract all parameters
- for (const [key, value] of searchParams.entries()) {
- params[key.toLowerCase()] = value;
- }
- } catch (e) {
- console.error("Error parsing query string for URL parameters:", e);
- }
-
- return params;
-}
-
-// Clear referrer if it's from the same domain
-export function clearSelfReferrer(referrer: string, hostname: string): string {
- if (!referrer || !hostname) return referrer;
-
- try {
- const referrerUrl = new URL(referrer);
- if (referrerUrl.hostname === hostname) {
- // Internal navigation, clear the referrer
- return "";
- }
- } catch (e) {
- // Invalid URL, return original referrer
- }
-
- return referrer;
-}
-
-// Channel detection logic
-
-function getMobileAppCategory(appId: string): { type: string; isPaid: boolean } | null {
- const appIdLower = appId.toLowerCase();
-
- if (socialAppIds.some((id) => appIdLower.includes(id))) {
- return { type: "Organic Social", isPaid: false };
- }
-
- if (videoAppIds.some((id) => appIdLower.includes(id))) {
- return { type: "Organic Video", isPaid: false };
- }
-
- if (searchAppIds.some((id) => appIdLower.includes(id))) {
- return { type: "Organic Search", isPaid: false };
- }
-
- if (emailAppIds.some((id) => appIdLower.includes(id))) {
- return { type: "Email", isPaid: false };
- }
-
- if (shoppingAppIds.some((id) => appIdLower.includes(id))) {
- return { type: "Organic Shopping", isPaid: false };
- }
-
- if (newsAppIds.some((id) => appIdLower.includes(id))) {
- return { type: "News", isPaid: false };
- }
-
- if (productivityAppIds.some((id) => appIdLower.includes(id))) {
- return { type: "Productivity", isPaid: false };
- }
-
- return null;
-}
-
-function getDomainFromReferrer(referrer: string): string {
- if (!referrer) return "$direct";
-
- try {
- const url = new URL(referrer);
- return url.hostname;
- } catch (e) {
- return "$direct";
- }
-}
-
-function isSelfReferral(referringDomain: string, hostname: string): boolean {
- if (!referringDomain || !hostname) return false;
-
- const stripWww = (domain: string) => domain.replace(/^www\./, "");
- const refDomain = stripWww(referringDomain);
- const currentDomain = stripWww(hostname);
-
- return refDomain === currentDomain || refDomain.endsWith("." + currentDomain);
-}
-
-export function getChannel(referrer: string, querystring: string, hostname?: string): string {
- const utmParams = getUTMParams(querystring);
- const referringDomain = getDomainFromReferrer(referrer);
-
- const selfReferral = hostname ? isSelfReferral(referringDomain, hostname) : false;
-
- const utmSource = utmParams["utm_source"] || "";
- const utmMedium = utmParams["utm_medium"] || "";
- const utmCampaign = utmParams["utm_campaign"] || "";
- const gclid = utmParams["gclid"] || "";
- const gadSource = utmParams["gad_source"] || "";
-
- if (utmSource && isMobileAppId(utmSource)) {
- const appCategory = getMobileAppCategory(utmSource);
- if (appCategory) {
- return appCategory.isPaid ? "Paid " + appCategory.type.split(" ")[1] : appCategory.type;
- }
- }
-
- if (!referrer && !utmSource && !utmMedium && !utmCampaign && !gclid && !gadSource) {
- return selfReferral ? "Internal" : "Direct";
- }
-
- const sourceType = getSourceType(utmSource || referringDomain);
- const mediumType = getMediumType(utmMedium);
- const isPaid = isPaidTraffic(utmMedium, utmSource) || gclid !== "" || gadSource !== "";
-
- // Cross Network
- if (utmCampaign === "cross-network") return "Cross-Network";
-
- // Direct traffic
- if (
- (referringDomain === "$direct" || (!referrer && !selfReferral)) &&
- !utmMedium &&
- (!utmSource || utmSource === "direct" || utmSource === "(direct)")
- ) {
- return "Direct";
- }
-
- // Paid channels
- if (isPaid) {
- switch (sourceType) {
- case "search":
- return "Paid Search";
- case "social":
- return "Paid Social";
- case "video":
- return "Paid Video";
- case "shopping":
- return "Paid Shopping";
- default:
- switch (mediumType) {
- case "social":
- return "Paid Social";
- case "video":
- return "Paid Video";
- case "display":
- case "cpm":
- return "Display";
- case "cpc":
- return "Paid Search";
- case "influencer":
- return "Paid Influencer";
- case "audio":
- return "Paid Audio";
- default:
- return "Paid Unknown";
- }
- }
- }
-
- // Organic channels
- switch (sourceType) {
- case "search":
- return "Organic Search";
- case "social":
- return "Organic Social";
- case "video":
- return "Organic Video";
- case "shopping":
- return "Organic Shopping";
- case "email":
- return "Email";
- case "sms":
- return "SMS";
- case "news":
- return "News";
- case "productivity":
- return "Productivity";
- }
-
- // Medium-based detection
- switch (mediumType) {
- case "social":
- return "Organic Social";
- case "video":
- return "Organic Video";
- case "affiliate":
- return "Affiliate";
- case "referral":
- return "Referral";
- case "display":
- return "Display";
- case "audio":
- return "Audio";
- case "push":
- return "Push";
- case "influencer":
- return "Influencer";
- case "content":
- return "Content";
- case "event":
- return "Event";
- case "email":
- return "Email";
- }
-
- // Campaign-based detection
- if (/video/.test(utmCampaign)) return "Organic Video";
- if (/shop|shopping/.test(utmCampaign)) return "Organic Shopping";
- if (/influencer|creator|sponsored/.test(utmCampaign)) return "Influencer";
- if (/event|conference|webinar/.test(utmCampaign)) return "Event";
- if (/social|facebook|twitter|instagram|linkedin/.test(utmCampaign)) return "Organic Social";
-
- if (referringDomain && referringDomain !== "$direct" && !selfReferral) return "Referral";
-
- return "Unknown";
-}
diff --git a/client/src/lib/import/mappers/umami.ts b/client/src/lib/import/mappers/umami.ts
deleted file mode 100644
index 5b8c98de3..000000000
--- a/client/src/lib/import/mappers/umami.ts
+++ /dev/null
@@ -1,207 +0,0 @@
-// Duplicated from server/src/services/import/mappings/umami.ts for client-side import processing
-
-import { z } from "zod";
-import { UmamiEvent, RybbitEvent } from "../types";
-import { getAllUrlParams, clearSelfReferrer, getChannel } from "./tracker-utils";
-
-export const umamiHeaders = [
- undefined,
- "session_id",
- undefined,
- undefined,
- "hostname",
- "browser",
- "os",
- "device",
- "screen",
- "language",
- "country",
- "region",
- "city",
- "url_path",
- "url_query",
- undefined,
- undefined,
- undefined,
- undefined,
- undefined,
- "referrer_path",
- "referrer_query",
- "referrer_domain",
- "page_title",
- undefined,
- undefined,
- undefined,
- undefined,
- undefined,
- undefined,
- "event_type",
- "event_name",
- undefined,
- "distinct_id",
- "created_at",
- undefined,
-];
-
-export class UmamiImportMapper {
- private static readonly browserMap: Record<string, string> = {
- chrome: "Chrome",
- opera: "Opera",
- crios: "Mobile Chrome",
- firefox: "Firefox",
- facebook: "Facebook",
- safari: "Safari",
- ios: "Mobile Safari",
- "ios-webview": "Mobile Safari",
- "edge-chromium": "Edge",
- samsung: "Samsung Internet",
- yandexbrowser: "Yandex",
- "edge-ios": "Edge",
- "chromium-webview": "Chrome WebView",
- fxios: "Mobile Firefox",
- edge: "Edge",
- };
-
- private static readonly browserSchema = z
- .string()
- .max(30)
- .transform((browser) => {
- const key = browser.toLowerCase();
- return UmamiImportMapper.browserMap[key] ?? browser;
- });
-
- private static readonly osMap: Record<string, string> = {
- "windows 10": "Windows",
- "windows 7": "Windows",
- "windows server 2003": "Windows",
- "mac os": "macOS",
- ios: "iOS",
- "android os": "Android",
- linux: "Linux",
- "chrome os": "Chrome OS",
- };
-
- private static readonly osSchema = z
- .string()
- .max(25)
- .transform((os) => {
- const key = os.toLowerCase();
- return UmamiImportMapper.osMap[key] ?? os;
- });
-
- private static readonly deviceMap: Record<string, "Desktop" | "Mobile"> = {
- laptop: "Desktop",
- desktop: "Desktop",
- mobile: "Mobile",
- tablet: "Mobile",
- };
-
- private static readonly deviceSchema = z
- .string()
- .max(20)
- .transform((device) => {
- const key = device.toLowerCase();
- return UmamiImportMapper.deviceMap[key] ?? device;
- });
-
- private static readonly umamiEventSchema = z.object({
- session_id: z.string().uuid(),
-
- hostname: z.string().max(253),
- browser: UmamiImportMapper.browserSchema,
- os: UmamiImportMapper.osSchema,
- device: UmamiImportMapper.deviceSchema,
- screen: z
- .string()
- .regex(/^\d{1,5}x\d{1,5}$/)
- .or(z.literal("")),
- language: z.string().max(35),
- country: z
- .string()
- .regex(/^[A-Z]{2}$/)
- .or(z.literal("")),
- region: z
- .string()
- .regex(/^[A-Z]{2}-[A-Z0-9]{1,3}$/)
- .or(z.literal("")),
- city: z.string().max(60),
-
- url_path: z.string().max(2048),
- url_query: z
- .string()
- .max(2048)
- .transform((querystring) => (querystring ? `?${querystring}` : "")),
- referrer_path: z.string().max(2048),
- referrer_query: z
- .string()
- .max(2048)
- .transform((querystring) => (querystring ? `?${querystring}` : "")),
- referrer_domain: z
- .string()
- .max(253)
- .transform((url) => (url ? `https://${url}` : "")),
- page_title: z.string().max(512),
-
- event_type: z.enum(["1", "2"]),
- event_name: z.string().max(256),
- distinct_id: z.string().max(64),
- created_at: z.string().regex(/^\d{4}-(0[1-9]|1[0-2])-(0[1-9]|[12]\d|3[01]) ([01]\d|2[0-3]):([0-5]\d):([0-5]\d)$/),
- });
-
- static transform(
- events: UmamiEvent[],
- site: string,
- importId: string,
- onError?: (error: { row: number; message: string }) => void
- ): RybbitEvent[] {
- return events.reduce<RybbitEvent[]>((acc, event, index) => {
- const parsed = UmamiImportMapper.umamiEventSchema.safeParse(event);
- if (!parsed.success) {
- if (onError) {
- onError({
- row: index + 1,
- message: `Validation failed: ${parsed.error.errors.map((e) => e.message).join(", ")}`,
- });
- }
- return acc;
- }
-
- const data = parsed.data;
- const referrer = data.referrer_domain + data.referrer_path + data.referrer_query;
- const [screenWidth, screenHeight] = data.screen ? data.screen.split("x") : ["0", "0"];
-
- acc.push({
- site_id: Number(site),
- timestamp: data.created_at,
- session_id: data.session_id,
- user_id: data.distinct_id,
- hostname: data.hostname,
- pathname: data.url_path,
- querystring: data.url_query,
- url_parameters: getAllUrlParams(data.url_query),
- page_title: data.page_title,
- referrer: clearSelfReferrer(referrer, data.hostname.replace(/^www\./, "")),
- channel: getChannel(referrer, data.url_query, data.hostname),
- browser: data.browser,
- browser_version: "",
- operating_system: data.os,
- operating_system_version: event.os === "Windows 10" ? "10" : event.os === "Windows 7" ? "7" : "",
- language: data.language,
- country: data.country,
- region: data.region,
- city: data.city,
- lat: 0,
- lon: 0,
- screen_width: parseInt(screenWidth, 10),
- screen_height: parseInt(screenHeight, 10),
- device_type: data.device,
- type: data.event_type === "1" ? "pageview" : "custom_event",
- event_name: data.event_type === "1" ? "" : data.event_name,
- props: {},
- import_id: importId,
- });
-
- return acc;
- }, []);
- }
-}
diff --git a/client/src/lib/import/quota-checker.ts b/client/src/lib/import/quota-checker.ts
deleted file mode 100644
index 90f852abb..000000000
--- a/client/src/lib/import/quota-checker.ts
+++ /dev/null
@@ -1,83 +0,0 @@
-// Client-side quota checking for import validation
-// Based on server/src/services/import/importQuotaChecker.ts
-
-import { DateTime } from "luxon";
-import { ImportQuotaInfo } from "./types";
-
-export class ClientQuotaChecker {
- private monthlyUsage: Map<string, number>;
- private readonly monthlyLimit: number;
- private readonly historicalWindowMonths: number;
- private readonly oldestAllowedMonth: string;
-
- constructor(quotaInfo: ImportQuotaInfo) {
- this.monthlyUsage = new Map(Object.entries(quotaInfo.monthlyUsage));
- this.monthlyLimit = quotaInfo.monthlyLimit;
- this.historicalWindowMonths = quotaInfo.historicalWindowMonths;
-
- const oldestAllowedDate = DateTime.utc().minus({ months: quotaInfo.historicalWindowMonths }).startOf("month");
- this.oldestAllowedMonth = oldestAllowedDate.toFormat("yyyyMM");
- }
-
- canImportEvent(timestamp: string): boolean {
- if (this.monthlyLimit === Infinity) {
- return true;
- }
-
- const dt = DateTime.fromFormat(timestamp, "yyyy-MM-dd HH:mm:ss", { zone: "utc" });
- if (!dt.isValid) {
- console.warn("Invalid timestamp format:", timestamp);
- return false;
- }
-
- const month = dt.toFormat("yyyyMM");
-
- // Check if event is within historical window
- if (month < this.oldestAllowedMonth) {
- return false;
- }
-
- const used = this.monthlyUsage.get(month) || 0;
- if (used >= this.monthlyLimit) {
- return false;
- }
-
- // Update usage tracker
- this.monthlyUsage.set(month, used + 1);
- return true;
- }
-
- getSummary(): {
- totalMonthsInWindow: number;
- monthsAtCapacity: number;
- monthsWithSpace: number;
- oldestAllowedMonth: string;
- } {
- if (this.monthlyLimit === Infinity) {
- return {
- totalMonthsInWindow: this.historicalWindowMonths,
- monthsAtCapacity: 0,
- monthsWithSpace: this.historicalWindowMonths,
- oldestAllowedMonth: this.oldestAllowedMonth,
- };
- }
-
- let monthsAtCapacity = 0;
- for (const usage of this.monthlyUsage.values()) {
- if (usage >= this.monthlyLimit) {
- monthsAtCapacity++;
- }
- }
-
- return {
- totalMonthsInWindow: this.historicalWindowMonths,
- monthsAtCapacity,
- monthsWithSpace: this.historicalWindowMonths - monthsAtCapacity,
- oldestAllowedMonth: this.oldestAllowedMonth,
- };
- }
-
- getMonthlyUsage(): Record<string, number> {
- return Object.fromEntries(this.monthlyUsage);
- }
-}
diff --git a/client/src/lib/import/types.ts b/client/src/lib/import/types.ts
index bbc65dd01..292e8ddd6 100644
--- a/client/src/lib/import/types.ts
+++ b/client/src/lib/import/types.ts
@@ -1,35 +1,5 @@
// Types for client-side CSV import system
-
-export interface RybbitEvent {
- site_id: number;
- timestamp: string;
- session_id: string;
- user_id: string;
- hostname: string;
- pathname: string;
- querystring: string;
- url_parameters: Record<string, string>;
- page_title: string;
- referrer: string;
- channel: string;
- browser: string;
- browser_version: string;
- operating_system: string;
- operating_system_version: string;
- language: string;
- country: string;
- region: string;
- city: string;
- lat: number;
- lon: number;
- screen_width: number;
- screen_height: number;
- device_type: string;
- type: string;
- event_name: string;
- props: Record<string, unknown>;
- import_id: string;
-}
+// Client parses CSV, server transforms and validates
export interface UmamiEvent {
session_id: string;
@@ -64,9 +34,6 @@ export type WorkerMessageToWorker =
platform: "umami";
startDate?: string;
endDate?: string;
- monthlyLimit: number;
- historicalWindowMonths: number;
- monthlyUsage: Record<string, number>;
}
| {
type: "CANCEL";
@@ -81,7 +48,7 @@ export type WorkerMessageToMain =
}
| {
type: "CHUNK_READY";
- events: RybbitEvent[];
+ events: UmamiEvent[]; // Raw CSV rows, not transformed
chunkIndex: number;
}
| {
@@ -97,18 +64,9 @@ export type WorkerMessageToMain =
error?: unknown;
};
-// Quota information from server
-export interface ImportQuotaInfo {
- monthlyLimit: number;
- historicalWindowMonths: number;
- monthlyUsage: Record<string, number>; // { "202401": 5000, "202402": 7500, ... }
- currentMonthUsage: number;
- organizationId: string;
-}
-
-// Batch import request
+// Batch import request (client sends raw rows to server)
export interface BatchImportRequest {
- events: RybbitEvent[];
+ events: UmamiEvent[]; // Raw Umami CSV rows
importId: string;
batchIndex: number;
totalBatches: number;
@@ -138,7 +96,7 @@ export interface ImportProgress {
// Failed batch info for retry
export interface FailedBatch {
batchIndex: number;
- events: RybbitEvent[];
+ events: UmamiEvent[]; // Raw rows, not transformed
error: string;
retryCount: number;
}
diff --git a/client/src/workers/csv-import.worker.ts b/client/src/workers/csv-import.worker.ts
index 4f1e6f19b..55dd61c4b 100644
--- a/client/src/workers/csv-import.worker.ts
+++ b/client/src/workers/csv-import.worker.ts
@@ -1,35 +1,68 @@
-// Web Worker for client-side CSV parsing and transformation
+// Web Worker for client-side CSV parsing only
// This worker runs in a separate thread to avoid blocking the main UI thread
+// Transformation happens on the server
import Papa from "papaparse";
import { DateTime } from "luxon";
-import type { WorkerMessageToWorker, WorkerMessageToMain, UmamiEvent, RybbitEvent } from "@/lib/import/types";
-import { UmamiImportMapper, umamiHeaders } from "@/lib/import/mappers/umami";
-import { ClientQuotaChecker } from "@/lib/import/quota-checker";
+import type { WorkerMessageToWorker, WorkerMessageToMain, UmamiEvent } from "@/lib/import/types";
-const CHUNK_SIZE = 5000; // Number of events per batch sent to main thread
+const CHUNK_SIZE = 5000; // Number of rows per batch sent to main thread
const PROGRESS_UPDATE_INTERVAL = 1000; // Update progress every 1000 rows
-let quotaChecker: ClientQuotaChecker | null = null;
-let currentBatch: RybbitEvent[] = [];
+let currentBatch: UmamiEvent[] = [];
let chunkIndex = 0;
let totalParsed = 0;
let totalSkipped = 0;
let totalErrors = 0;
let errorDetails: Array<{ row: number; message: string }> = [];
-let siteId: number;
-let importId: string;
-let platform: "umami";
let lastProgressUpdate = 0;
-// Date range filter
+// Date range filter (client-side optimization to reduce server load)
let startDate: DateTime | null = null;
let endDate: DateTime | null = null;
+// Umami CSV header mapping
+const umamiHeaders = [
+ undefined,
+ "session_id",
+ undefined,
+ undefined,
+ "hostname",
+ "browser",
+ "os",
+ "device",
+ "screen",
+ "language",
+ "country",
+ "region",
+ "city",
+ "url_path",
+ "url_query",
+ undefined,
+ undefined,
+ undefined,
+ undefined,
+ undefined,
+ "referrer_path",
+ "referrer_query",
+ "referrer_domain",
+ "page_title",
+ undefined,
+ undefined,
+ undefined,
+ undefined,
+ undefined,
+ undefined,
+ "event_type",
+ "event_name",
+ undefined,
+ "distinct_id",
+ "created_at",
+ undefined,
+];
+
function createDateRangeFilter(startDateStr?: string, endDateStr?: string) {
- startDate = startDateStr
- ? DateTime.fromFormat(startDateStr, "yyyy-MM-dd", { zone: "utc" }).startOf("day")
- : null;
+ startDate = startDateStr ? DateTime.fromFormat(startDateStr, "yyyy-MM-dd", { zone: "utc" }).startOf("day") : null;
endDate = endDateStr ? DateTime.fromFormat(endDateStr, "yyyy-MM-dd", { zone: "utc" }).endOf("day") : null;
if (startDate && !startDate.isValid) {
@@ -82,42 +115,29 @@ function sendProgress() {
}
function handleParsedRow(row: unknown, rowIndex: number) {
- // Skip rows with missing or invalid dates
const umamiEvent = row as UmamiEvent;
- if (!umamiEvent.created_at) {
- totalSkipped++;
- return;
- }
-
- // Apply user-specified date range filter
- if (!isDateInRange(umamiEvent.created_at)) {
- totalSkipped++;
- return;
- }
- // Check per-month quota
- if (quotaChecker && !quotaChecker.canImportEvent(umamiEvent.created_at)) {
+ // Skip rows with missing created_at (required field)
+ if (!umamiEvent.created_at) {
totalSkipped++;
return;
}
- // Transform the event
- const transformed = UmamiImportMapper.transform([umamiEvent], siteId.toString(), importId, (error) => {
- totalErrors++;
- if (errorDetails.length < 100) {
- // Limit error collection
- errorDetails.push({ row: rowIndex, message: error.message });
+ // Apply optional date range filter (client-side optimization)
+ if (startDate || endDate) {
+ if (!isDateInRange(umamiEvent.created_at)) {
+ totalSkipped++;
+ return;
}
- });
+ }
- if (transformed.length > 0) {
- currentBatch.push(transformed[0]);
- totalParsed++;
+ // Add to batch (no transformation, send raw row to server)
+ currentBatch.push(umamiEvent);
+ totalParsed++;
- // Send batch when it reaches chunk size
- if (currentBatch.length >= CHUNK_SIZE) {
- sendChunk();
- }
+ // Send batch when it reaches chunk size
+ if (currentBatch.length >= CHUNK_SIZE) {
+ sendChunk();
}
// Send progress update periodically
@@ -132,12 +152,14 @@ function parseCSV(file: File) {
Papa.parse<UmamiEvent>(file, {
header: true,
- skipEmptyLines: true,
+ dynamicTyping: true, // Auto-convert strings to numbers/booleans
+ skipEmptyLines: "greedy", // Skip all empty lines (improved)
+ delimiter: "", // Auto-detect delimiter (comma, tab, semicolon, etc.)
transformHeader: (header, index) => {
- // Use predefined Umami headers mapping
+ // Map Umami CSV column positions to field names
return umamiHeaders[index] || header;
},
- step: (results) => {
+ step: results => {
if (results.data) {
handleParsedRow(results.data, rowIndex);
rowIndex++;
@@ -145,9 +167,10 @@ function parseCSV(file: File) {
if (results.errors && results.errors.length > 0) {
totalErrors++;
if (errorDetails.length < 100) {
+ // Limit error collection to avoid memory issues
errorDetails.push({
row: rowIndex,
- message: results.errors.map((e) => e.message).join(", "),
+ message: results.errors.map(e => e.message).join(", "),
});
}
}
@@ -159,20 +182,6 @@ function parseCSV(file: File) {
// Send final progress
sendProgress();
- // Check if we skipped events due to quota
- const quotaSummary = quotaChecker?.getSummary();
- if (quotaSummary && quotaSummary.monthsAtCapacity > 0 && totalSkipped > 0) {
- const quotaErrorMessage =
- `${totalSkipped} events exceeded monthly quotas or fell outside the ${quotaSummary.totalMonthsInWindow}-month historical window. ` +
- `${quotaSummary.monthsAtCapacity} of ${quotaSummary.totalMonthsInWindow} months are at full capacity. ` +
- `Try importing newer data or upgrade your plan for higher monthly quotas.`;
-
- errorDetails.push({
- row: 0,
- message: quotaErrorMessage,
- });
- }
-
// Send completion message
const message: WorkerMessageToMain = {
type: "COMPLETE",
@@ -183,7 +192,7 @@ function parseCSV(file: File) {
};
self.postMessage(message);
},
- error: (error) => {
+ error: error => {
const message: WorkerMessageToMain = {
type: "ERROR",
message: error.message,
@@ -209,21 +218,7 @@ self.onmessage = (event: MessageEvent<WorkerMessageToWorker>) => {
errorDetails = [];
lastProgressUpdate = 0;
- // Set up parameters
- siteId = message.siteId;
- importId = message.importId;
- platform = message.platform;
-
- // Create quota checker
- quotaChecker = new ClientQuotaChecker({
- monthlyLimit: message.monthlyLimit,
- historicalWindowMonths: message.historicalWindowMonths,
- monthlyUsage: message.monthlyUsage,
- currentMonthUsage: 0,
- organizationId: "",
- });
-
- // Set up date range filter
+ // Set up date range filter (optional client-side optimization)
createDateRangeFilter(message.startDate, message.endDate);
// Start parsing
diff --git a/server/src/api/sites/batchImportEvents.ts b/server/src/api/sites/batchImportEvents.ts
index 0ee4cbfd1..264da7b41 100644
--- a/server/src/api/sites/batchImportEvents.ts
+++ b/server/src/api/sites/batchImportEvents.ts
@@ -3,7 +3,11 @@ import { z } from "zod";
import { getUserHasAdminAccessToSite } from "../../lib/auth-utils.js";
import { clickhouse } from "../../db/clickhouse/clickhouse.js";
import { updateImportProgress, updateImportStatus, getImportById } from "../../services/import/importStatusManager.js";
-import { RybbitEvent } from "../../services/import/mappings/rybbit.js";
+import { UmamiImportMapper, type UmamiEvent } from "../../services/import/mappings/umami.js";
+import { ImportQuotaTracker } from "../../services/import/importQuotaChecker.js";
+import { db } from "../../db/postgres/postgres.js";
+import { sites } from "../../db/postgres/schema.js";
+import { eq } from "drizzle-orm";
import { createServiceLogger } from "../../lib/logger/logger.js";
const logger = createServiceLogger("import:batch");
@@ -14,7 +18,7 @@ const batchImportRequestSchema = z
site: z.string().min(1),
}),
body: z.object({
- events: z.array(z.any()).min(1).max(10000), // Limit batch size to 10k events
+ events: z.array(z.any()).min(1).max(10000), // Limit batch size to 10k events, raw Umami rows
importId: z.string().uuid(),
batchIndex: z.number().int().min(0),
totalBatches: z.number().int().min(1),
@@ -75,26 +79,96 @@ export async function batchImportEvents(request: FastifyRequest<BatchImportReque
await updateImportStatus(importId, "processing");
}
+ // Get organization ID for quota checking
+ const [siteRecord] = await db
+ .select({ organizationId: sites.organizationId })
+ .from(sites)
+ .where(eq(sites.siteId, siteId))
+ .limit(1);
+
+ if (!siteRecord) {
+ logger.error({ siteId }, "Site not found");
+ return reply.status(404).send({ error: "Site not found" });
+ }
+
try {
- // Insert events into ClickHouse
+ // Create quota tracker for server-side quota checking
+ const quotaTracker = await ImportQuotaTracker.create(siteRecord.organizationId);
+
+ // Filter events based on quota availability
+ const eventsWithinQuota: UmamiEvent[] = [];
+ let skippedDueToQuota = 0;
+
+ for (const event of events as UmamiEvent[]) {
+ if (!event.created_at) {
+ continue; // Skip events without timestamp
+ }
+
+ if (quotaTracker.canImportEvent(event.created_at)) {
+ eventsWithinQuota.push(event);
+ } else {
+ skippedDueToQuota++;
+ }
+ }
+
+ // If all events were skipped due to quota, fail the batch
+ if (eventsWithinQuota.length === 0 && events.length > 0) {
+ const quotaSummary = quotaTracker.getSummary();
+ const errorMessage =
+ `All ${events.length} events in batch ${batchIndex} exceeded monthly quotas or fell outside the ${quotaSummary.totalMonthsInWindow}-month historical window. ` +
+ `${quotaSummary.monthsAtCapacity} of ${quotaSummary.totalMonthsInWindow} months are at full capacity.`;
+
+ logger.warn({ importId, batchIndex, skippedDueToQuota }, errorMessage);
+
+ // Update import status to failed if this was the only batch or early in the import
+ if (batchIndex === 0 || totalBatches === 1) {
+ await updateImportStatus(importId, "failed", errorMessage);
+ }
+
+ return reply.status(400).send({
+ success: false,
+ error: "Quota exceeded",
+ message: errorMessage,
+ });
+ }
+
+ // Transform events using server-side mapper (single source of truth)
+ const transformedEvents = UmamiImportMapper.transform(eventsWithinQuota, site, importId);
+
+ if (transformedEvents.length === 0) {
+ logger.warn({ importId, batchIndex }, "No valid events after transformation");
+ return reply.send({
+ success: true,
+ importedCount: 0,
+ message: `Batch ${batchIndex + 1}/${totalBatches}: No valid events`,
+ });
+ }
+
+ // Insert transformed events into ClickHouse
await clickhouse.insert({
table: "events",
- values: events as RybbitEvent[],
+ values: transformedEvents,
format: "JSONEachRow",
});
logger.info(
- { importId, batchIndex, eventCount: events.length, totalBatches },
+ {
+ importId,
+ batchIndex,
+ eventCount: transformedEvents.length,
+ skippedDueToQuota,
+ totalBatches,
+ },
"Batch inserted successfully"
);
// Update progress
- await updateImportProgress(importId, events.length);
+ await updateImportProgress(importId, transformedEvents.length);
return reply.send({
success: true,
- importedCount: events.length,
- message: `Batch ${batchIndex + 1}/${totalBatches} imported successfully`,
+ importedCount: transformedEvents.length,
+ message: `Batch ${batchIndex + 1}/${totalBatches} imported successfully${skippedDueToQuota > 0 ? ` (${skippedDueToQuota} events skipped due to quota)` : ""}`,
});
} catch (insertError) {
logger.error({ importId, batchIndex, error: insertError }, "Failed to insert batch");
diff --git a/server/src/api/sites/getImportQuota.ts b/server/src/api/sites/getImportQuota.ts
deleted file mode 100644
index 48339a18e..000000000
--- a/server/src/api/sites/getImportQuota.ts
+++ /dev/null
@@ -1,75 +0,0 @@
-import { FastifyReply, FastifyRequest } from "fastify";
-import { z } from "zod";
-import { getUserHasAdminAccessToSite } from "../../lib/auth-utils.js";
-import { ImportQuotaTracker } from "../../services/import/importQuotaChecker.js";
-import { db } from "../../db/postgres/postgres.js";
-import { sites } from "../../db/postgres/schema.js";
-import { eq } from "drizzle-orm";
-
-const getImportQuotaRequestSchema = z
- .object({
- params: z.object({
- site: z.string().min(1),
- }),
- })
- .strict();
-
-type GetImportQuotaRequest = {
- Params: z.infer<typeof getImportQuotaRequestSchema.shape.params>;
-};
-
-export async function getImportQuota(request: FastifyRequest<GetImportQuotaRequest>, reply: FastifyReply) {
- try {
- const parsed = getImportQuotaRequestSchema.safeParse({
- params: request.params,
- });
-
- if (!parsed.success) {
- return reply.status(400).send({ error: "Validation error" });
- }
-
- const { site } = parsed.data.params;
- const siteId = Number(site);
-
- const userHasAccess = await getUserHasAdminAccessToSite(request, site);
- if (!userHasAccess) {
- return reply.status(403).send({ error: "Forbidden" });
- }
-
- // Get organization ID from site
- const [siteRecord] = await db
- .select({ organizationId: sites.organizationId })
- .from(sites)
- .where(eq(sites.siteId, siteId))
- .limit(1);
-
- if (!siteRecord) {
- return reply.status(404).send({ error: "Site not found" });
- }
-
- // Create quota tracker to get quota information
- const quotaTracker = await ImportQuotaTracker.create(siteRecord.organizationId);
- const summary = quotaTracker.getSummary();
-
- // Get current month usage from the monthly usage map
- const currentMonth = new Date().toISOString().slice(0, 7).replace("-", ""); // YYYYMM format
- const monthlyUsageMap = await quotaTracker["queryMonthlyUsage"](
- [siteId],
- new Date(new Date().setMonth(new Date().getMonth() - summary.totalMonthsInWindow)).toISOString().split("T")[0]
- );
-
- return reply.send({
- data: {
- monthlyLimit: quotaTracker["monthlyLimit"],
- historicalWindowMonths: summary.totalMonthsInWindow,
- monthlyUsage: Object.fromEntries(monthlyUsageMap),
- currentMonthUsage: monthlyUsageMap.get(currentMonth) || 0,
- organizationId: siteRecord.organizationId,
- oldestAllowedMonth: summary.oldestAllowedMonth,
- },
- });
- } catch (error) {
- console.error("Error fetching import quota:", error);
- return reply.status(500).send({ error: "Internal server error" });
- }
-}
diff --git a/server/src/index.ts b/server/src/index.ts
index a45c36160..c1631ce89 100644
--- a/server/src/index.ts
+++ b/server/src/index.ts
@@ -90,7 +90,6 @@ import { selectGSCProperty } from "./api/gsc/selectProperty.js";
import { getSiteImports } from "./api/sites/getSiteImports.js";
import { importSiteData } from "./api/sites/importSiteData.js";
import { deleteSiteImport } from "./api/sites/deleteSiteImport.js";
-import { getImportQuota } from "./api/sites/getImportQuota.js";
import { batchImportEvents } from "./api/sites/batchImportEvents.js";
import { getJobQueue } from "./services/import/queues/jobQueueFactory.js";
@@ -367,8 +366,7 @@ server.post("/api/import-site-data/:site", {
}, importSiteData);
server.get("/api/get-site-imports/:site", getSiteImports);
server.delete("/api/delete-site-import/:site/:importId", deleteSiteImport);
-// Client-side import endpoints
-server.get("/api/import-quota/:site", getImportQuota);
+// Client-side import endpoint (parses CSV, server transforms)
server.post("/api/batch-import-events/:site", batchImportEvents);
// Administrative