Skip to content

Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.

New remap sources migration that's case insensitive (#4771)

Uku Taht9d06d45e454c
diff --git a/priv/ingest_repo/migrations/20241104082248_remap_sources_v2.exs b/priv/ingest_repo/migrations/20241104082248_remap_sources_v2.exs
new file mode 100644
index 0000000000..3995af1994
--- /dev/null
+++ b/priv/ingest_repo/migrations/20241104082248_remap_sources_v2.exs
@@ -0,0 +1,245 @@
+defmodule Plausible.IngestRepo.Migrations.RemapSourcesV2 do
+ use Ecto.Migration
+
+ @mappings %{
+ # UTM sources
+ "fb" => "Facebook",
+ "fb-ads" => "Facebook",
+ "fbads" => "Facebook",
+ "fbad" => "Facebook",
+ "facebook-ads" => "Facebook",
+ "facebook_ads" => "Facebook",
+ "fcb" => "Facebook",
+ "facebook_ad" => "Facebook",
+ "facebook_feed_ad" => "Facebook",
+ "ig" => "Instagram",
+ "yt" => "Youtube",
+ "yt-ads" => "Youtube",
+ "reddit-ads" => "Reddit",
+ "google_ads" => "Google",
+ "google-ads" => "Google",
+ "googleads" => "Google",
+ "gads" => "Google",
+ "google ads" => "Google",
+ "adwords" => "Google",
+ "twitter-ads" => "Twitter",
+ "tiktokads" => "TikTok",
+ "tik.tok" => "TikTok",
+ "perplexity" => "Perplexity",
+ "linktree" => "Linktree",
+
+ # Referrers
+ "android-app://com.reddit.frontpage" => "Reddit",
+ "perplexity.ai" => "Perplexity",
+ "search.brave.com" => "Brave",
+ "yandex.com.tr" => "Yandex",
+ "yandex.kz" => "Yandex",
+ "ya.ru" => "Yandex",
+ "yandex.uz" => "Yandex",
+ "yandex.fr" => "Yandex",
+ "yandex.eu" => "Yandex",
+ "yandex.tm" => "Yandex",
+ "discord.com" => "Discord",
+ "t.me" => "Telegram",
+ "webk.telegram.org" => "Telegram",
+ "sogou.com" => "Sogou",
+ "m.sogou.com" => "Sogou",
+ "wap.sogou.com" => "Sogou",
+ "canary.discord.com" => "Discord",
+ "ptb.discord.com" => "Discord",
+ "discordapp.com" => "Discord",
+ "linktr.ee" => "Linktree",
+ "baidu.com" => "Baidu",
+ "statics.teams.cdn.office.net" => "Microsoft Teams",
+ "ntp.msn.com" => "Bing",
+ "en.wikipedia.org" => "Wikipedia",
+ "en.m.wikipedia.org" => "Wikipedia",
+ "de.wikipedia.org" => "Wikipedia",
+ "de.m.wikipedia.org" => "Wikipedia",
+ "fr.wikipedia.org" => "Wikipedia",
+ "ru.wikipedia.org" => "Wikipedia",
+ "fr.m.wikipedia.org" => "Wikipedia",
+ "es.wikipedia.org" => "Wikipedia",
+ "ja.wikipedia.org" => "Wikipedia",
+ "nl.wikipedia.org" => "Wikipedia",
+ "ru.m.wikipedia.org" => "Wikipedia",
+ "da.m.wikipedia.org" => "Wikipedia",
+ "no.wikipedia.org" => "Wikipedia",
+ "es.m.wikipedia.org" => "Wikipedia",
+ "it.wikipedia.org" => "Wikipedia",
+ "nl.m.wikipedia.org" => "Wikipedia",
+ "da.wikipedia.org" => "Wikipedia",
+ "sv.wikipedia.org" => "Wikipedia",
+ "it.m.wikipedia.org" => "Wikipedia",
+ "zh.wikipedia.org" => "Wikipedia",
+ "sv.m.wikipedia.org" => "Wikipedia",
+ "no.m.wikipedia.org" => "Wikipedia",
+ "fi.m.wikipedia.org" => "Wikipedia",
+ "fi.wikipedia.org" => "Wikipedia",
+ "ca.wikipedia.org" => "Wikipedia",
+ "ja.m.wikipedia.org" => "Wikipedia",
+ "pt.wikipedia.org" => "Wikipedia",
+ "pl.wikipedia.org" => "Wikipedia",
+ "pt.m.wikipedia.org" => "Wikipedia",
+ "cs.wikipedia.org" => "Wikipedia",
+ "zh.m.wikipedia.org" => "Wikipedia",
+ "ca.m.wikipedia.org" => "Wikipedia",
+ "pl.m.wikipedia.org" => "Wikipedia",
+ "cs.m.wikipedia.org" => "Wikipedia",
+ "is.wikipedia.org" => "Wikipedia",
+ "ko.wikipedia.org" => "Wikipedia",
+ "uk.wikipedia.org" => "Wikipedia",
+ "is.m.wikipedia.org" => "Wikipedia",
+ "he.m.wikipedia.org" => "Wikipedia",
+ "tr.wikipedia.org" => "Wikipedia",
+ "he.wikipedia.org" => "Wikipedia",
+ "id.m.wikipedia.org" => "Wikipedia",
+ "tr.m.wikipedia.org" => "Wikipedia",
+ "et.wikipedia.org" => "Wikipedia",
+ "fa.m.wikipedia.org" => "Wikipedia",
+ "uk.m.wikipedia.org" => "Wikipedia",
+ "simple.wikipedia.org" => "Wikipedia",
+ "ko.m.wikipedia.org" => "Wikipedia",
+ "id.wikipedia.org" => "Wikipedia",
+ "hr.m.wikipedia.org" => "Wikipedia",
+ "simple.m.wikipedia.org" => "Wikipedia",
+ "vi.wikipedia.org" => "Wikipedia",
+ "el.wikipedia.org" => "Wikipedia",
+ "hr.wikipedia.org" => "Wikipedia",
+ "sk.wikipedia.org" => "Wikipedia",
+ "hu.wikipedia.org" => "Wikipedia",
+ "hu.m.wikipedia.org" => "Wikipedia",
+ "fa.wikipedia.org" => "Wikipedia",
+ "el.m.wikipedia.org" => "Wikipedia",
+ "arz.m.wikipedia.org" => "Wikipedia",
+ "th.m.wikipedia.org" => "Wikipedia",
+ "ta.m.wikipedia.org" => "Wikipedia",
+ "ga.wikipedia.org" => "Wikipedia",
+ "et.m.wikipedia.org" => "Wikipedia",
+ "vi.m.wikipedia.org" => "Wikipedia",
+ "ro.wikipedia.org" => "Wikipedia",
+ "ro.m.wikipedia.org" => "Wikipedia",
+ "ms.m.wikipedia.org" => "Wikipedia",
+ "bs.m.wikipedia.org" => "Wikipedia",
+ "az.m.wikipedia.org" => "Wikipedia",
+ "bg.m.wikipedia.org" => "Wikipedia",
+ "nn.wikipedia.org" => "Wikipedia",
+ "bg.wikipedia.org" => "Wikipedia",
+ "ml.m.wikipedia.org" => "Wikipedia",
+ "bn.m.wikipedia.org" => "Wikipedia",
+ "sl.wikipedia.org" => "Wikipedia",
+ "nn.m.wikipedia.org" => "Wikipedia",
+ "sk.m.wikipedia.org" => "Wikipedia",
+ "ms.wikipedia.org" => "Wikipedia",
+ "uz.wikipedia.org" => "Wikipedia",
+ "th.wikipedia.org" => "Wikipedia",
+ "sr.m.wikipedia.org" => "Wikipedia",
+ "hi.m.wikipedia.org" => "Wikipedia",
+ "eu.wikipedia.org" => "Wikipedia",
+ "uz.m.wikipedia.org" => "Wikipedia",
+ "sr.wikipedia.org" => "Wikipedia",
+ "lv.wikipedia.org" => "Wikipedia",
+ "la.wikipedia.org" => "Wikipedia",
+ "sl.m.wikipedia.org" => "Wikipedia",
+ "arz.wikipedia.org" => "Wikipedia",
+ "ta.wikipedia.org" => "Wikipedia",
+ "ka.m.wikipedia.org" => "Wikipedia",
+ "ga.m.wikipedia.org" => "Wikipedia",
+ "lt.wikipedia.org" => "Wikipedia",
+ "lv.m.wikipedia.org" => "Wikipedia",
+ "kk.m.wikipedia.org" => "Wikipedia",
+ "lt.m.wikipedia.org" => "Wikipedia",
+ "ar.wikipedia.org" => "Wikipedia",
+ "eo.wikipedia.org" => "Wikipedia",
+ "sw.m.wikipedia.org" => "Wikipedia",
+ "sh.wikipedia.org" => "Wikipedia",
+ "bs.wikipedia.org" => "Wikipedia",
+ "ml.wikipedia.org" => "Wikipedia",
+ "hy.wikipedia.org" => "Wikipedia",
+ "ka.wikipedia.org" => "Wikipedia",
+ "hi.wikipedia.org" => "Wikipedia",
+ "la.m.wikipedia.org" => "Wikipedia",
+ "bn.wikipedia.org" => "Wikipedia",
+ "ur.m.wikipedia.org" => "Wikipedia",
+ "sh.m.wikipedia.org" => "Wikipedia",
+ "az.wikipedia.org" => "Wikipedia",
+ "si.m.wikipedia.org" => "Wikipedia",
+ "sq.wikipedia.org" => "Wikipedia",
+ "zh-yue.wikipedia.org" => "Wikipedia",
+ "ckb.m.wikipedia.org" => "Wikipedia",
+ "kn.wikipedia.org" => "Wikipedia",
+ "lij.wikipedia.org" => "Wikipedia",
+ "fy.wikipedia.org" => "Wikipedia",
+ "lij.m.wikipedia.org" => "Wikipedia",
+ "hy.m.wikipedia.org" => "Wikipedia",
+ "mn.m.wikipedia.org" => "Wikipedia",
+ "ar.m.wikipedia.org" => "Wikipedia",
+ "tl.wikipedia.org" => "Wikipedia",
+ "eu.m.wikipedia.org" => "Wikipedia",
+ "fo.wikipedia.org" => "Wikipedia",
+ "mn.wikipedia.org" => "Wikipedia",
+ "mr.wikipedia.org" => "Wikipedia",
+ "zh-classical.wikipedia.org" => "Wikipedia",
+ "cy.wikipedia.org" => "Wikipedia",
+ "olo.wikipedia.org" => "Wikipedia",
+ "te.m.wikipedia.org" => "Wikipedia",
+ "mk.wikipedia.org" => "Wikipedia",
+ "dz.wikipedia.org" => "Wikipedia",
+ "as.m.wikipedia.org" => "Wikipedia",
+ "szl.m.wikipedia.org" => "Wikipedia",
+ "oc.wikipedia.org" => "Wikipedia",
+ "rw.m.wikipedia.org" => "Wikipedia",
+ "tl.m.wikipedia.org" => "Wikipedia",
+ "si.wikipedia.org" => "Wikipedia",
+ "nostalgia.wikipedia.org" => "Wikipedia",
+ "lrc.wikipedia.org" => "Wikipedia",
+ "eo.m.wikipedia.org" => "Wikipedia",
+ "ky.wikipedia.org" => "Wikipedia",
+ "new.wikipedia.org" => "Wikipedia",
+ "be.wikipedia.org" => "Wikipedia",
+ "bcl.m.wikipedia.org" => "Wikipedia",
+ "sq.m.wikipedia.org" => "Wikipedia",
+ "am.wikipedia.org" => "Wikipedia",
+ "nds-nl.m.wikipedia.org" => "Wikipedia",
+ "gu.m.wikipedia.org" => "Wikipedia",
+ "bjn.wikipedia.org" => "Wikipedia",
+ "pa.m.wikipedia.org" => "Wikipedia",
+ "sa.m.wikipedia.org" => "Wikipedia",
+ "tt.wikipedia.org" => "Wikipedia",
+ "qu.wikipedia.org" => "Wikipedia",
+ "be-tarask.wikipedia.org" => "Wikipedia",
+ "mg.m.wikipedia.org" => "Wikipedia",
+ "dga.m.wikipedia.org" => "Wikipedia",
+ "bat-smg.wikipedia.org" => "Wikipedia",
+ "sw.wikipedia.org" => "Wikipedia",
+ "wuu.wikipedia.org" => "Wikipedia",
+ "ne.m.wikipedia.org" => "Wikipedia",
+ "yi.m.wikipedia.org" => "Wikipedia",
+ "vec.wikipedia.org" => "Wikipedia",
+ "be-tarask.m.wikipedia.org" => "Wikipedia",
+ "crh.wikipedia.org" => "Wikipedia"
+ }
+
+ def up do
+ {keys, values} = Enum.unzip(@mappings)
+
+ events_sql = """
+ ALTER TABLE events_v2
+ UPDATE referrer_source = transform(lower(referrer_source), {$0:Array(String)}, {$1:Array(String)})
+ WHERE lower(referrer_source) IN {$0:Array(String)}
+ """
+
+ sessions_sql = """
+ ALTER TABLE sessions_v2
+ UPDATE referrer_source = transform(lower(referrer_source), {$0:Array(String)}, {$1:Array(String)})
+ WHERE lower(referrer_source) IN {$0:Array(String)}
+ """
+
+ execute(fn -> repo().query!(events_sql, [keys, values]) end)
+ execute(fn -> repo().query!(sessions_sql, [keys, values]) end)
+ end
+
+ def down do
+ raise "irreversible"
+ end
+end