Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.
TypeScript analyticsanalytics-dashboardclickhouseelixirgoogle-analyticsmarketingopen-source-analyticsphoenixplausible-analyticspostgresqlprivacyprivacy-friendlyself-hostedsimple-analyticsstatisticstailwindcssweb-analyticswebsite-analyticswebsite-statswebsite-tracking
Channel and source data updates (#4599)
Uku Tahtc3a06caa9787
diff --git a/.codespellignore b/.codespellignoreindex da4c972dae..cb20144e71 100644--- a/.codespellignore+++ b/.codespellignore@@ -5,4 +5,4 @@ Tahttahtrefererreferers-+staticsdiff --git a/lib/plausible/application.ex b/lib/plausible/application.exindex 04fea5be71..32bc19658a 100644--- a/lib/plausible/application.ex+++ b/lib/plausible/application.ex@@ -108,7 +108,7 @@ defmodule Plausible.Application dosetup_geolocation()Location.load_all()- Plausible.Ingestion.Acquisition.init()+ Plausible.Ingestion.Source.init()Plausible.Geo.await_loader()Supervisor.start_link(List.flatten(children), opts)diff --git a/lib/plausible/imported/google_analytics4.ex b/lib/plausible/imported/google_analytics4.exindex 2118d57972..4cc4de2187 100644--- a/lib/plausible/imported/google_analytics4.ex+++ b/lib/plausible/imported/google_analytics4.ex@@ -174,7 +174,7 @@ defmodule Plausible.Imported.GoogleAnalytics4 dosite_id: site_id,import_id: import_id,date: get_date(row),- source: row.dimensions |> Map.fetch!("sessionSource") |> parse_referrer(),+ source: row.dimensions |> Map.fetch!("sessionSource") |> parse_source(),referrer: nil,# Only `source` exists in GA4 APIutm_source: nil,@@ -343,14 +343,13 @@ defmodule Plausible.Imported.GoogleAnalytics4 dodefp default_if_missing(value, default) when value in @missing_values, do: defaultdefp default_if_missing(value, _default), do: value- defp parse_referrer(nil), do: nil- defp parse_referrer("(direct)"), do: nil- defp parse_referrer("google"), do: "Google"- defp parse_referrer("bing"), do: "Bing"- defp parse_referrer("duckduckgo"), do: "DuckDuckGo"+ defp parse_source(nil), do: nil+ defp parse_source("(direct)"), do: nil+ defp parse_source("google"), do: "Google"+ defp parse_source("bing"), do: "Bing"+ defp parse_source("duckduckgo"), do: "DuckDuckGo"- defp parse_referrer(ref) do- RefInspector.parse("https://" <> ref)- |> PlausibleWeb.RefInspector.parse()+ defp parse_source(ref) do+ Plausible.Ingestion.Source.parse("https://" <> ref)endenddiff --git a/lib/plausible/ingestion/acquisition.ex b/lib/plausible/ingestion/acquisition.exindex d6024bf0e1..d72dc11cba 100644--- a/lib/plausible/ingestion/acquisition.ex+++ b/lib/plausible/ingestion/acquisition.ex@@ -1,37 +1,53 @@defmodule Plausible.Ingestion.Acquisition do- @moduledoc false+ @moduledoc """+ This module is responsible for figuring out acquisition channel from event referrer_source.++ Acquisition channel is the marketing channel where people come from and convert and help+ users to understand and improve their marketing flow.++ Note it uses priv/ga4-source-categories.csv as a source, which comes from https://support.google.com/analytics/answer/9756891?hl=en.++ Notable differences from GA4 that have been implemented just for Plausible:+ 1. The @custom_source_categories module attribute contains a list of custom source categories that we have manually+ added based on our own judgement and user feedback. For example we treat AI tools (ChatGPT, Perplexity) as search engines.+ 2. Google is in a privileged position to analyze paid traffic from within their own network. The biggest use-case is auto-tagged adwords campaigns.+ We do our best by categorizing as paid search when source is Google and the url has `gclid` parameter. Same for source Bing and `msclkid` url parameter.+ 3. The @paid_sources module attribute in Plausible.Ingestion.Source contains a list of utm_sources that we will automatically categorize as paid traffic+ regardless of the medium. Examples are `yt-ads`, `facebook_ad`, `adwords`, etc. See also: Plausible.Ingestion.Source.paid_source?/1+ """+@external_resource "priv/ga4-source-categories.csv"+ @custom_source_categories [+ {"hacker news", "SOURCE_CATEGORY_SOCIAL"},+ {"yahoo!", "SOURCE_CATEGORY_SEARCH"},+ {"gmail", "SOURCE_CATEGORY_EMAIL"},+ {"telegram", "SOURCE_CATEGORY_SOCIAL"},+ {"slack", "SOURCE_CATEGORY_SOCIAL"},+ {"producthunt", "SOURCE_CATEGORY_SOCIAL"},+ {"github", "SOURCE_CATEGORY_SOCIAL"},+ {"steamcommunity.com", "SOURCE_CATEGORY_SOCIAL"},+ {"statics.teams.cdn.office.net", "SOURCE_CATEGORY_SOCIAL"},+ {"vkontakte", "SOURCE_CATEGORY_SOCIAL"},+ {"threads", "SOURCE_CATEGORY_SOCIAL"},+ {"ecosia", "SOURCE_CATEGORY_SEARCH"},+ {"perplexity", "SOURCE_CATEGORY_SEARCH"},+ {"brave", "SOURCE_CATEGORY_SEARCH"},+ {"chatgpt.com", "SOURCE_CATEGORY_SEARCH"},+ {"temu.com", "SOURCE_CATEGORY_SHOPPING"},+ {"discord", "SOURCE_CATEGORY_SOCIAL"},+ {"sogou", "SOURCE_CATEGORY_SEARCH"},+ {"microsoft teams", "SOURCE_CATEGORY_SOCIAL"}+ ]@source_categories Application.app_dir(:plausible, "priv/ga4-source-categories.csv")|> File.read!()|> NimbleCSV.RFC4180.parse_string(skip_headers: false)|> Enum.map(fn [source, category] -> {source, category} end)+ |> then(&(@custom_source_categories ++ &1))|> Enum.into(%{})- def init() do- :ets.new(__MODULE__, [- :named_table,- :set,- :public,- {:read_concurrency, true}- ])-- [{"referers.yml", map}] = RefInspector.Database.list(:default)-- Enum.flat_map(map, fn {_, entries} ->- Enum.map(entries, fn {_, _, _, _, _, _, name} ->- :ets.insert(__MODULE__, {String.downcase(name), name})- end)- end)- end-- def find_mapping(source) do- case :ets.lookup(__MODULE__, source) do- [{_, name}] -> name- _ -> source- end- end-def get_channel(request, source) do+ source = source && String.downcase(source)+cond docross_network?(request) -> "Cross-network"paid_shopping?(request, source) -> "Paid Shopping"@@ -44,7 +60,7 @@ defmodule Plausible.Ingestion.Acquisition doorganic_social?(request, source) -> "Organic Social"organic_video?(request, source) -> "Organic Video"search_source?(source) -> "Organic Search"- email?(request) -> "Email"+ email?(request, source) -> "Email"affiliates?(request) -> "Affiliates"audio?(request) -> "Audio"sms?(request) -> "SMS"@@ -55,30 +71,32 @@ defmodule Plausible.Ingestion.Acquisition doenddefp cross_network?(request) do- String.contains?(request.query_params["utm_campaign"] || "", "cross-network")+ String.contains?(query_param(request, "utm_campaign"), "cross-network")enddefp paid_shopping?(request, source) do- (shopping_source?(source) or shopping_campaign?(request.query_params["utm_campaign"])) and- paid_medium?(request.query_params["utm_medium"])+ (shopping_source?(source) or shopping_campaign?(request)) and paid_medium?(request)enddefp paid_search?(request, source) do- (search_source?(source) and paid_medium?(request.query_params["utm_medium"])) or- (source == "Google" and !!request.query_params["gclid"]) or- (source == "Bing" and !!request.query_params["msclkid"])+ (search_source?(source) and paid_medium?(request)) or+ (search_source?(source) and paid_source?(request)) or+ (source == "google" and !!request.query_params["gclid"]) or+ (source == "bing" and !!request.query_params["msclkid"])enddefp paid_social?(request, source) do- social_source?(source) and paid_medium?(request.query_params["utm_medium"])+ (social_source?(source) and paid_medium?(request)) or+ (social_source?(source) and paid_source?(request))enddefp paid_video?(request, source) do- video_source?(source) and paid_medium?(request.query_params["utm_medium"])+ (video_source?(source) and paid_medium?(request)) or+ (video_source?(source) and paid_source?(request))enddefp display?(request) do- request.query_params["utm_medium"] in [+ query_param(request, "utm_medium") in ["display","banner","expandable",@@ -88,16 +106,16 @@ defmodule Plausible.Ingestion.Acquisition doenddefp paid_other?(request) do- paid_medium?(request.query_params["utm_medium"])+ paid_medium?(request)enddefp organic_shopping?(request, source) do- shopping_source?(source) or shopping_campaign?(request.query_params["utm_campaign"])+ shopping_source?(source) or shopping_campaign?(request)enddefp organic_social?(request, source) dosocial_source?(source) or- request.query_params["utm_medium"] in [+ query_param(request, "utm_medium") in ["social","social-network","social-media",@@ -108,71 +126,88 @@ defmodule Plausible.Ingestion.Acquisition doenddefp organic_video?(request, source) do- video_source?(source) or String.contains?(request.query_params["utm_medium"] || "", "video")+ video_source?(source) or String.contains?(query_param(request, "utm_medium"), "video")enddefp referral?(request, source) do- request.query_params["utm_medium"] in ["referral", "app", "link"] or+ query_param(request, "utm_medium") in ["referral", "app", "link"] or!!sourceend- @email_tags ["email", "e-mail", "e_mail", "e mail"]- defp email?(request) do- String.contains?(request.query_params["utm_source"] || "", @email_tags) or- String.contains?(request.query_params["utm_medium"] || "", @email_tags)+ @email_tags ["email", "e-mail", "e_mail", "e mail", "newsletter"]+ defp email?(request, source) do+ email_source?(source) or+ String.contains?(query_param(request, "utm_source"), @email_tags) or+ String.contains?(query_param(request, "utm_medium"), @email_tags)enddefp affiliates?(request) do- request.query_params["utm_medium"] == "affiliate"+ query_param(request, "utm_medium") == "affiliate"enddefp audio?(request) do- request.query_params["utm_medium"] == "audio"+ query_param(request, "utm_medium") == "audio"enddefp sms?(request) do- request.query_params["utm_source"] == "sms" or- request.query_params["utm_medium"] == "sms"+ query_param(request, "utm_source") == "sms" or+ query_param(request, "utm_medium") == "sms"enddefp mobile_push_notifications?(request, source) do- medium = request.query_params["utm_medium"] || ""+ medium = query_param(request, "utm_medium")String.ends_with?(medium, "push") orString.contains?(medium, ["mobile", "notification"]) orsource == "firebase"end- # # Helper functions for source and medium checksdefp shopping_source?(nil), do: falsedefp shopping_source?(source) do- @source_categories[String.downcase(source)] == "SOURCE_CATEGORY_SHOPPING"- end-- defp shopping_campaign?(campaign_name) do- Regex.match?(~r/^(.*(([^a-df-z]|^)shop|shopping).*)$/, campaign_name || "")+ @source_categories[source] == "SOURCE_CATEGORY_SHOPPING"enddefp search_source?(nil), do: falsedefp search_source?(source) do- @source_categories[String.downcase(source)] == "SOURCE_CATEGORY_SEARCH"+ @source_categories[source] == "SOURCE_CATEGORY_SEARCH"enddefp social_source?(nil), do: falsedefp social_source?(source) do- @source_categories[String.downcase(source)] == "SOURCE_CATEGORY_SOCIAL"+ @source_categories[source] == "SOURCE_CATEGORY_SOCIAL"enddefp video_source?(nil), do: falsedefp video_source?(source) do- @source_categories[String.downcase(source)] == "SOURCE_CATEGORY_VIDEO"+ @source_categories[source] == "SOURCE_CATEGORY_VIDEO"+ end++ defp email_source?(nil), do: false++ defp email_source?(source) do+ @source_categories[source] == "SOURCE_CATEGORY_EMAIL"+ end++ defp shopping_campaign?(request) do+ campaign_name = query_param(request, "utm_campaign")+ Regex.match?(~r/^(.*(([^a-df-z]|^)shop|shopping).*)$/, campaign_name)+ end++ defp paid_medium?(request) do+ medium = query_param(request, "utm_medium")+ Regex.match?(~r/^(.*cp.*|ppc|retargeting|paid.*)$/, medium)+ end++ defp paid_source?(request) do+ query_param(request, "utm_source")+ |> Plausible.Ingestion.Source.paid_source?()end- defp paid_medium?(medium) do- Regex.match?(~r/^(.*cp.*|ppc|retargeting|paid.*)$/, medium || "")+ defp query_param(request, name) do+ String.downcase(request.query_params[name] || "")endenddiff --git a/lib/plausible/ingestion/event.ex b/lib/plausible/ingestion/event.exindex f177ee9408..4c2c887692 100644--- a/lib/plausible/ingestion/event.ex+++ b/lib/plausible/ingestion/event.ex@@ -251,14 +251,13 @@ defmodule Plausible.Ingestion.Event doenddefp put_referrer(%__MODULE__{} = event, _context) do- ref = parse_referrer(event.request.uri, event.request.referrer)- source = get_referrer_source(event.request, ref)+ source = Plausible.Ingestion.Source.resolve(event.request)channel = Plausible.Ingestion.Acquisition.get_channel(event.request, source)update_session_attrs(event, %{channel: channel,referrer_source: source,- referrer: clean_referrer(ref)+ referrer: Plausible.Ingestion.Source.format_referrer(event.request.referrer)})end@@ -392,40 +391,6 @@ defmodule Plausible.Ingestion.Event doeventend- defp parse_referrer(_uri, _referrer_str = nil), do: nil-- defp parse_referrer(uri, referrer_str) do- referrer_uri = URI.parse(referrer_str)-- if Request.sanitize_hostname(referrer_uri.host) !== Request.sanitize_hostname(uri.host) &&- referrer_uri.host !== "localhost" do- RefInspector.parse(referrer_str)- end- end-- defp get_referrer_source(request, ref) do- tagged_source =- request.query_params["utm_source"] ||- request.query_params["source"] ||- request.query_params["ref"]-- if tagged_source do- Plausible.Ingestion.Acquisition.find_mapping(tagged_source)- else- PlausibleWeb.RefInspector.parse(ref)- end- end-- defp clean_referrer(nil), do: nil-- defp clean_referrer(ref) do- uri = URI.parse(ref.referer)-- if PlausibleWeb.RefInspector.right_uri?(uri) do- PlausibleWeb.RefInspector.format_referrer(uri)- end- end-defp parse_user_agent(%Request{user_agent: user_agent}) when is_binary(user_agent) doPlausible.Cache.Adapter.get(:user_agents, user_agent, fn ->UAInspector.parse(user_agent)diff --git a/lib/plausible/ingestion/source.ex b/lib/plausible/ingestion/source.exnew file mode 100644index 0000000000..2866db1914--- /dev/null+++ b/lib/plausible/ingestion/source.ex@@ -0,0 +1,147 @@+defmodule Plausible.Ingestion.Source do+ @moduledoc """+ Resolves the `source` dimension from a combination of `referer` header and either `utm_source`, `source`, or `ref` query parameter.++ """+ alias Plausible.Ingestion.Request++ @external_resource "priv/custom_sources.json"+ @custom_sources Application.app_dir(:plausible, "priv/custom_sources.json")+ |> File.read!()+ |> Jason.decode!()++ @paid_sources Map.keys(@custom_sources)+ |> Enum.filter(&String.ends_with?(&1, ["ads", "ad"]))+ |> then(&["adwords" | &1])+ |> MapSet.new()++ def init() do+ :ets.new(__MODULE__, [+ :named_table,+ :set,+ :public,+ {:read_concurrency, true}+ ])++ [{"referers.yml", map}] = RefInspector.Database.list(:default)++ Enum.each(map, fn {_, entries} ->+ Enum.each(entries, fn {_, _, _, _, _, _, name} ->+ :ets.insert(__MODULE__, {String.downcase(name), name})+ end)+ end)++ Enum.each(@custom_sources, fn {key, val} ->+ :ets.insert(__MODULE__, {key, val})+ :ets.insert(__MODULE__, {String.downcase(val), val})+ end)+ end++ def paid_source?(source) do+ MapSet.member?(@paid_sources, source)+ end++ @doc """+ Resolves the source of a session based on query params and the `Referer` header.++ When a query parameter like `utm_source` is present, it will be prioritized over the `Referer` header. When the URL does not contain a source tag, we fall+ back to using `Referer` to determine the source. This module also takes care of certain transformations to make the data more useful for the user:+ 1. The RefInspector library is used to categorize referrers into "known" sources. For example, when the referrer is google.com or google.co.uk,+ it will always be stored as "Google" which is more useful for marketers.+ 2. On top of the standard RefInspector behaviour, we also keep a list of `custom_sources.json` which extends it with referrers that we have seen in the wild.+ For example, Wikipedia has many domains that need to be combined into a single known source. These could all in theory be [upstreamed](https://github.com/snowplow-referer-parser/referer-parser).+ 3. When a known source is supplied in utm_source (or source, ref) query parameter, we merge it with our known sources in a case-insensitive manner.+ 4. Our list of `custom_sources.json` also contains some commonly used utm_source shorthands for certain sources. URL tagging is a mess, and we can never do it+ perfectly, but at least we're making an effort for the most commonly used ones. For example, `ig -> Instagram` and `adwords -> Google`.++ ### Examples:++ iex> alias Plausible.Ingestion.{Source, Request}+ iex> base_request = %Request{uri: URI.parse("https://plausible.io")}+ iex> Source.resolve(%{base_request | referrer: "https://google.com"}) # Known referrer from RefInspector+ "Google"+ iex> Source.resolve(%{base_request | query_params: %{"utm_source" => "google"}}) # Known source from RefInspector supplied as downcased utm_source by user+ "Google"+ iex> Source.resolve(%{base_request | query_params: %{"utm_source" => "GOOGLE"}}) # Known source from RefInspector supplied as uppercased utm_source by user+ "Google"+ iex> Source.resolve(%{base_request | referrer: "https://en.m.wikipedia.org"}) # Known referrer from custom_sources.json+ "Wikipedia"+ iex> Source.resolve(%{base_request | query_params: %{"utm_source" => "wikipedia"}}) # Known source from custom_sources.json supplied as downcased utm_source by user+ "Wikipedia"+ iex> Source.resolve(%{base_request | query_params: %{"utm_source" => "ig"}}) # Known utm_source from custom_sources.json+ "Instagram"+ iex> Source.resolve(%{base_request | referrer: "https://www.markosaric.com"}) # Unknown source, it is just stored as the domain name+ "markosaric.com"+ """+ def resolve(request) do+ tagged_source =+ request.query_params["utm_source"] ||+ request.query_params["source"] ||+ request.query_params["ref"]++ source =+ cond do+ tagged_source -> tagged_source+ has_referral?(request) -> parse(request.referrer)+ true -> nil+ end++ find_mapping(source)+ end++ def parse(ref) do+ case RefInspector.parse(ref).source do+ :unknown ->+ uri = URI.parse(String.trim(ref))++ if valid_referrer?(uri) do+ format_referrer_host(uri)+ end++ source ->+ source+ end+ end++ def find_mapping(nil), do: nil++ def find_mapping(source) do+ case :ets.lookup(__MODULE__, String.downcase(source)) do+ [{_, name}] -> name+ _ -> source+ end+ end++ def format_referrer(nil), do: nil++ def format_referrer(referrer) do+ referrer_uri = URI.parse(referrer)++ if valid_referrer?(referrer_uri) do+ path = String.trim_trailing(referrer_uri.path || "", "/")+ format_referrer_host(referrer_uri) <> path+ end+ end++ defp valid_referrer?(%URI{host: host, scheme: scheme})+ when scheme in ["http", "https", "android-app"] and byte_size(host) > 0,+ do: true++ defp valid_referrer?(_), do: false++ defp has_referral?(%Request{referrer: nil}), do: nil++ defp has_referral?(%Request{referrer: referrer, uri: uri}) do+ referrer_uri = URI.parse(referrer)++ Request.sanitize_hostname(referrer_uri.host) !== Request.sanitize_hostname(uri.host) and+ referrer_uri.host !== "localhost"+ end++ defp format_referrer_host(uri) do+ protocol = if uri.scheme == "android-app", do: "android-app://", else: ""+ host = String.replace_prefix(uri.host, "www.", "")++ protocol <> host+ end+enddiff --git a/lib/plausible_web/plugs/favicon.ex b/lib/plausible_web/plugs/favicon.exindex e8dc9a5e20..da676dff0a 100644--- a/lib/plausible_web/plugs/favicon.ex+++ b/lib/plausible_web/plugs/favicon.ex@@ -31,11 +31,20 @@ defmodule PlausibleWeb.Favicon do@placeholder_icon_location "priv/placeholder_favicon.ico"@placeholder_icon File.read!(@placeholder_icon_location)+ @custom_icons %{+ "Brave" => "search.brave.com",+ "Sogou" => "sogou.com",+ "Wikipedia" => "en.wikipedia.org",+ "Discord" => "discord.com",+ "Perplexity" => "perplexity.ai",+ "Microsoft Teams" => "microsoft.com"+ }def init(_) dodomains =File.read!(Application.app_dir(:plausible, @referer_domains_file))|> Jason.decode!()+ |> Map.merge(@custom_icons)[favicon_domains: domains]enddiff --git a/lib/plausible_web/refinspector.ex b/lib/plausible_web/refinspector.exdeleted file mode 100644index 0bfbcb456b..0000000000--- a/lib/plausible_web/refinspector.ex+++ /dev/null@@ -1,37 +0,0 @@-defmodule PlausibleWeb.RefInspector do- def parse(nil), do: nil-- def parse(ref) do- case ref.source do- :unknown ->- uri = URI.parse(String.trim(ref.referer))-- if right_uri?(uri) do- format_referrer_host(uri)- end-- source ->- source- end- end-- def format_referrer(uri) do- path = String.trim_trailing(uri.path || "", "/")- format_referrer_host(uri) <> path- end-- def right_uri?(%URI{host: nil}), do: false-- def right_uri?(%URI{host: host, scheme: scheme})- when scheme in ["http", "https", "android-app"] and byte_size(host) > 0,- do: true-- def right_uri?(_), do: false-- defp format_referrer_host(uri) do- protocol = if uri.scheme == "android-app", do: "android-app://", else: ""- host = String.replace_prefix(uri.host, "www.", "")-- protocol <> host- end-enddiff --git a/priv/custom_sources.json b/priv/custom_sources.jsonnew file mode 100644index 0000000000..16bc9f0d4a--- /dev/null+++ b/priv/custom_sources.json@@ -0,0 +1,215 @@+{+ "android-app://com.reddit.frontpage":"Reddit",+ "baidu.com":"Baidu",+ "discord.com":"Discord",+ "discordapp.com":"Discord",+ "linktr.ee":"Linktree",+ "m.sogou.com":"Sogou",+ "ntp.msn.com":"Bing",+ "perplexity.ai":"Perplexity",+ "ptb.discord.com":"Discord",+ "search.brave.com":"Brave",+ "sogou.com":"Sogou",+ "statics.teams.cdn.office.net":"Microsoft Teams",+ "t.me":"Telegram",+ "wap.sogou.com":"Sogou",+ "ya.ru":"Yandex",+ "yandex.com.tr":"Yandex",+ "yandex.eu":"Yandex",+ "yandex.fr":"Yandex",+ "yandex.kz":"Yandex",+ "yandex.tm":"Yandex",+ "yandex.uz":"Yandex",+ "fb": "Facebook",+ "fb-ads": "Facebook",+ "fbads": "Facebook",+ "fbad": "Facebook",+ "facebook-ads": "Facebook",+ "facebook_ads": "Facebook",+ "fcb": "Facebook",+ "facebook_ad": "Facebook",+ "facebook_feed_ad": "Facebook",+ "ig": "Instagram",+ "yt": "Youtube",+ "yt-ads": "Youtube",+ "reddit-ads": "Reddit",+ "google_ads": "Google",+ "google-ads": "Google",+ "googleads": "Google",+ "gads": "Google",+ "google ads": "Google",+ "adwords": "Google",+ "twitter-ads": "Twitter",+ "tiktokads": "TikTok",+ "tik.tok": "TikTok",+ "perplexity": "Perplexity",+ "linktree": "Linktree",+ "fo.wikipedia.org":"Wikipedia",+ "ga.wikipedia.org":"Wikipedia",+ "el.m.wikipedia.org":"Wikipedia",+ "eo.m.wikipedia.org":"Wikipedia",+ "ms.m.wikipedia.org":"Wikipedia",+ "nl.wikipedia.org":"Wikipedia",+ "dga.m.wikipedia.org":"Wikipedia",+ "th.wikipedia.org":"Wikipedia",+ "oc.wikipedia.org":"Wikipedia",+ "da.wikipedia.org":"Wikipedia",+ "pt.m.wikipedia.org":"Wikipedia",+ "szl.m.wikipedia.org":"Wikipedia",+ "be-tarask.wikipedia.org":"Wikipedia",+ "ta.m.wikipedia.org":"Wikipedia",+ "pa.m.wikipedia.org":"Wikipedia",+ "mn.wikipedia.org":"Wikipedia",+ "sv.m.wikipedia.org":"Wikipedia",+ "sk.wikipedia.org":"Wikipedia",+ "it.wikipedia.org":"Wikipedia",+ "el.wikipedia.org":"Wikipedia",+ "olo.wikipedia.org":"Wikipedia",+ "hi.m.wikipedia.org":"Wikipedia",+ "bn.m.wikipedia.org":"Wikipedia",+ "uz.wikipedia.org":"Wikipedia",+ "fr.m.wikipedia.org":"Wikipedia",+ "fa.wikipedia.org":"Wikipedia",+ "fi.wikipedia.org":"Wikipedia",+ "arz.m.wikipedia.org":"Wikipedia",+ "si.m.wikipedia.org":"Wikipedia",+ "bjn.wikipedia.org":"Wikipedia",+ "kn.wikipedia.org":"Wikipedia",+ "is.m.wikipedia.org":"Wikipedia",+ "nostalgia.wikipedia.org":"Wikipedia",+ "en.wikipedia.org":"Wikipedia",+ "nl.m.wikipedia.org":"Wikipedia",+ "nn.m.wikipedia.org":"Wikipedia",+ "bs.wikipedia.org":"Wikipedia",+ "sh.m.wikipedia.org":"Wikipedia",+ "vi.m.wikipedia.org":"Wikipedia",+ "ru.wikipedia.org":"Wikipedia",+ "tr.m.wikipedia.org":"Wikipedia",+ "he.wikipedia.org":"Wikipedia",+ "ta.wikipedia.org":"Wikipedia",+ "es.wikipedia.org":"Wikipedia",+ "si.wikipedia.org":"Wikipedia",+ "pl.wikipedia.org":"Wikipedia",+ "hu.wikipedia.org":"Wikipedia",+ "lij.m.wikipedia.org":"Wikipedia",+ "nn.wikipedia.org":"Wikipedia",+ "ko.m.wikipedia.org":"Wikipedia",+ "da.m.wikipedia.org":"Wikipedia",+ "zh.m.wikipedia.org":"Wikipedia",+ "vec.wikipedia.org":"Wikipedia",+ "ar.wikipedia.org":"Wikipedia",+ "bcl.m.wikipedia.org":"Wikipedia",+ "en.m.wikipedia.org":"Wikipedia",+ "sw.wikipedia.org":"Wikipedia",+ "la.m.wikipedia.org":"Wikipedia",+ "ur.m.wikipedia.org":"Wikipedia",+ "id.m.wikipedia.org":"Wikipedia",+ "crh.wikipedia.org":"Wikipedia",+ "sr.wikipedia.org":"Wikipedia",+ "sw.m.wikipedia.org":"Wikipedia",+ "ka.m.wikipedia.org":"Wikipedia",+ "lt.m.wikipedia.org":"Wikipedia",+ "fy.wikipedia.org":"Wikipedia",+ "ro.m.wikipedia.org":"Wikipedia",+ "hr.wikipedia.org":"Wikipedia",+ "mn.m.wikipedia.org":"Wikipedia",+ "pt.wikipedia.org":"Wikipedia",+ "it.m.wikipedia.org":"Wikipedia",+ "lv.m.wikipedia.org":"Wikipedia",+ "fa.m.wikipedia.org":"Wikipedia",+ "ja.wikipedia.org":"Wikipedia",+ "lv.wikipedia.org":"Wikipedia",+ "hu.m.wikipedia.org":"Wikipedia",+ "de.wikipedia.org":"Wikipedia",+ "uk.wikipedia.org":"Wikipedia",+ "ml.wikipedia.org":"Wikipedia",+ "te.m.wikipedia.org":"Wikipedia",+ "bg.wikipedia.org":"Wikipedia",+ "eu.wikipedia.org":"Wikipedia",+ "arz.wikipedia.org":"Wikipedia",+ "id.wikipedia.org":"Wikipedia",+ "mg.m.wikipedia.org":"Wikipedia",+ "sq.m.wikipedia.org":"Wikipedia",+ "ca.wikipedia.org":"Wikipedia",+ "sk.m.wikipedia.org":"Wikipedia",+ "az.wikipedia.org":"Wikipedia",+ "ru.m.wikipedia.org":"Wikipedia",+ "uz.m.wikipedia.org":"Wikipedia",+ "wuu.wikipedia.org":"Wikipedia",+ "hy.wikipedia.org":"Wikipedia",+ "la.wikipedia.org":"Wikipedia",+ "ca.m.wikipedia.org":"Wikipedia",+ "ckb.m.wikipedia.org":"Wikipedia",+ "tt.wikipedia.org":"Wikipedia",+ "gu.m.wikipedia.org":"Wikipedia",+ "lrc.wikipedia.org":"Wikipedia",+ "be-tarask.m.wikipedia.org":"Wikipedia",+ "no.m.wikipedia.org":"Wikipedia",+ "simple.m.wikipedia.org":"Wikipedia",+ "eu.m.wikipedia.org":"Wikipedia",+ "ne.m.wikipedia.org":"Wikipedia",+ "sr.m.wikipedia.org":"Wikipedia",+ "vi.wikipedia.org":"Wikipedia",+ "lt.wikipedia.org":"Wikipedia",+ "cs.m.wikipedia.org":"Wikipedia",+ "hy.m.wikipedia.org":"Wikipedia",+ "mr.wikipedia.org":"Wikipedia",+ "sv.wikipedia.org":"Wikipedia",+ "eo.wikipedia.org":"Wikipedia",+ "as.m.wikipedia.org":"Wikipedia",+ "is.wikipedia.org":"Wikipedia",+ "sh.wikipedia.org":"Wikipedia",+ "zh-classical.wikipedia.org":"Wikipedia",+ "nds-nl.m.wikipedia.org":"Wikipedia",+ "tl.m.wikipedia.org":"Wikipedia",+ "tr.wikipedia.org":"Wikipedia",+ "cs.wikipedia.org":"Wikipedia",+ "uk.m.wikipedia.org":"Wikipedia",+ "sq.wikipedia.org":"Wikipedia",+ "et.m.wikipedia.org":"Wikipedia",+ "hr.m.wikipedia.org":"Wikipedia",+ "bn.wikipedia.org":"Wikipedia",+ "sl.wikipedia.org":"Wikipedia",+ "th.m.wikipedia.org":"Wikipedia",+ "hi.wikipedia.org":"Wikipedia",+ "he.m.wikipedia.org":"Wikipedia",+ "bat-smg.wikipedia.org":"Wikipedia",+ "ml.m.wikipedia.org":"Wikipedia",+ "zh.wikipedia.org":"Wikipedia",+ "fi.m.wikipedia.org":"Wikipedia",+ "de.m.wikipedia.org":"Wikipedia",+ "be.wikipedia.org":"Wikipedia",+ "pl.m.wikipedia.org":"Wikipedia",+ "simple.wikipedia.org":"Wikipedia",+ "rw.m.wikipedia.org":"Wikipedia",+ "no.wikipedia.org":"Wikipedia",+ "ja.m.wikipedia.org":"Wikipedia",+ "yi.m.wikipedia.org":"Wikipedia",+ "ga.m.wikipedia.org":"Wikipedia",+ "ar.m.wikipedia.org":"Wikipedia",+ "canary.discord.com":"Discord",+ "sa.m.wikipedia.org":"Wikipedia",+ "ky.wikipedia.org":"Wikipedia",+ "es.m.wikipedia.org":"Wikipedia",+ "new.wikipedia.org":"Wikipedia",+ "lij.wikipedia.org":"Wikipedia",+ "zh-yue.wikipedia.org":"Wikipedia",+ "bg.m.wikipedia.org":"Wikipedia",+ "bs.m.wikipedia.org":"Wikipedia",+ "dz.wikipedia.org":"Wikipedia",+ "kk.m.wikipedia.org":"Wikipedia",+ "fr.wikipedia.org":"Wikipedia",+ "qu.wikipedia.org":"Wikipedia",+ "ka.wikipedia.org":"Wikipedia",+ "webk.telegram.org":"Telegram",+ "et.wikipedia.org":"Wikipedia",+ "ms.wikipedia.org":"Wikipedia",+ "az.m.wikipedia.org":"Wikipedia",+ "cy.wikipedia.org":"Wikipedia",+ "ro.wikipedia.org":"Wikipedia",+ "mk.wikipedia.org":"Wikipedia",+ "tl.wikipedia.org":"Wikipedia",+ "am.wikipedia.org":"Wikipedia",+ "ko.wikipedia.org":"Wikipedia",+ "sl.m.wikipedia.org":"Wikipedia"+}diff --git a/priv/referer_favicon_domains.json b/priv/referer_favicon_domains.jsonindex 0d3786d532..6430d7f773 100644--- a/priv/referer_favicon_domains.json+++ b/priv/referer_favicon_domains.json@@ -1 +1 @@-{"White Pages":"www.whitepages.com.au","QQ Mail":"mail.qq.com","eo":"eo.st","Toolbarhome":"www.toolbarhome.com","YouGoo":"www.yougoo.fr","Walhello":"www.walhello.info","Tixuma":"www.tixuma.de","Hyves":"hyves.nl","PriceRunner":"www.pricerunner.co.uk","Euroseek":"www.euroseek.com","Web.nl":"www.web.nl","360.cn":"so.360.cn","Seznam":"search.seznam.cz","Nigma":"nigma.ru","Wirtualna Polska":"szukaj.wp.pl","Google Product Search":"google.ac/products","Picsearch":"www.picsearch.com","Suchnase":"www.suchnase.de","WebSearch":"www.websearch.com","Lycos":"search.lycos.com","Conduit":"search.conduit.com","StackOverflow":"stackoverflow.com","GitHub":"github.com","FriendFeed":"friendfeed.com","Flickr":"flickr.com","Google+":"url.google.com","APOLL07":"apollo7.de","Douban":"douban.com","Pocket":"getpocket.com","Doubleclick":"ad.doubleclick.net","Exalead":"www.exalead.fr","Sharelook":"www.sharelook.fr","Adam Internet":"webmail.adam.com.au","Friendster":"friendster.com","2degrees":"webmail.2degreesbroadband.co.nz","ZEDO":"zedo.com","Outlook.com":"mail.live.com","1&1":"search.1and1.com","URL.ORGanizier":"www.url.org","Google":"support.google.com","Snapdo":"search.snapdo.com","Bebo":"bebo.com","Biglobe":"cgi.search.biglobe.ne.jp","1und1":"search.1und1.de","BlackPlanet":"blackplanet.com","Findwide":"search.findwide.com","Digg":"digg.com","Eurip":"www.eurip.com","SearchCanvas":"www.searchcanvas.com","Sapo":"pesquisa.sapo.pt","Yieldmo":"yieldmo.com","ABCsøk":"abcsolk.no","Gaia Online":"gaiaonline.com","blekko":"blekko.com","Vodafone":"webmail.vodafone.co.nz","qip":"search.qip.ru","Hooseek.com":"www.hooseek.com","Tumblr":"tumblr.com","GAIS":"gais.cs.ccu.edu.tw","AllTheWeb":"www.alltheweb.com","Terra":"buscador.terra.es","Buzznet":"buzznet.com","Searchy":"www.searchy.co.uk","Commander":"webmail.commander.net.au","Twingly":"www.twingly.com","Yandex":"yandex.ru","Najdi":"www.najdi.si","Sociomantic Labs":"sociomantic.com","Netlog":"netlog.com","Volny":"web.volny.cz","Criteo":"cas.jp.as.criteo.com","dmoz":"dmoz.org","Bing":"bing.com","Altavista":"www.altavista.com","Kataweb":"www.kataweb.it","Charter":"www.charter.net","Startpagina":"startgoogle.startpagina.nl","Search.ch":"www.search.ch","Uludag Sozluk":"uludagsozluk.com","Orkut":"orkut.com","UKR.net":"search.ukr.net","Sonico.com":"sonico.com","Comcast":"serach.comcast.net","Yandex Images":"images.yandex.ru","Vindex":"www.vindex.nl","Zoho":"mail.zoho.com","SteelHouse":"steelhousemedia.com","Geona":"geona.net","Excite":"search.excite.it","1.cz":"1.cz","Vimeo":"vimeo.com","Rubicon Project":"optimized-by.rubiconproject.com","Yasni":"www.yasni.de","Badoo":"badoo.com","Tiscali":"search.tiscali.it","Inbox.com":"inbox.com","Orange Webmail":"orange.fr/webmail","Blogpulse":"www.blogpulse.com","Delfi latvia":"smart.delfi.lv","iPrimus":"webmail.iprimus.com.au","Mail.ru":"my.mail.ru","Google News":"news.google.ac","Delicious":"delicious.com","Sonobi":"sonobi.com","Road Runner Search":"search.rr.com","Fast Browser Search":"www.fastbrowsersearch.com","WAYN":"wayn.com","Weibo":"weibo.com","Windows Live Spaces":"login.live.com","Clix":"pesquisa.clix.pt","T-Online":"suche.t-online.de","Ask Toolbar":"search.tb.ask.com","Flyingbird":"inspsearch.com","Yahoo!":"finance.yahoo.com","Alexa":"alexa.com","Jungle Key":"junglekey.com","Gmail":"mail.google.com","Optus Zoo":"webmail.optuszoo.com.au","Web.de":"suche.web.de","Odnoklassniki":"odnoklassniki.ru","GMX":"suche.gmx.net","Freshweather":"www.fresh-weather.com","Quora":"quora.com","AppNexus":"ib.adnxs.com","Onet":"szukaj.onet.pl","Geni":"geni.com","Naver Images":"image.search.naver.com","Qzone":"qzone.qq.com","Mozbot":"www.mozbot.fr","Adform":"adform.net","Blogdigger":"www.blogdigger.com","Netspace":"webmail.netspace.net.au","Apontador":"apontador.com.br","Jungle Spider":"www.jungle-spider.de","Mozo":"mozo.com.au","Zapmeta":"www.zapmeta.com","MySearch":"www.mysearch.com","X-recherche":"www.x-recherche.com","Lo.st":"lo.st","TrovaRapido":"www.trovarapido.com","Dodo":"webmail.dodo.com.au","Flix":"www.flix.de","Flashtalking":"flashtalking.com","Nasza-klasa.pl":"nk.pl","AOL Mail":"mail.aol.com","Virgilio":"ricerca.virgilio.it","Rambler":"nova.rambler.ru","Atlas":"searchatlas.centrum.cz","Austronaut":"www2.austronaut.at","Xanga":"xanga.com","vKruguDruzei.ru":"vkrugudruzei.ru","Friends Reunited":"friendsreunited.com","Nifty":"search.nifty.com","Plaxo":"plaxo.com","Sizmek":"bs.serving-sys.com","ONE by AOL":"nexage.com","Gomeo":"www.gomeo.com","BidSwitch":"bidswitch.net","Yahoo! Images":"image.yahoo.cn","ITU Sozluk":"itusozluk.com","Instagram":"instagram.com","AOL":"search.aol.com","Compuserve":"websearch.cs.com","Free":"search.free.fr","Reddit":"reddit.com","Metager2":"metager2.de","Tuenti":"tuenti.com","Rakuten":"websearch.rakuten.co.jp","126 Mail":"mail.126.com","Centrum":"serach.centrum.cz","Dalesearch":"www.dalesearch.com","Freecause":"search.freecause.com","Viadeo":"viadeo.com","Bing Images":"bing.com/images/search","Softonic":"search.softonic.com","ICQ":"www.icq.com","Gule Sider":"www.gulesider.no","Winamp":"search.winamp.com","Paperball":"www.paperball.de","Gigablast":"www.gigablast.com","Inci Sozluk":"inci.sozlukspot.com","Outbrain":"paid.outbrain.com","Plista":"farm.plista.com","Neti":"www.neti.ee","LifeStreet":"lfstmedia.com","Finderoo":"www.finderoo.com","Virgin":"webmail.virginbroadband.com.au","Latne":"www.latne.lv","Meinestadt":"www.meinestadt.de","Google Video":"video.google.com","Babylon":"search.babylon.com","Mixi":"mixi.jp","Twitter":"twitter.com","earthlink":"search.earthlink.net","Pinterest":"pinterest.com","Online.no":"online.no","Foursquare":"foursquare.com","Skynet":"www.skynet.be","Amazon":"amazon.com","Crawler":"www.crawler.com","Voila":"search.ke.voila.fr","Orange":"busca.orange.es","Apollo Latvia":"apollo.lv/portal/search/","Zoeken":"www.zoeken.nl","Vinden":"www.vinden.nl","163 Mail":"mail.163.com","Google Images":"google.ac/imgres","Opplysningen 1881":"www.1881.no","Classmates":"classmates.com","Jivox":"jivox.com","Naver Mail":"mail.naver.com","Arianna":"arianna.libero.it","Skyrock":"skyrock.com","Eksi Sozluk":"Sozluk.com","goo":"search.goo.ne.jp","Hacker News":"news.ycombinator.com","Metager":"meta.rrzn.uni-hannover.de","Witch":"www.witch.de","suche.info":"suche.info","SoSoDesk":"sosodesktop.com","Fixsuche":"www.fixsuche.de","Everyclick":"www.everyclick.com","Weborama":"www.weborama.com","Freenet":"webmail.freenet.de","Icerockeet":"blogs.icerocket.com","Vkontakte":"vk.com","Firstfind":"www.firstsfind.com","SourceForge":"sourceforge.net","Donanimhaber":"donanimhaber.com","OpenX":"us-ads.openx.net","Qualigo":"www.qualigo.at","Zoohoo":"zoohoo.cz","Aport":"sm.aport.ru","Tribal Fusion":"cdnx.tribalfusion.com","Ecosia":"ecosia.org","Nate":"search.nate.com","Last.fm":"lastfm.ru","Jyxo":"jyxo.1188.cz","Flixster":"flixster.com","Youtube":"youtube.com","Eniro":"www.eniro.se","Needtofind":"ko.search.need2find.com","Disqus":"redirect.disqus.com","Eyeota":"eyeota.net","PubMatic":"sshowads.pubmatic.com","Holmes":"holmes.ge","Looksmart":"www.looksmart.com","Yatedo":"www.yatedo.com","Telstra":"search.media.telstra.com.au","El Mundo":"ariadna.elmundo.es","Baidu":"www.baidu.com","Trusted-Search":"www.trusted--search.com","WeeWorld":"weeworld.com","MetaCrawler.de":"s1.metacrawler.de","maailm":"www.maailm.com","RPMFind":"rpmfind.net","British Telecommunications":"search.bt.com","WWW":"search.www.ee","AdRoll":"adroll.com","Hit-Parade":"req.-hit-parade.com","Tagged":"login.tagged.com","Paper.li":"paper.li","AudienceScience":"wunderloop.net","Marktplaats":"www.marktplaats.nl","StickyADS.tv":"stickyadstv.com","MyLife":"mylife.ru","Yahoo! Mail":"mail.yahoo.net","Search This":"www.searchthis.com","XING":"xing.com","Monstercrawler":"www.monstercrawler.com","Habbo":"habbo.com","MyHeritage":"myheritage.com","La Toile Du Quebec Via Google":"www.toile.com","Gnadenmeer":"www.gnadenmeer.de","Ask":"ask.com","Yippy":"search.yippy.com","Bigpond":"webmail.bigpond.com","Seznam Mail":"email.seznam.cz","Plazoo":"www.plazoo.com","Goyellow.de":"www.goyellow.de","Fluct":"adingo.jp","LinkedIn":"linkedin.com","PeoplePC":"search.peoplepc.com","I.ua":"search.i.ua","Mixpo":"mixpo.com","Hotbot":"www.hotbot.com","Daum Mail":"mail2.daum.net","Cuil":"www.cuil.com","Francite":"recherche.francite.com","Maxwebsearch":"maxwebsearch.com","Sovrn":"lijit.com","Daum":"search.daum.net","Suchmaschine.com":"www.suchmaschine.com","Myspace":"myspace.com","Instela":"instela.com","Forestle":"forestle.org","Alice Adsl":"rechercher.aliceadsl.fr","Zoek":"www3.zoek.nl","Certified-Toolbar":"search.certified-toolbar.com","MicroAd":"microad.jp","IXquick":"ixquick.com","Trouvez.com":"www.trouvez.com","Globososo":"searches.globososo.com","Startsiden":"www.startsiden.no","LiveJournal":"livejournal.ru","Daemon search":"daemon-search.com","Hocam.com":"hocam.com","Sogou":"www.sougou.com","Renren":"renren.com","Mamma":"www.mamma.com","Fireball":"www.fireball.de","Neustar AdAdvisor":"adadvisor.net","Technorati":"technorati.com","myYearbook":"myyearbook.com","Poisk.ru":"poisk.ru","Mister Wong":"www.mister-wong.com","MoiKrug.ru":"moikrug.ru","Casale Media":"casalemedia.com","Google Blogsearch":"blogsearch.google.ac","The Smart Search":"thesmartsearch.net","all.by":"all.by","Multiply":"multiply.com","Ilse":"www.ilse.nl","DasOertliche":"www.dasoertliche.de","Genieo":"search.genieo.com","Zhongsou":"p.zhongsou.com","Kvasir":"www.kvasir.no","kununu":"kununu.com","StudiVZ":"studivz.net","I-play":"start.iplay.com","iiNet":"webmail.iinet.net.au","DasTelefonbuch":"www1.dastelefonbuch.de","Tut.by":"search.tut.by","Interia":"www.google.interia.pl","Naver":"search.naver.com","Facebook":"facebook.com","Yam":"search.yam.com","Acoon":"www.acoon.de","Searchalot":"searchalot.com","StumbleUpon":"stumbleupon.com","Taboola":"trc.taboola.com","Abacho":"www.abacho.de","Acuity Ads":"acuityplatform.com","Meta":"meta.ua","Cyworld":"global.cyworld.com","canoe.ca":"web.canoe.ca","DuckDuckGo":"duckduckgo.com","Identi.ca":"identi.ca","Bluewin":"search.bluewin.ch","Taringa!":"taringa.net","Teoma":"www.teoma.com","Mynet Mail":"mail.mynet.com","InfoSpace":"infospace.com","arama":"arama.com","Delfi":"otsing.delfi.ee","TalkTalk":"www.talktalk.co.uk","hi5":"hi5.com","HighBeam":"www.highbeam.com","uol.com.br":"busca.uol.com.br","Westnet":"webmail.westnet.com.au","Fotolog":"fotolog.com","Arcor":"www.arcor.de","Search.com":"www.search.com"}+{"Friends Reunited":"friendsreunited.com","I-play":"start.iplay.com","White Pages":"www.whitepages.com.au","Eniro":"www.eniro.se","Kvasir":"www.kvasir.no","Geona":"geona.net","Neustar AdAdvisor":"adadvisor.net","Wirtualna Polska":"szukaj.wp.pl","Tagged":"login.tagged.com","Liveinternet":"liveinternet.ru","Xanga":"xanga.com","Metager2":"metager2.de","Commander":"webmail.commander.net.au","QIP":"mail.qip.ru","Plazoo":"www.plazoo.com","Globososo":"searches.globososo.com","Zoohoo":"zoohoo.cz","Eurip":"www.eurip.com","Web.de":"suche.web.de","Mailchimp":"com.mailchimp.mailchimp","DuckDuckGo":"duckduckgo.com","Google Blogsearch":"blogsearch.google.ac","Searchy":"www.searchy.co.uk","MySearch":"mysearch.com","AppNexus":"ib.adnxs.com","Outbrain":"paid.outbrain.com","Metager":"meta.rrzn.uni-hannover.de","Search.com":"www.search.com","Westnet":"webmail.westnet.com.au","BlackPlanet":"blackplanet.com","Arcor":"www.arcor.de","Bing":"bing.com","Orange Webmail":"orange.fr/webmail","Nasza-klasa.pl":"nk.pl","URL.ORGanizier":"www.url.org","HighBeam":"www.highbeam.com","Instagram":"instagram.com","Rubicon Project":"optimized-by.rubiconproject.com","Mixi":"mixi.jp","Web.nl":"www.web.nl","X-recherche":"www.x-recherche.com","Freenet":"webmail.freenet.de","Altavista":"www.altavista.com","Teoma":"www.teoma.com","2gis":"2gis.ru","Google Video":"video.google.com","Viadeo":"viadeo.com","DasOertliche":"www.dasoertliche.de","canoe.ca":"web.canoe.ca","Sapo":"pesquisa.sapo.pt","TikTok":"tiktok.com","Voila":"search.ke.voila.fr","Sovrn":"lijit.com","Icerockeet":"blogs.icerocket.com","Babylon":"search.babylon.com","Yandex Images":"images.yandex.ru","Euroseek":"www.euroseek.com","Hacker News":"news.ycombinator.com","Vimeo":"vimeo.com","Shenma":"so.m.sm.cn","AOL":"search.aol.com","Tribal Fusion":"cdnx.tribalfusion.com","Flyingbird":"inspsearch.com","Austronaut":"www2.austronaut.at","Hocam.com":"hocam.com","Yahoo! Mail":"mail.yahoo.net","LinkedIn":"com.linkedin.android","SoSoDesk":"sosodesktop.com","Yippy":"search.yippy.com","PubMatic":"sshowads.pubmatic.com","Fireball":"www.fireball.de","Adition":"adition.com","arama":"arama.com","Hyves":"hyves.nl","Classmates":"classmates.com","Lo.st":"lo.st","Google+":"url.google.com","DasTelefonbuch":"www1.dastelefonbuch.de","E1.ru":"mail.e1.ru","Forestle":"forestle.org","StackOverflow":"stackoverflow.com","AdRoll":"adroll.com","Zoho":"mail.zoho.com","TrovaRapido":"www.trovarapido.com","Finderoo":"www.finderoo.com","Dodo":"webmail.dodo.com.au","Flixster":"flixster.com","GitHub":"github.com","Instela":"instela.com","BidSwitch":"bidswitch.net","ONE by AOL":"nexage.com","Cuil":"www.cuil.com","Donanimhaber":"donanimhaber.com","Threads":"threads.net","Opplysningen 1881":"www.1881.no","Virgin":"webmail.virginbroadband.com.au","Google News":"news.google.ac","Acuity Ads":"acuityplatform.com","myYearbook":"myyearbook.com","SearchCanvas":"www.searchcanvas.com","Goyellow.de":"www.goyellow.de","Taringa!":"taringa.net","Nifty":"search.nifty.com","T-Online":"suche.t-online.de","Startsiden":"www.startsiden.no","Alice Adsl":"rechercher.aliceadsl.fr","MyLife":"mylife.ru","Acoon":"www.acoon.de","Freshweather":"www.fresh-weather.com","Last.fm":"lastfm.ru","Telegram":"web.telegram.org","LiveJournal":"livejournal.ru","Plaxo":"plaxo.com","GAIS":"gais.cs.ccu.edu.tw","Sogou":"www.sougou.com","ZEDO":"zedo.com","WhatsApp":"web.whatsapp.com","Lycos":"search.lycos.com","Paper.li":"paper.li","Criteo":"cas.jp.as.criteo.com","Zhongsou":"p.zhongsou.com","Everyclick":"www.everyclick.com","Adform":"adform.net","Picsearch":"www.picsearch.com","iPrimus":"webmail.iprimus.com.au","SourceForge":"sourceforge.net","Kataweb":"www.kataweb.it","dmoz":"dmoz.org","WWW":"search.www.ee","Sonico.com":"sonico.com","OpenX":"us-ads.openx.net","Vindex":"www.vindex.nl","Alexa":"alexa.com","Arianna":"arianna.libero.it","Fluct":"adingo.jp","AllTheWeb":"www.alltheweb.com","Nigma":"nigma.ru","XING":"xing.com","PeoplePC":"search.peoplepc.com","Online.no":"online.no","Myspace":"myspace.com","Witch":"www.witch.de","Monstercrawler":"www.monstercrawler.com","Meta":"meta.ua","Findwide":"search.findwide.com","Foursquare":"foursquare.com","Cyworld":"global.cyworld.com","British Telecommunications":"search.bt.com","Yahoo!":"finance.yahoo.com","Paperball":"www.paperball.de","AudienceScience":"wunderloop.net","Looksmart":"www.looksmart.com","all.by":"all.by","PriceRunner":"www.pricerunner.co.uk","Mastermail":"mastermail.ru","Excite":"search.excite.it","Blogdigger":"www.blogdigger.com","Apollo Latvia":"apollo.lv/portal/search/","Rakuten":"websearch.rakuten.co.jp","Firstfind":"www.firstsfind.com","Zoek":"www3.zoek.nl","Bebo":"bebo.com","Amazon":"amazon.com","Mail.ru":"e.mail.ru","Quora":"quora.com","Tildes":"tildes.net","Certified-Toolbar":"search.certified-toolbar.com","AdNET":"adnet.de","Hotbot":"www.hotbot.com","Outlook.com":"mail.live.com","Vkontakte":"m.vk.com","Twingly":"www.twingly.com","IXquick":"ixquick.com","WebSearch":"www.websearch.com","Digg":"digg.com","vKruguDruzei.ru":"vkrugudruzei.ru","Identi.ca":"identi.ca","Seznam":"search.seznam.cz","Mixpo":"mixpo.com","Sharelook":"www.sharelook.fr","El Mundo":"ariadna.elmundo.es","Compuserve":"websearch.cs.com","TalkTalk":"www.talktalk.co.uk","Bing Images":"bing.com/images/search","Douban":"douban.com","StumbleUpon":"stumbleupon.com","Poisk.ru":"poisk.ru","Yieldmo":"yieldmo.com","Yandex Maps":"maps.yandex.ru","Beeline":"post.ru","Trouvez.com":"www.trouvez.com","Ukr.net":"mail.ukr.net","Mister Wong":"www.mister-wong.com","Daum":"search.daum.net","Exalead":"www.exalead.fr","Suchmaschine.com":"www.suchmaschine.com","AdSpirit":"adspirit.de","Conduit":"search.conduit.com","ABCsøk":"abcsolk.no","Indeed":"de.indeed.com","WAYN":"wayn.com","ADFOX":"adfox.ru","Search This":"www.searchthis.com","uol.com.br":"busca.uol.com.br","Atlas":"searchatlas.centrum.cz","Biglobe":"cgi.search.biglobe.ne.jp","Seznam Mail":"email.seznam.cz","Uludag Sozluk":"uludagsozluk.com","Optus Zoo":"webmail.optuszoo.com.au","blekko":"blekko.com","Qualigo":"www.qualigo.at","Fast Browser Search":"www.fastbrowsersearch.com","LowerMyBills":"lowermybills.com","Delfi":"otsing.delfi.ee","Weibo":"weibo.com","Gule Sider":"www.gulesider.no","Zoeken":"www.zoeken.nl","Taboola":"trc.taboola.com","Tut.by":"search.tut.by","InfoSpace":"infospace.com","Yam":"search.yam.com","Trusted-Search":"www.trusted--search.com","Needtofind":"ko.search.need2find.com","GMX":"suche.gmx.net","126 Mail":"mail.126.com","ITU Sozluk":"itusozluk.com","Qzone":"qzone.qq.com","MyHeritage":"myheritage.com","Maxwebsearch":"maxwebsearch.com","Torg.Mail.ru":"torg.mail.ru","Apontador":"apontador.com.br","APOLL07":"apollo7.de","Ecosia":"ecosia.org","Dalesearch":"www.dalesearch.com","suche.info":"suche.info","iiNet":"webmail.iinet.net.au","Mamma":"www.mamma.com","Buzznet":"buzznet.com","Road Runner Search":"search.rr.com","WeeWorld":"weeworld.com","Toolbarhome":"www.toolbarhome.com","Odnoklassniki":"odnoklassniki.ru","Inbox.com":"inbox.com","Vodafone":"webmail.vodafone.co.nz","Hooseek.com":"www.hooseek.com","Zapmeta":"www.zapmeta.com","Tuenti":"tuenti.com","Adam Internet":"webmail.adam.com.au","Snapdo":"search.snapdo.com","Startpagina":"startgoogle.startpagina.nl","Aport":"sm.aport.ru","Pocket":"getpocket.com","Jungle Key":"junglekey.com","I.ua":"search.i.ua","Eyeota":"eyeota.net","Eksi Sozluk":"Sozluk.com","Renren":"renren.com","Qwant":"www.qwant.com","Terra":"buscador.terra.es","Naver Mail":"mail.naver.com","Delfi latvia":"smart.delfi.lv","Netlog":"netlog.com","Skynet":"www.skynet.be","Daemon search":"daemon-search.com","Badoo":"badoo.com","StickyADS.tv":"stickyadstv.com","Yandex.Market":"market.yandex.ru","Interia":"www.google.interia.pl","Yasni":"www.yasni.de","Searchalot":"searchalot.com","Hit-Parade":"req.-hit-parade.com","Gnadenmeer":"www.gnadenmeer.de","Rambler":"mail.rambler.ru","Gomeo":"www.gomeo.com","Windows Live Spaces":"login.live.com","Sibmail":"sibmail.com","Disqus":"redirect.disqus.com","Mozo":"mozo.com.au","Snapchat":"com.snapchat.android","Whirlpool":"forums.whirlpool.net.au","AOL Mail":"mail.aol.com","Crawler":"www.crawler.com","Najdi":"www.najdi.si","Jyxo":"jyxo.1188.cz","Orange":"busca.orange.es","Tumblr":"tumblr.com","La Toile Du Quebec Via Google":"www.toile.com","Netspace":"webmail.netspace.net.au","Doubleclick":"ad.doubleclick.net","earthlink":"com.earthlink.myearthlink","Twitter":"twitter.com","Ilse":"www.ilse.nl","Jungle Spider":"www.jungle-spider.de","Softonic":"search.softonic.com","Youtube":"youtube.com","Volny":"web.volny.cz","Habbo":"habbo.com","Coccoc":"coccoc.com","Charter":"www.charter.net","1&1":"search.1and1.com","Gigablast":"www.gigablast.com","Flickr":"flickr.com","Google":"support.google.com","LifeStreet":"lfstmedia.com","Search.ch":"www.search.ch","QQ Mail":"mail.qq.com","Casale Media":"casalemedia.com","Weborama":"www.weborama.com","Neti":"www.neti.ee","Flashtalking":"flashtalking.com","Virgilio":"ricerca.virgilio.it","Yandex":"mail.yandex.ru","163 Mail":"mail.163.com","StepStone":"www.stepstone.de","Onet":"szukaj.onet.pl","Fixsuche":"www.fixsuche.de","Clix":"pesquisa.clix.pt","Baidu":"www.baidu.com","Telstra":"search.media.telstra.com.au","Ask":"ask.com","qip":"search.qip.ru","Vinden":"www.vinden.nl","Plista":"farm.plista.com","SteelHouse":"steelhousemedia.com","Winamp":"search.winamp.com","Sociomantic Labs":"sociomantic.com","Sonobi":"sonobi.com","Jivox":"jivox.com","Mynet Mail":"mail.mynet.com","Inci Sozluk":"inci.sozlukspot.com","YouGoo":"www.yougoo.fr","Centrum":"serach.centrum.cz","Reddit":"reddit.com","Francite":"recherche.francite.com","Google Product Search":"google.ac/products","2degrees":"webmail.2degreesbroadband.co.nz","Ask Toolbar":"search.tb.ask.com","Holmes":"holmes.ge","Gmail":"mail.google.com","eo":"eo.st","Yahoo! Images":"image.yahoo.cn","Fotolog":"fotolog.com","Naver":"search.naver.com","Pinterest":"pinterest.ca","Freecause":"search.freecause.com","Skyrock":"skyrock.com","Delicious":"delicious.com","Nate":"search.nate.com","Tiscali":"search.tiscali.it","Tixuma":"www.tixuma.de","MoiKrug.ru":"moikrug.ru","Sizmek":"bs.serving-sys.com","Multiply":"multiply.com","Orkut":"orkut.com","kununu":"kununu.com","UKR.net":"search.ukr.net","RPMFind":"rpmfind.net","Abacho":"www.abacho.de","Free":"search.free.fr","Friendster":"friendster.com","Meinestadt":"www.meinestadt.de","Lilo":"search.lilo.org","ICQ":"www.icq.com","FriendFeed":"friendfeed.com","Slack":"app.slack.com","Mozbot":"www.mozbot.fr","Genieo":"search.genieo.com","The Smart Search":"thesmartsearch.net","Yandex.Direct":"an.yandex.ru","Daum Mail":"mail2.daum.net","1und1":"search.1und1.de","goo":"search.goo.ne.jp","maailm":"www.maailm.com","1.cz":"1.cz","Suchnase":"www.suchnase.de","Technorati":"technorati.com","Naver Images":"image.search.naver.com","Gaia Online":"gaiaonline.com","MetaCrawler.de":"s1.metacrawler.de","Blogpulse":"www.blogpulse.com","MicroAd":"microad.jp","StudiVZ":"studivz.net","Facebook":"facebook.com","Monster":"www.monster.be","hi5":"hi5.com","Bluewin":"search.bluewin.ch","Yatedo":"www.yatedo.com","Walhello":"www.walhello.info","Flix":"www.flix.de","Google Images":"google.ac/imgres","360.cn":"so.360.cn","Comcast":"serach.comcast.net","Skype":"web.skype.com","Latne":"www.latne.lv","Bigpond":"webmail.bigpond.com","Marktplaats":"www.marktplaats.nl","Geni":"geni.com","Price.ru":"price.ru"}diff --git a/test/plausible/ingestion/source_test.exs b/test/plausible/ingestion/source_test.exsnew file mode 100644index 0000000000..2037f16535--- /dev/null+++ b/test/plausible/ingestion/source_test.exs@@ -0,0 +1,4 @@+defmodule Plausible.Ingestion.SourceTest do+ use ExUnit.Case, async: true+ doctest Plausible.Ingestion.Source+enddiff --git a/test/plausible_web/controllers/api/external_controller_test.exs b/test/plausible_web/controllers/api/external_controller_test.exsindex 0e33cbc4ee..fd84e99f3e 100644--- a/test/plausible_web/controllers/api/external_controller_test.exs+++ b/test/plausible_web/controllers/api/external_controller_test.exs@@ -1944,6 +1944,1210 @@ defmodule PlausibleWeb.Api.ExternalControllerTest doendend+ describe "custom source parsing rules" do+ setup do+ site = insert(:site)+ {:ok, site: site}+ end++ test "threads is Threads", %{+ conn: conn,+ site: site+ } do+ params = %{+ name: "pageview",+ url: "http://example.com?utm_source=threads",+ domain: site.domain+ }++ conn =+ conn+ |> put_req_header("user-agent", @user_agent)+ |> post("/api/event", params)++ session = get_created_session(site)++ assert response(conn, 202) == "ok"+ assert session.referrer_source == "Threads"+ assert session.utm_source == "threads"+ assert session.channel == "Organic Social"+ end++ test "ig is Instagram", %{+ conn: conn,+ site: site+ } do+ params = %{+ name: "pageview",+ url: "http://example.com?utm_source=ig",+ domain: site.domain+ }++ conn =+ conn+ |> put_req_header("user-agent", @user_agent)+ |> post("/api/event", params)++ session = get_created_session(site)++ assert response(conn, 202) == "ok"+ assert session.referrer_source == "Instagram"+ assert session.utm_source == "ig"+ assert session.channel == "Organic Social"+ end++ test "yt is Youtube", %{+ conn: conn,+ site: site+ } do+ params = %{+ name: "pageview",+ url: "http://example.com?utm_source=yt",+ domain: site.domain+ }++ conn =+ conn+ |> put_req_header("user-agent", @user_agent)+ |> post("/api/event", params)++ session = get_created_session(site)++ assert response(conn, 202) == "ok"+ assert session.referrer_source == "Youtube"+ assert session.utm_source == "yt"+ assert session.channel == "Organic Video"+ end++ test "yt-ads is Youtube paid", %{+ conn: conn,+ site: site+ } do+ params = %{+ name: "pageview",+ url: "http://example.com?utm_source=yt-ads",+ domain: site.domain+ }++ conn =+ conn+ |> put_req_header("user-agent", @user_agent)+ |> post("/api/event", params)++ session = get_created_session(site)++ assert response(conn, 202) == "ok"+ assert session.referrer_source == "Youtube"+ assert session.utm_source == "yt-ads"+ assert session.channel == "Paid Video"+ end++ test "fb is Facebook", %{+ conn: conn,+ site: site+ } do+ params = %{+ name: "pageview",+ url: "http://example.com?utm_source=fb",+ domain: site.domain+ }++ conn =+ conn+ |> put_req_header("user-agent", @user_agent)+ |> post("/api/event", params)++ session = get_created_session(site)++ assert response(conn, 202) == "ok"+ assert session.referrer_source == "Facebook"+ assert session.utm_source == "fb"+ assert session.channel == "Organic Social"+ end++ test "fb-ads is Facebook", %{+ conn: conn,+ site: site+ } do+ params = %{+ name: "pageview",+ url: "http://example.com?utm_source=fb-ads",+ domain: site.domain+ }++ conn =+ conn+ |> put_req_header("user-agent", @user_agent)+ |> post("/api/event", params)++ session = get_created_session(site)++ assert response(conn, 202) == "ok"+ assert session.referrer_source == "Facebook"+ assert session.utm_source == "fb-ads"+ assert session.channel == "Paid Social"+ end++ test "fbad is Facebook", %{+ conn: conn,+ site: site+ } do+ params = %{+ name: "pageview",+ url: "http://example.com?utm_source=fbad",+ domain: site.domain+ }++ conn =+ conn+ |> put_req_header("user-agent", @user_agent)+ |> post("/api/event", params)++ session = get_created_session(site)++ assert response(conn, 202) == "ok"+ assert session.referrer_source == "Facebook"+ assert session.utm_source == "fbad"+ assert session.channel == "Paid Social"+ end++ test "facebook-ads is Facebook", %{+ conn: conn,+ site: site+ } do+ params = %{+ name: "pageview",+ url: "http://example.com?utm_source=facebook-ads",+ domain: site.domain+ }++ conn =+ conn+ |> put_req_header("user-agent", @user_agent)+ |> post("/api/event", params)++ session = get_created_session(site)++ assert response(conn, 202) == "ok"+ assert session.referrer_source == "Facebook"+ assert session.utm_source == "facebook-ads"+ assert session.channel == "Paid Social"+ end++ test "Reddit-ads is Reddit", %{+ conn: conn,+ site: site+ } do+ params = %{+ name: "pageview",+ url: "http://example.com?utm_source=Reddit-ads",+ domain: site.domain+ }++ conn =+ conn+ |> put_req_header("user-agent", @user_agent)+ |> post("/api/event", params)++ session = get_created_session(site)++ assert response(conn, 202) == "ok"+ assert session.referrer_source == "Reddit"+ assert session.utm_source == "Reddit-ads"+ assert session.channel == "Paid Social"+ end++ test "google_ads is Google", %{+ conn: conn,+ site: site+ } do+ params = %{+ name: "pageview",+ url: "http://example.com?utm_source=google_ads",+ domain: site.domain+ }++ conn =+ conn+ |> put_req_header("user-agent", @user_agent)+ |> post("/api/event", params)++ session = get_created_session(site)++ assert response(conn, 202) == "ok"+ assert session.referrer_source == "Google"+ assert session.utm_source == "google_ads"+ assert session.channel == "Paid Search"+ end++ test "Google-ads is Google", %{+ conn: conn,+ site: site+ } do+ params = %{+ name: "pageview",+ url: "http://example.com?utm_source=Google-ads",+ domain: site.domain+ }++ conn =+ conn+ |> put_req_header("user-agent", @user_agent)+ |> post("/api/event", params)++ session = get_created_session(site)++ assert response(conn, 202) == "ok"+ assert session.referrer_source == "Google"+ assert session.utm_source == "Google-ads"+ assert session.channel == "Paid Search"+ end++ test "utm_source=Adwords is Google paid search", %{+ conn: conn,+ site: site+ } do+ params = %{+ name: "pageview",+ url: "http://example.com?utm_source=Adwords",+ domain: site.domain+ }++ conn =+ conn+ |> put_req_header("user-agent", @user_agent)+ |> post("/api/event", params)++ session = get_created_session(site)++ assert response(conn, 202) == "ok"+ assert session.referrer_source == "Google"+ assert session.utm_source == "Adwords"+ assert session.channel == "Paid Search"+ end++ test "twitter-ads is Twitter", %{+ conn: conn,+ site: site+ } do+ params = %{+ name: "pageview",+ url: "http://example.com?utm_source=twitter-ads",+ domain: site.domain+ }++ conn =+ conn+ |> put_req_header("user-agent", @user_agent)+ |> post("/api/event", params)++ session = get_created_session(site)++ assert response(conn, 202) == "ok"+ assert session.referrer_source == "Twitter"+ assert session.utm_source == "twitter-ads"+ assert session.channel == "Paid Social"+ end++ test "android-app://com.reddit.frontpage is Reddit", %{+ conn: conn,+ site: site+ } do+ params = %{+ name: "pageview",+ url: "http://example.com",+ referrer: "android-app://com.reddit.frontpage",+ domain: site.domain+ }++ conn =+ conn+ |> put_req_header("user-agent", @user_agent)+ |> post("/api/event", params)++ session = get_created_session(site)++ assert response(conn, 202) == "ok"+ assert session.referrer_source == "Reddit"+ assert session.channel == "Organic Social"+ end++ test "perplexity.ai is Perplexity", %{+ conn: conn,+ site: site+ } do+ params = %{+ name: "pageview",+ url: "http://example.com",+ referrer: "https://perplexity.ai",+ domain: site.domain+ }++ conn =+ conn+ |> put_req_header("user-agent", @user_agent)+ |> post("/api/event", params)++ session = get_created_session(site)++ assert response(conn, 202) == "ok"+ assert session.referrer_source == "Perplexity"+ assert session.channel == "Organic Search"+ end++ test "utm_source=perplexity is Perplexity", %{+ conn: conn,+ site: site+ } do+ params = %{+ name: "pageview",+ url: "http://example.com?utm_source=perplexity",+ domain: site.domain+ }++ conn =+ conn+ |> put_req_header("user-agent", @user_agent)+ |> post("/api/event", params)++ session = get_created_session(site)++ assert response(conn, 202) == "ok"+ assert session.referrer_source == "Perplexity"+ assert session.channel == "Organic Search"+ end++ test "statics.teams.cdn.office.net is Microsoft Teams", %{+ conn: conn,+ site: site+ } do+ params = %{+ name: "pageview",+ url: "http://example.com",+ referrer: "https://statics.teams.cdn.office.net",+ domain: site.domain+ }++ conn =+ conn+ |> put_req_header("user-agent", @user_agent)+ |> post("/api/event", params)++ session = get_created_session(site)++ assert response(conn, 202) == "ok"+ assert session.referrer_source == "Microsoft Teams"+ assert session.channel == "Organic Social"+ end++ test "wikipedia domain is resolved as Wikipedia", %{+ conn: conn,+ site: site+ } do+ params = %{+ name: "pageview",+ url: "http://example.com",+ referrer: "https://en.wikipedia.org",+ domain: site.domain+ }++ conn =+ conn+ |> put_req_header("user-agent", @user_agent)+ |> post("/api/event", params)++ session = get_created_session(site)++ assert response(conn, 202) == "ok"+ assert session.referrer_source == "Wikipedia"+ assert session.channel == "Referral"+ end++ test "ntp.msn.com is Bing", %{+ conn: conn,+ site: site+ } do+ params = %{+ name: "pageview",+ url: "http://example.com",+ referrer: "https://ntp.msn.com",+ domain: site.domain+ }++ conn =+ conn+ |> put_req_header("user-agent", @user_agent)+ |> post("/api/event", params)++ session = get_created_session(site)++ assert response(conn, 202) == "ok"+ assert session.referrer_source == "Bing"+ assert session.channel == "Organic Search"+ end++ test "search.brave.com is Brave", %{+ conn: conn,+ site: site+ } do+ params = %{+ name: "pageview",+ url: "http://example.com",+ referrer: "https://search.brave.com",+ domain: site.domain+ }++ conn =+ conn+ |> put_req_header("user-agent", @user_agent)+ |> post("/api/event", params)++ session = get_created_session(site)++ assert response(conn, 202) == "ok"+ assert session.referrer_source == "Brave"+ assert session.channel == "Organic Search"+ end++ test "yandex.com.tr is Yandex", %{+ conn: conn,+ site: site+ } do+ params = %{+ name: "pageview",+ url: "http://example.com",+ referrer: "https://yandex.com.tr",+ domain: site.domain+ }++ conn =+ conn+ |> put_req_header("user-agent", @user_agent)+ |> post("/api/event", params)++ session = get_created_session(site)++ assert response(conn, 202) == "ok"+ assert session.referrer_source == "Yandex"+ assert session.channel == "Organic Search"+ end++ test "yandex.kz is Yandex", %{+ conn: conn,+ site: site+ } do+ params = %{+ name: "pageview",+ url: "http://example.com",+ referrer: "https://yandex.kz",+ domain: site.domain+ }++ conn =+ conn+ |> put_req_header("user-agent", @user_agent)+ |> post("/api/event", params)++ session = get_created_session(site)++ assert response(conn, 202) == "ok"+ assert session.referrer_source == "Yandex"+ assert session.channel == "Organic Search"+ end++ test "ya.ru is Yandex", %{+ conn: conn,+ site: site+ } do+ params = %{+ name: "pageview",+ url: "http://example.com",+ referrer: "https://ya.ru",+ domain: site.domain+ }++ conn =+ conn+ |> put_req_header("user-agent", @user_agent)+ |> post("/api/event", params)++ session = get_created_session(site)++ assert response(conn, 202) == "ok"+ assert session.referrer_source == "Yandex"+ assert session.channel == "Organic Search"+ end++ test "yandex.uz is Yandex", %{+ conn: conn,+ site: site+ } do+ params = %{+ name: "pageview",+ url: "http://example.com",+ referrer: "https://yandex.uz",+ domain: site.domain+ }++ conn =+ conn+ |> put_req_header("user-agent", @user_agent)+ |> post("/api/event", params)++ session = get_created_session(site)++ assert response(conn, 202) == "ok"+ assert session.referrer_source == "Yandex"+ assert session.channel == "Organic Search"+ end++ test "yandex.fr is Yandex", %{+ conn: conn,+ site: site+ } do+ params = %{+ name: "pageview",+ url: "http://example.com",+ referrer: "https://yandex.fr",+ domain: site.domain+ }++ conn =+ conn+ |> put_req_header("user-agent", @user_agent)+ |> post("/api/event", params)++ session = get_created_session(site)++ assert response(conn, 202) == "ok"+ assert session.referrer_source == "Yandex"+ assert session.channel == "Organic Search"+ end++ test "yandex.eu is Yandex", %{+ conn: conn,+ site: site+ } do+ params = %{+ name: "pageview",+ url: "http://example.com",+ referrer: "https://yandex.eu",+ domain: site.domain+ }++ conn =+ conn+ |> put_req_header("user-agent", @user_agent)+ |> post("/api/event", params)++ session = get_created_session(site)++ assert response(conn, 202) == "ok"+ assert session.referrer_source == "Yandex"+ assert session.channel == "Organic Search"+ end++ test "yandex.tm is Yandex", %{+ conn: conn,+ site: site+ } do+ params = %{+ name: "pageview",+ url: "http://example.com",+ referrer: "https://yandex.tm",+ domain: site.domain+ }++ conn =+ conn+ |> put_req_header("user-agent", @user_agent)+ |> post("/api/event", params)++ session = get_created_session(site)++ assert response(conn, 202) == "ok"+ assert session.referrer_source == "Yandex"+ assert session.channel == "Organic Search"+ end++ test "discord.com is Discord", %{+ conn: conn,+ site: site+ } do+ params = %{+ name: "pageview",+ url: "http://example.com",+ referrer: "https://discord.com",+ domain: site.domain+ }++ conn =+ conn+ |> put_req_header("user-agent", @user_agent)+ |> post("/api/event", params)++ session = get_created_session(site)++ assert response(conn, 202) == "ok"+ assert session.referrer_source == "Discord"+ assert session.channel == "Organic Social"+ end++ test "discordapp.com is Discord", %{+ conn: conn,+ site: site+ } do+ params = %{+ name: "pageview",+ url: "http://example.com",+ referrer: "https://discordapp.com",+ domain: site.domain+ }++ conn =+ conn+ |> put_req_header("user-agent", @user_agent)+ |> post("/api/event", params)++ session = get_created_session(site)++ assert response(conn, 202) == "ok"+ assert session.referrer_source == "Discord"+ assert session.channel == "Organic Social"+ end++ test "canary.discord.com is Discord", %{+ conn: conn,+ site: site+ } do+ params = %{+ name: "pageview",+ url: "http://example.com",+ referrer: "https://canary.discord.com",+ domain: site.domain+ }++ conn =+ conn+ |> put_req_header("user-agent", @user_agent)+ |> post("/api/event", params)++ session = get_created_session(site)++ assert response(conn, 202) == "ok"+ assert session.referrer_source == "Discord"+ assert session.channel == "Organic Social"+ end++ test "ptb.discord.com is Discord", %{+ conn: conn,+ site: site+ } do+ params = %{+ name: "pageview",+ url: "http://example.com",+ referrer: "https://ptb.discord.com",+ domain: site.domain+ }++ conn =+ conn+ |> put_req_header("user-agent", @user_agent)+ |> post("/api/event", params)++ session = get_created_session(site)++ assert response(conn, 202) == "ok"+ assert session.referrer_source == "Discord"+ assert session.channel == "Organic Social"+ end++ test "www.baidu.com is Baidu", %{conn: conn, site: site} do+ params = %{+ name: "pageview",+ url: "http://example.com",+ referrer: "https://baidu.com",+ domain: site.domain+ }++ conn =+ conn+ |> put_req_header("user-agent", @user_agent)+ |> post("/api/event", params)++ session = get_created_session(site)++ assert response(conn, 202) == "ok"+ assert session.referrer_source == "Baidu"+ assert session.channel == "Organic Search"+ end++ test "t.me is Telegram", %{conn: conn, site: site} do+ params = %{+ name: "pageview",+ url: "http://example.com",+ referrer: "https://t.me",+ domain: site.domain+ }++ conn =+ conn+ |> put_req_header("user-agent", @user_agent)+ |> post("/api/event", params)++ session = get_created_session(site)++ assert response(conn, 202) == "ok"+ assert session.referrer_source == "Telegram"+ assert session.channel == "Organic Social"+ end++ test "webk.telegram.org is Telegram", %{conn: conn, site: site} do+ params = %{+ name: "pageview",+ url: "http://example.com",+ referrer: "https://webk.telegram.org",+ domain: site.domain+ }++ conn =+ conn+ |> put_req_header("user-agent", @user_agent)+ |> post("/api/event", params)++ session = get_created_session(site)++ assert response(conn, 202) == "ok"+ assert session.referrer_source == "Telegram"+ assert session.channel == "Organic Social"+ end++ test "sogou.com is Sogou", %{conn: conn, site: site} do+ params = %{+ name: "pageview",+ url: "http://example.com",+ referrer: "https://sogou.com",+ domain: site.domain+ }++ conn =+ conn+ |> put_req_header("user-agent", @user_agent)+ |> post("/api/event", params)++ session = get_created_session(site)++ assert response(conn, 202) == "ok"+ assert session.referrer_source == "Sogou"+ assert session.channel == "Organic Search"+ end++ test "m.sogou.com is Sogou", %{conn: conn, site: site} do+ params = %{+ name: "pageview",+ url: "http://example.com",+ referrer: "https://m.sogou.com",+ domain: site.domain+ }++ conn =+ conn+ |> put_req_header("user-agent", @user_agent)+ |> post("/api/event", params)++ session = get_created_session(site)++ assert response(conn, 202) == "ok"+ assert session.referrer_source == "Sogou"+ assert session.channel == "Organic Search"+ end++ test "wap.sogou.com is Sogou", %{conn: conn, site: site} do+ params = %{+ name: "pageview",+ url: "http://example.com",+ referrer: "https://wap.sogou.com",+ domain: site.domain+ }++ conn =+ conn+ |> put_req_header("user-agent", @user_agent)+ |> post("/api/event", params)++ session = get_created_session(site)++ assert response(conn, 202) == "ok"+ assert session.referrer_source == "Sogou"+ assert session.channel == "Organic Search"+ end++ test "linktr.ee is Linktree", %{conn: conn, site: site} do+ params = %{+ name: "pageview",+ url: "http://example.com",+ referrer: "https://linktr.ee",+ domain: site.domain+ }++ conn =+ conn+ |> put_req_header("user-agent", @user_agent)+ |> post("/api/event", params)++ session = get_created_session(site)++ assert response(conn, 202) == "ok"+ assert session.referrer_source == "Linktree"+ assert session.channel == "Referral"+ end++ test "linktree is Linktree", %{conn: conn, site: site} do+ params = %{+ name: "pageview",+ url: "http://example.com?utm_source=linktree",+ domain: site.domain+ }++ conn =+ conn+ |> put_req_header("user-agent", @user_agent)+ |> post("/api/event", params)++ session = get_created_session(site)++ assert response(conn, 202) == "ok"+ assert session.referrer_source == "Linktree"+ assert session.channel == "Referral"+ end+ end++ describe "custom channel parsing rules" do+ setup do+ site = insert(:site)+ {:ok, site: site}+ end++ test "hacker news is social channel", %{+ conn: conn,+ site: site+ } do+ params = %{+ name: "pageview",+ url: "http://example.com",+ referrer: "https://news.ycombinator.com",+ domain: site.domain+ }++ conn =+ conn+ |> put_req_header("user-agent", @user_agent)+ |> post("/api/event", params)++ session = get_created_session(site)++ assert response(conn, 202) == "ok"+ assert session.referrer_source == "Hacker News"+ assert session.channel == "Organic Social"+ end++ test "yahoo is organic search", %{+ conn: conn,+ site: site+ } do+ params = %{+ name: "pageview",+ url: "http://example.com",+ referrer: "https://search.yahoo.com",+ domain: site.domain+ }++ conn =+ conn+ |> put_req_header("user-agent", @user_agent)+ |> post("/api/event", params)++ session = get_created_session(site)++ assert response(conn, 202) == "ok"+ assert session.referrer_source == "Yahoo!"+ assert session.channel == "Organic Search"+ end++ test "gmail is email channel", %{+ conn: conn,+ site: site+ } do+ params = %{+ name: "pageview",+ url: "http://example.com",+ referrer: "https://mail.google.com",+ domain: site.domain+ }++ conn =+ conn+ |> put_req_header("user-agent", @user_agent)+ |> post("/api/event", params)++ session = get_created_session(site)++ assert response(conn, 202) == "ok"+ assert session.referrer_source == "Gmail"+ assert session.channel == "Email"+ end++ test "utm_source=newsletter is email channel", %{+ conn: conn,+ site: site+ } do+ params = %{+ name: "pageview",+ url: "http://example.com?utm_source=Newsletter-UK",+ domain: site.domain+ }++ conn =+ conn+ |> put_req_header("user-agent", @user_agent)+ |> post("/api/event", params)++ session = get_created_session(site)++ assert response(conn, 202) == "ok"+ assert session.referrer_source == "Newsletter-UK"+ assert session.channel == "Email"+ end++ test "temu.com is shopping channel", %{+ conn: conn,+ site: site+ } do+ params = %{+ name: "pageview",+ url: "http://example.com",+ referrer: "https://temu.com",+ domain: site.domain+ }++ conn =+ conn+ |> put_req_header("user-agent", @user_agent)+ |> post("/api/event", params)++ session = get_created_session(site)++ assert response(conn, 202) == "ok"+ assert session.referrer_source == "temu.com"+ assert session.channel == "Organic Shopping"+ end++ test "utm_source=Telegram is social channel", %{+ conn: conn,+ site: site+ } do+ params = %{+ name: "pageview",+ url: "http://example.com?utm_source=Telegram",+ domain: site.domain+ }++ conn =+ conn+ |> put_req_header("user-agent", @user_agent)+ |> post("/api/event", params)++ session = get_created_session(site)++ assert response(conn, 202) == "ok"+ assert session.referrer_source == "Telegram"+ assert session.channel == "Organic Social"+ end++ test "chatgpt.com is search channel", %{+ conn: conn,+ site: site+ } do+ params = %{+ name: "pageview",+ url: "http://example.com",+ referrer: "https://chatgpt.com",+ domain: site.domain+ }++ conn =+ conn+ |> put_req_header("user-agent", @user_agent)+ |> post("/api/event", params)++ session = get_created_session(site)++ assert response(conn, 202) == "ok"+ assert session.referrer_source == "chatgpt.com"+ assert session.channel == "Organic Search"+ end++ test "Slack is social channel", %{+ conn: conn,+ site: site+ } do+ params = %{+ name: "pageview",+ url: "http://example.com",+ referrer: "https://app.slack.com",+ domain: site.domain+ }++ conn =+ conn+ |> put_req_header("user-agent", @user_agent)+ |> post("/api/event", params)++ session = get_created_session(site)++ assert response(conn, 202) == "ok"+ assert session.referrer_source == "Slack"+ assert session.channel == "Organic Social"+ end++ test "producthunt is social", %{+ conn: conn,+ site: site+ } do+ params = %{+ name: "pageview",+ url: "http://example.com?ref=producthunt",+ domain: site.domain+ }++ conn =+ conn+ |> put_req_header("user-agent", @user_agent)+ |> post("/api/event", params)++ session = get_created_session(site)++ assert response(conn, 202) == "ok"+ assert session.referrer_source == "producthunt"+ assert session.channel == "Organic Social"+ end++ test "github is social", %{+ conn: conn,+ site: site+ } do+ params = %{+ name: "pageview",+ url: "http://example.com",+ referrer: "https://github.com",+ domain: site.domain+ }++ conn =+ conn+ |> put_req_header("user-agent", @user_agent)+ |> post("/api/event", params)++ session = get_created_session(site)++ assert response(conn, 202) == "ok"+ assert session.referrer_source == "GitHub"+ assert session.channel == "Organic Social"+ end++ test "steamcommunity.com is social", %{+ conn: conn,+ site: site+ } do+ params = %{+ name: "pageview",+ url: "http://example.com",+ referrer: "https://steamcommunity.com",+ domain: site.domain+ }++ conn =+ conn+ |> put_req_header("user-agent", @user_agent)+ |> post("/api/event", params)++ session = get_created_session(site)++ assert response(conn, 202) == "ok"+ assert session.referrer_source == "steamcommunity.com"+ assert session.channel == "Organic Social"+ end++ test "Vkontakte is social", %{+ conn: conn,+ site: site+ } do+ params = %{+ name: "pageview",+ url: "http://example.com",+ referrer: "https://vkontakte.ru",+ domain: site.domain+ }++ conn =+ conn+ |> put_req_header("user-agent", @user_agent)+ |> post("/api/event", params)++ session = get_created_session(site)++ assert response(conn, 202) == "ok"+ assert session.referrer_source == "Vkontakte"+ assert session.channel == "Organic Social"+ end++ test "Threads is social", %{+ conn: conn,+ site: site+ } do+ params = %{+ name: "pageview",+ url: "http://example.com",+ referrer: "https://threads.net",+ domain: site.domain+ }++ conn =+ conn+ |> put_req_header("user-agent", @user_agent)+ |> post("/api/event", params)++ session = get_created_session(site)++ assert response(conn, 202) == "ok"+ assert session.referrer_source == "Threads"+ assert session.channel == "Organic Social"+ end++ test "Ecosia is search", %{+ conn: conn,+ site: site+ } do+ params = %{+ name: "pageview",+ url: "http://example.com",+ referrer: "https://ecosia.org",+ domain: site.domain+ }++ conn =+ conn+ |> put_req_header("user-agent", @user_agent)+ |> post("/api/event", params)++ session = get_created_session(site)++ assert response(conn, 202) == "ok"+ assert session.referrer_source == "Ecosia"+ assert session.channel == "Organic Search"+ end+ end+describe "user_id generation" dosetup dosite = insert(:site)