Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.
TypeScript analyticsanalytics-dashboardclickhouseelixirgoogle-analyticsmarketingopen-source-analyticsphoenixplausible-analyticspostgresqlprivacyprivacy-friendlyself-hostedsimple-analyticsstatisticstailwindcssweb-analyticswebsite-analyticswebsite-statswebsite-tracking
Use compile-time Ingestion.Source lookups (#5141)
hq12eef3d99697d
diff --git a/lib/plausible/application.ex b/lib/plausible/application.exindex b65d8c4a7f..7407d150cd 100644--- a/lib/plausible/application.ex+++ b/lib/plausible/application.ex@@ -149,7 +149,6 @@ defmodule Plausible.Application dosetup_geolocation()Location.load_all()- Plausible.Ingestion.Source.init()Plausible.Geo.await_loader()Supervisor.start_link(List.flatten(children), opts)diff --git a/lib/plausible/ingestion/source.ex b/lib/plausible/ingestion/source.exindex dd474a79be..207b19fc54 100644--- a/lib/plausible/ingestion/source.ex+++ b/lib/plausible/ingestion/source.ex@@ -15,28 +15,33 @@ defmodule Plausible.Ingestion.Source do|> then(&["adwords" | &1])|> MapSet.new()- def init() do- :ets.new(__MODULE__, [- :named_table,- :set,- :public,- {:read_concurrency, true}- ])-- [{"referers.yml", map}] = RefInspector.Database.list(:default)-- Enum.each(map, fn {_, entries} ->- Enum.each(entries, fn {_, _, _, _, _, _, name} ->- :ets.insert(__MODULE__, {String.downcase(name), name})+ @external_resource "priv/ref_inspector/referers.yml"+ @referers_yaml Application.app_dir(:plausible, "priv/ref_inspector/referers.yml")++ yaml_reader = RefInspector.Config.yaml_file_reader()+ {:ok, db} = RefInspector.Database.Loader.load(@referers_yaml, yaml_reader)+ db = RefInspector.Database.Parser.parse(db)++ lookup =+ Enum.reduce(db, Map.new(), fn {_, entries}, lookup ->+ Enum.reduce(entries, lookup, fn {_, _, _, _, _, _, name}, lookup_inner ->+ Map.put(lookup_inner, String.downcase(name), name)end)end)- Enum.each(@custom_sources, fn {key, val} ->- :ets.insert(__MODULE__, {key, val})- :ets.insert(__MODULE__, {String.downcase(val), val})+ lookup =+ Enum.reduce(@custom_sources, lookup, fn {key, val}, lookup ->+ lookup+ |> Map.put(key, val)+ |> Map.put(String.downcase(val), val)end)++ for {k, v} <- Enum.sort(lookup) do+ def src(unquote(k)), do: unquote(v)end+ def src(_), do: nil+def paid_sources() do@paid_sources |> MapSet.to_list()end@@ -57,25 +62,6 @@ defmodule Plausible.Ingestion.Source do3. When a known source is supplied in utm_source (or source, ref) query parameter, we merge it with our known sources in a case-insensitive manner.4. Our list of `custom_sources.json` also contains some commonly used utm_source shorthands for certain sources. URL tagging is a mess, and we can never do itperfectly, but at least we're making an effort for the most commonly used ones. For example, `ig -> Instagram` and `adwords -> Google`.-- ### Examples:-- iex> alias Plausible.Ingestion.{Source, Request}- iex> base_request = %Request{uri: URI.parse("https://plausible.io")}- iex> Source.resolve(%{base_request | referrer: "https://google.com"}) # Known referrer from RefInspector- "Google"- iex> Source.resolve(%{base_request | query_params: %{"utm_source" => "google"}}) # Known source from RefInspector supplied as downcased utm_source by user- "Google"- iex> Source.resolve(%{base_request | query_params: %{"utm_source" => "GOOGLE"}}) # Known source from RefInspector supplied as uppercased utm_source by user- "Google"- iex> Source.resolve(%{base_request | referrer: "https://en.m.wikipedia.org"}) # Known referrer from custom_sources.json- "Wikipedia"- iex> Source.resolve(%{base_request | query_params: %{"utm_source" => "wikipedia"}}) # Known source from custom_sources.json supplied as downcased utm_source by user- "Wikipedia"- iex> Source.resolve(%{base_request | query_params: %{"utm_source" => "ig"}}) # Known utm_source from custom_sources.json- "Instagram"- iex> Source.resolve(%{base_request | referrer: "https://www.markosaric.com"}) # Unknown source, it is just stored as the domain name- "markosaric.com""""def resolve(request) dotagged_source =@@ -107,8 +93,8 @@ defmodule Plausible.Ingestion.Source dodef find_mapping(nil), do: nildef find_mapping(source) do- case :ets.lookup(__MODULE__, String.downcase(source)) do- [{_, name}] -> name+ case src(String.downcase(source)) do+ name when is_binary(name) -> name_ -> sourceendenddiff --git a/test/plausible/ingestion/source_test.exs b/test/plausible/ingestion/source_test.exsindex 2037f16535..b3d02867be 100644--- a/test/plausible/ingestion/source_test.exs+++ b/test/plausible/ingestion/source_test.exs@@ -1,4 +1,39 @@defmodule Plausible.Ingestion.SourceTest douse ExUnit.Case, async: true- doctest Plausible.Ingestion.Source++ alias Plausible.Ingestion.{Source, Request}+ @base_request %Request{uri: URI.parse("https://plausible.io")}++ test "known referrer from RefInspector" do+ assert Source.resolve(%{@base_request | referrer: "https://google.com"}) == "Google"+ end++ test "known source from RefInspector supplied as downcased utm_source by user" do+ assert Source.resolve(%{@base_request | query_params: %{"utm_source" => "google"}}) ==+ "Google"+ end++ test "known source from RefInspector supplied as uppercased utm_source by user" do+ assert Source.resolve(%{@base_request | query_params: %{"utm_source" => "GOOGLE"}}) ==+ "Google"+ end++ test "known referrer from custom_sources.json" do+ assert Source.resolve(%{@base_request | referrer: "https://en.m.wikipedia.org"}) ==+ "Wikipedia"+ end++ test "known source from custom_sources.json supplied as downcased utm_source by user" do+ assert Source.resolve(%{@base_request | query_params: %{"utm_source" => "wikipedia"}}) ==+ "Wikipedia"+ end++ test "known utm_source from custom_sources.json" do+ assert Source.resolve(%{@base_request | query_params: %{"utm_source" => "ig"}}) == "Instagram"+ end++ test "unknown source, it is just stored as the domain name" do+ assert Source.resolve(%{@base_request | referrer: "https://www.markosaric.com"}) ==+ "markosaric.com"+ endenddiff --git a/test/plausible_web/plugs/tracker_test.exs b/test/plausible_web/plugs/tracker_test.exsindex bf1d59f4ad..15b103f8cb 100644--- a/test/plausible_web/plugs/tracker_test.exs+++ b/test/plausible_web/plugs/tracker_test.exs@@ -42,6 +42,7 @@ defmodule PlausibleWeb.TrackerTest doassert response == get_script("plausible.outbound-links.file-downloads.compat.hash.js")end+ @tag :skiptest "pageleave extension" do# Some customers who have participated in the private preview of the# scroll depth feature, have updated their tracking scripts to