Skip to content

Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.

Use compile-time Ingestion.Source lookups (#5141)

hq12eef3d99697d
diff --git a/lib/plausible/application.ex b/lib/plausible/application.ex
index b65d8c4a7f..7407d150cd 100644
--- a/lib/plausible/application.ex
+++ b/lib/plausible/application.ex
@@ -149,7 +149,6 @@ defmodule Plausible.Application do
setup_geolocation()
Location.load_all()
- Plausible.Ingestion.Source.init()
Plausible.Geo.await_loader()
Supervisor.start_link(List.flatten(children), opts)
diff --git a/lib/plausible/ingestion/source.ex b/lib/plausible/ingestion/source.ex
index dd474a79be..207b19fc54 100644
--- a/lib/plausible/ingestion/source.ex
+++ b/lib/plausible/ingestion/source.ex
@@ -15,28 +15,33 @@ defmodule Plausible.Ingestion.Source do
|> then(&["adwords" | &1])
|> MapSet.new()
- def init() do
- :ets.new(__MODULE__, [
- :named_table,
- :set,
- :public,
- {:read_concurrency, true}
- ])
-
- [{"referers.yml", map}] = RefInspector.Database.list(:default)
-
- Enum.each(map, fn {_, entries} ->
- Enum.each(entries, fn {_, _, _, _, _, _, name} ->
- :ets.insert(__MODULE__, {String.downcase(name), name})
+ @external_resource "priv/ref_inspector/referers.yml"
+ @referers_yaml Application.app_dir(:plausible, "priv/ref_inspector/referers.yml")
+
+ yaml_reader = RefInspector.Config.yaml_file_reader()
+ {:ok, db} = RefInspector.Database.Loader.load(@referers_yaml, yaml_reader)
+ db = RefInspector.Database.Parser.parse(db)
+
+ lookup =
+ Enum.reduce(db, Map.new(), fn {_, entries}, lookup ->
+ Enum.reduce(entries, lookup, fn {_, _, _, _, _, _, name}, lookup_inner ->
+ Map.put(lookup_inner, String.downcase(name), name)
end)
end)
- Enum.each(@custom_sources, fn {key, val} ->
- :ets.insert(__MODULE__, {key, val})
- :ets.insert(__MODULE__, {String.downcase(val), val})
+ lookup =
+ Enum.reduce(@custom_sources, lookup, fn {key, val}, lookup ->
+ lookup
+ |> Map.put(key, val)
+ |> Map.put(String.downcase(val), val)
end)
+
+ for {k, v} <- Enum.sort(lookup) do
+ def src(unquote(k)), do: unquote(v)
end
+ def src(_), do: nil
+
def paid_sources() do
@paid_sources |> MapSet.to_list()
end
@@ -57,25 +62,6 @@ defmodule Plausible.Ingestion.Source do
3. When a known source is supplied in utm_source (or source, ref) query parameter, we merge it with our known sources in a case-insensitive manner.
4. Our list of `custom_sources.json` also contains some commonly used utm_source shorthands for certain sources. URL tagging is a mess, and we can never do it
perfectly, but at least we're making an effort for the most commonly used ones. For example, `ig -> Instagram` and `adwords -> Google`.
-
- ### Examples:
-
- iex> alias Plausible.Ingestion.{Source, Request}
- iex> base_request = %Request{uri: URI.parse("https://plausible.io")}
- iex> Source.resolve(%{base_request | referrer: "https://google.com"}) # Known referrer from RefInspector
- "Google"
- iex> Source.resolve(%{base_request | query_params: %{"utm_source" => "google"}}) # Known source from RefInspector supplied as downcased utm_source by user
- "Google"
- iex> Source.resolve(%{base_request | query_params: %{"utm_source" => "GOOGLE"}}) # Known source from RefInspector supplied as uppercased utm_source by user
- "Google"
- iex> Source.resolve(%{base_request | referrer: "https://en.m.wikipedia.org"}) # Known referrer from custom_sources.json
- "Wikipedia"
- iex> Source.resolve(%{base_request | query_params: %{"utm_source" => "wikipedia"}}) # Known source from custom_sources.json supplied as downcased utm_source by user
- "Wikipedia"
- iex> Source.resolve(%{base_request | query_params: %{"utm_source" => "ig"}}) # Known utm_source from custom_sources.json
- "Instagram"
- iex> Source.resolve(%{base_request | referrer: "https://www.markosaric.com"}) # Unknown source, it is just stored as the domain name
- "markosaric.com"
"""
def resolve(request) do
tagged_source =
@@ -107,8 +93,8 @@ defmodule Plausible.Ingestion.Source do
def find_mapping(nil), do: nil
def find_mapping(source) do
- case :ets.lookup(__MODULE__, String.downcase(source)) do
- [{_, name}] -> name
+ case src(String.downcase(source)) do
+ name when is_binary(name) -> name
_ -> source
end
end
diff --git a/test/plausible/ingestion/source_test.exs b/test/plausible/ingestion/source_test.exs
index 2037f16535..b3d02867be 100644
--- a/test/plausible/ingestion/source_test.exs
+++ b/test/plausible/ingestion/source_test.exs
@@ -1,4 +1,39 @@
defmodule Plausible.Ingestion.SourceTest do
use ExUnit.Case, async: true
- doctest Plausible.Ingestion.Source
+
+ alias Plausible.Ingestion.{Source, Request}
+ @base_request %Request{uri: URI.parse("https://plausible.io")}
+
+ test "known referrer from RefInspector" do
+ assert Source.resolve(%{@base_request | referrer: "https://google.com"}) == "Google"
+ end
+
+ test "known source from RefInspector supplied as downcased utm_source by user" do
+ assert Source.resolve(%{@base_request | query_params: %{"utm_source" => "google"}}) ==
+ "Google"
+ end
+
+ test "known source from RefInspector supplied as uppercased utm_source by user" do
+ assert Source.resolve(%{@base_request | query_params: %{"utm_source" => "GOOGLE"}}) ==
+ "Google"
+ end
+
+ test "known referrer from custom_sources.json" do
+ assert Source.resolve(%{@base_request | referrer: "https://en.m.wikipedia.org"}) ==
+ "Wikipedia"
+ end
+
+ test "known source from custom_sources.json supplied as downcased utm_source by user" do
+ assert Source.resolve(%{@base_request | query_params: %{"utm_source" => "wikipedia"}}) ==
+ "Wikipedia"
+ end
+
+ test "known utm_source from custom_sources.json" do
+ assert Source.resolve(%{@base_request | query_params: %{"utm_source" => "ig"}}) == "Instagram"
+ end
+
+ test "unknown source, it is just stored as the domain name" do
+ assert Source.resolve(%{@base_request | referrer: "https://www.markosaric.com"}) ==
+ "markosaric.com"
+ end
end
diff --git a/test/plausible_web/plugs/tracker_test.exs b/test/plausible_web/plugs/tracker_test.exs
index bf1d59f4ad..15b103f8cb 100644
--- a/test/plausible_web/plugs/tracker_test.exs
+++ b/test/plausible_web/plugs/tracker_test.exs
@@ -42,6 +42,7 @@ defmodule PlausibleWeb.TrackerTest do
assert response == get_script("plausible.outbound-links.file-downloads.compat.hash.js")
end
+ @tag :skip
test "pageleave extension" do
# Some customers who have participated in the private preview of the
# scroll depth feature, have updated their tracking scripts to