Skip to content

Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.

Map lowercase tagged sources to capitalized form during ingestion (#4417)

Uku Taht2e89a501881c
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 0511859cdc..1480582fea 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -33,6 +33,7 @@ All notable changes to this project will be documented in this file.
- `is` and `is not` filters in dashboard no longer support wildcards. Use contains/does not contain filter instead.
- `bounce_rate` metric now returns 0 instead of null for event:page breakdown when page has never been entry page.
- Make `TOTP_VAULT_KEY` optional plausible/analytics#4317
+- Sources like 'google' and 'facebook' are now stored in capitalized forms ('Google', 'Facebook') plausible/analytics#4417
### Fixed
diff --git a/lib/mix/tasks/send_pageview.ex b/lib/mix/tasks/send_pageview.ex
index 53cc510cc7..b25cf740ab 100644
--- a/lib/mix/tasks/send_pageview.ex
+++ b/lib/mix/tasks/send_pageview.ex
@@ -17,6 +17,7 @@ defmodule Mix.Tasks.SendPageview do
@default_referrer "https://google.com"
@default_event "pageview"
@default_props "{}"
+ @default_queryparams ""
@options [
ip: :string,
user_agent: :string,
@@ -28,7 +29,8 @@ defmodule Mix.Tasks.SendPageview do
event: :string,
props: :string,
revenue_currency: :string,
- revenue_amount: :string
+ revenue_amount: :string,
+ queryparams: :string
]
def run(opts) do
@@ -88,6 +90,7 @@ defmodule Mix.Tasks.SendPageview do
event = Keyword.get(opts, :event, @default_event)
props = Keyword.get(opts, :props, @default_props)
hostname = Keyword.get(opts, :hostname, domain)
+ queryparams = Keyword.get(opts, :queryparams, @default_queryparams)
revenue =
if Keyword.get(opts, :revenue_currency) do
@@ -99,7 +102,7 @@ defmodule Mix.Tasks.SendPageview do
%{
name: event,
- url: "http://#{hostname}#{page}",
+ url: "http://#{hostname}#{page}?#{queryparams}",
domain: domain,
referrer: referrer,
props: props,
diff --git a/lib/plausible/application.ex b/lib/plausible/application.ex
index a15ef5f77b..8296f9074b 100644
--- a/lib/plausible/application.ex
+++ b/lib/plausible/application.ex
@@ -105,6 +105,7 @@ defmodule Plausible.Application do
setup_geolocation()
Location.load_all()
+ Plausible.Ingestion.Acquisition.init()
Plausible.Geo.await_loader()
Supervisor.start_link(List.flatten(children), opts)
diff --git a/lib/plausible/ingestion/acquisition.ex b/lib/plausible/ingestion/acquisition.ex
new file mode 100644
index 0000000000..510e9966c6
--- /dev/null
+++ b/lib/plausible/ingestion/acquisition.ex
@@ -0,0 +1,27 @@
+defmodule Plausible.Ingestion.Acquisition do
+ @moduledoc false
+
+ def init() do
+ :ets.new(__MODULE__, [
+ :named_table,
+ :set,
+ :public,
+ {:read_concurrency, true}
+ ])
+
+ [{"referers.yml", map}] = RefInspector.Database.list(:default)
+
+ Enum.flat_map(map, fn {_, entries} ->
+ Enum.map(entries, fn {_, _, _, _, _, _, name} ->
+ :ets.insert(__MODULE__, {String.downcase(name), name})
+ end)
+ end)
+ end
+
+ def find_mapping(source) do
+ case :ets.lookup(__MODULE__, source) do
+ [{_, name}] -> name
+ _ -> source
+ end
+ end
+end
diff --git a/lib/plausible/ingestion/event.ex b/lib/plausible/ingestion/event.ex
index 3f905416a2..089ca7e45b 100644
--- a/lib/plausible/ingestion/event.ex
+++ b/lib/plausible/ingestion/event.ex
@@ -385,12 +385,16 @@ defmodule Plausible.Ingestion.Event do
end
defp get_referrer_source(request, ref) do
- source =
+ tagged_source =
request.query_params["utm_source"] ||
request.query_params["source"] ||
request.query_params["ref"]
- source || PlausibleWeb.RefInspector.parse(ref)
+ if tagged_source do
+ Plausible.Ingestion.Acquisition.find_mapping(tagged_source)
+ else
+ PlausibleWeb.RefInspector.parse(ref)
+ end
end
defp clean_referrer(nil), do: nil
diff --git a/test/plausible_web/controllers/api/external_controller_test.exs b/test/plausible_web/controllers/api/external_controller_test.exs
index 69f0ac06b8..80c5a42967 100644
--- a/test/plausible_web/controllers/api/external_controller_test.exs
+++ b/test/plausible_web/controllers/api/external_controller_test.exs
@@ -239,7 +239,7 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
assert event.browser_version == "70.0"
end
- test "parses referrer", %{conn: conn, site: site} do
+ test "parses referrer source", %{conn: conn, site: site} do
params = %{
name: "pageview",
url: "http://example.com/",
@@ -407,6 +407,22 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
assert session.referrer_source == "betalist"
end
+ test "if utm_source matches a capitalized form from ref_inspector, the capitalized form is recorded",
+ %{conn: conn, site: site} do
+ params = %{
+ name: "pageview",
+ url: "http://www.example.com/?utm_source=facebook",
+ domain: site.domain
+ }
+
+ conn
+ |> post("/api/event", params)
+
+ session = get_created_session(site)
+
+ assert session.referrer_source == "Facebook"
+ end
+
test "utm tags are stored", %{conn: conn, site: site} do
params = %{
name: "pageview",