Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.
TypeScript analyticsanalytics-dashboardclickhouseelixirgoogle-analyticsmarketingopen-source-analyticsphoenixplausible-analyticspostgresqlprivacyprivacy-friendlyself-hostedsimple-analyticsstatisticstailwindcssweb-analyticswebsite-analyticswebsite-statswebsite-tracking
Ignore unknown countries (#2556)
Uku Taht1785653b1e7d
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex f6d7fdd3d0..3d47785321 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -9,6 +9,7 @@ All notable changes to this project will be documented in this file.### Changed- Reject events with long URIs and data URIs plausible/analytics#2536- Always show direct traffic in sources reports plausible/analytics#2531+- Stop recording XX and T1 country codes plausible/analytics#2556## Fixed- Cascade delete sent_renewal_notifications table when user is deleted plausible/analytics#2549diff --git a/config/test.exs b/config/test.exsindex 475cd355a3..df2aba6382 100644--- a/config/test.exs+++ b/config/test.exs@@ -56,7 +56,9 @@ config :geolix,{1, 1, 1, 1} => %{country: %{iso_code: "US"}},{2, 2, 2, 2} => geolix_sample_lookup,{1, 1, 1, 1, 1, 1, 1, 1} => %{country: %{iso_code: "US"}},- {0, 0, 0, 0} => %{country: %{iso_code: "ZZ"}}+ {0, 0, 0, 0} => %{country: %{iso_code: "ZZ"}, city: %{geoname_id: 123_123}},+ {0, 0, 0, 1} => %{country: %{iso_code: "XX"}, subdivisions: [%{iso_code: "IDF"}]},+ {0, 0, 0, 2} => %{country: %{iso_code: "T1"}, subdivisions: [%{}, %{iso_code: "IDF"}]}}}]diff --git a/lib/plausible/ingestion/city_overrides.ex b/lib/plausible/ingestion/city_overrides.exindex 61f046c94a..a00eb24ab9 100644--- a/lib/plausible/ingestion/city_overrides.ex+++ b/lib/plausible/ingestion/city_overrides.ex@@ -198,5 +198,5 @@ defmodule Plausible.Ingestion.CityOverrides do# Hackney -> London2_647_694 => 2_643_743}- def get, do: @overrides+ def get(key, default), do: Map.get(@overrides, key, default)enddiff --git a/lib/plausible/ingestion/event.ex b/lib/plausible/ingestion/event.exindex f4c4cfb92d..84d79ed9c9 100644--- a/lib/plausible/ingestion/event.ex+++ b/lib/plausible/ingestion/event.ex@@ -5,7 +5,7 @@ defmodule Plausible.Ingestion.Event doare uniformly either buffered in batches (to Clickhouse) or dropped(e.g. due to spam blocklist) from the processing pipeline."""- alias Plausible.Ingestion.{Request, CityOverrides}+ alias Plausible.Ingestion.Requestalias Plausible.ClickhouseEventalias Plausible.Site.GateKeeper@@ -168,39 +168,9 @@ defmodule Plausible.Ingestion.Event doenddefp put_geolocation(%__MODULE__{} = event) do- result = Geolix.lookup(event.request.remote_ip, where: :geolocation)+ result = Plausible.Ingestion.Geolocation.lookup(event.request.remote_ip)- country_code =- get_in(result, [:country, :iso_code])- |> ignore_unknown_country()-- city_geoname_id = get_in(result, [:city, :geoname_id])- city_geoname_id = Map.get(CityOverrides.get(), city_geoname_id, city_geoname_id)-- subdivision1_code =- case result do- %{subdivisions: [%{iso_code: iso_code} | _rest]} ->- country_code <> "-" <> iso_code-- _ ->- ""- end-- subdivision2_code =- case result do- %{subdivisions: [_first, %{iso_code: iso_code} | _rest]} ->- country_code <> "-" <> iso_code-- _ ->- ""- end-- update_attrs(event, %{- country_code: country_code,- subdivision1_code: subdivision1_code,- subdivision2_code: subdivision2_code,- city_geoname_id: city_geoname_id- })+ update_attrs(event, result)enddefp put_screen_size(%__MODULE__{} = event) do@@ -372,9 +342,6 @@ defmodule Plausible.Ingestion.Event doendend- defp ignore_unknown_country("ZZ"), do: nil- defp ignore_unknown_country(country), do: country-defp generate_user_id(request, domain, hostname, salt) docond dois_nil(salt) ->diff --git a/lib/plausible/ingestion/geolocation.ex b/lib/plausible/ingestion/geolocation.exnew file mode 100644index 0000000000..f30f30af13--- /dev/null+++ b/lib/plausible/ingestion/geolocation.ex@@ -0,0 +1,47 @@+defmodule Plausible.Ingestion.Geolocation do+ @moduledoc false+ alias Plausible.Ingestion.CityOverrides++ def lookup(remote_ip) do+ result = Geolix.lookup(remote_ip, where: :geolocation)++ country_code =+ get_in(result, [:country, :iso_code])+ |> ignore_unknown_country()++ city_geoname_id = country_code && get_in(result, [:city, :geoname_id])+ city_geoname_id = CityOverrides.get(city_geoname_id, city_geoname_id)++ %{+ country_code: country_code,+ subdivision1_code: subdivision1_code(country_code, result),+ subdivision2_code: subdivision2_code(country_code, result),+ city_geoname_id: city_geoname_id+ }+ end++ defp subdivision1_code(country_code, %{subdivisions: [%{iso_code: iso_code} | _rest]})+ when not is_nil(country_code) do+ country_code <> "-" <> iso_code+ end++ defp subdivision1_code(_, _), do: nil++ defp subdivision2_code(country_code, %{subdivisions: [_first, %{iso_code: iso_code} | _rest]})+ when not is_nil(country_code) do+ country_code <> "-" <> iso_code+ end++ defp subdivision2_code(_, _), do: nil++ @ignored_countries [+ # Worldwide+ "ZZ",+ # Disputed territory+ "XX",+ # Tor exit node+ "T1"+ ]+ defp ignore_unknown_country(code) when code in @ignored_countries, do: nil+ defp ignore_unknown_country(country), do: country+enddiff --git a/test/plausible_web/controllers/api/external_controller_test.exs b/test/plausible_web/controllers/api/external_controller_test.exsindex 315b7423a7..061be8b578 100644--- a/test/plausible_web/controllers/api/external_controller_test.exs+++ b/test/plausible_web/controllers/api/external_controller_test.exs@@ -685,6 +685,47 @@ defmodule PlausibleWeb.Api.ExternalControllerTest dopageview = get_event(domain)assert pageview.country_code == <<0, 0>>+ assert pageview.subdivision1_code == ""+ assert pageview.subdivision2_code == ""+ assert pageview.city_geoname_id == 0+ end++ test "ignores disputed territory code XX", %{conn: conn, domain: domain} do+ params = %{+ name: "pageview",+ domain: domain,+ url: "http://gigride.live/"+ }++ conn+ |> put_req_header("x-forwarded-for", "0.0.0.1")+ |> post("/api/event", params)++ pageview = get_event(domain)++ assert pageview.country_code == <<0, 0>>+ assert pageview.subdivision1_code == ""+ assert pageview.subdivision2_code == ""+ assert pageview.city_geoname_id == 0+ end++ test "ignores TOR exit node country code T1", %{conn: conn, domain: domain} do+ params = %{+ name: "pageview",+ domain: domain,+ url: "http://gigride.live/"+ }++ conn+ |> put_req_header("x-forwarded-for", "0.0.0.2")+ |> post("/api/event", params)++ pageview = get_event(domain)++ assert pageview.country_code == <<0, 0>>+ assert pageview.subdivision1_code == ""+ assert pageview.subdivision2_code == ""+ assert pageview.city_geoname_id == 0endtest "scrubs port from x-forwarded-for", %{conn: conn, domain: domain} do