Skip to content

Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.

Refactor event struct creation function (#2098)

Vinicius Brasila10d44a0d78e
diff --git a/lib/plausible/ingestion/city_overrides.ex b/lib/plausible/ingestion/city_overrides.ex
new file mode 100644
index 0000000000..781c7913dc
--- /dev/null
+++ b/lib/plausible/ingestion/city_overrides.ex
@@ -0,0 +1,198 @@
+defmodule Plausible.Ingestion.CityOverrides do
+ @moduledoc false
+
+ @overrides %{
+ # Austria
+ # Gemeindebezirk Floridsdorf -> Vienna
+ 2_779_467 => 2_761_369,
+ # Gemeindebezirk Leopoldstadt -> Vienna
+ 2_772_614 => 2_761_369,
+ # Gemeindebezirk Landstrasse -> Vienna
+ 2_773_040 => 2_761_369,
+ # Gemeindebezirk Donaustadt -> Vienna
+ 2_780_851 => 2_761_369,
+ # Gemeindebezirk Favoriten -> Vienna
+ 2_779_776 => 2_761_369,
+ # Gemeindebezirk Währing -> Vienna
+ 2_762_091 => 2_761_369,
+ # Gemeindebezirk Wieden -> Vienna
+ 2_761_393 => 2_761_369,
+ # Gemeindebezirk Innere Stadt -> Vienna
+ 2_775_259 => 2_761_369,
+ # Gemeindebezirk Alsergrund -> Vienna
+ 2_782_729 => 2_761_369,
+ # Gemeindebezirk Liesing -> Vienna
+ 2_772_484 => 2_761_369,
+ # Urfahr -> Linz
+ 2_762_518 => 2_772_400,
+
+ # Canada
+ # Old Toronto -> Toronto
+ 8_436_019 => 6_167_865,
+ # Etobicoke -> Toronto
+ 5_950_267 => 6_167_865,
+ # East York -> Toronto
+ 5_946_235 => 6_167_865,
+ # Scarborough -> Toronto
+ 6_948_711 => 6_167_865,
+ # North York -> Toronto
+ 6_091_104 => 6_167_865,
+
+ # Czech republic
+ # Praha 5 -> Prague
+ 11_951_220 => 3_067_696,
+ # Praha 4 -> Prague
+ 11_951_218 => 3_067_696,
+ # Praha 11 -> Prague
+ 11_951_232 => 3_067_696,
+ # Praha 10 -> Prague
+ 11_951_210 => 3_067_696,
+ # Praha 4 -> Prague
+ 8_378_772 => 3_067_696,
+
+ # Denmark
+ # København SV -> Copenhagen
+ 11_747_123 => 2_618_425,
+ # København NV -> Copenhagen
+ 11_746_894 => 2_618_425,
+ # Odense S -> Odense
+ 11_746_825 => 2_615_876,
+ # Odense M -> Odense
+ 11_746_974 => 2_615_876,
+ # Odense SØ -> Odense
+ 11_746_888 => 2_615_876,
+ # Aarhus C -> Aarhus
+ 11_746_746 => 2_624_652,
+ # Aarhus N -> Aarhus
+ 11_746_890 => 2_624_652,
+
+ # Estonia
+ # Kristiine linnaosa -> Tallinn
+ 11_050_530 => 588_409,
+ # Kesklinna linnaosa -> Tallinn
+ 11_053_706 => 588_409,
+ # Lasnamäe linnaosa -> Tallinn
+ 11_050_526 => 588_409,
+ # Põhja-Tallinna linnaosa -> Tallinn
+ 11_049_594 => 588_409,
+ # Mustamäe linnaosa -> Tallinn
+ 11_050_531 => 588_409,
+ # Haabersti linnaosa -> Tallinn
+ 11_053_707 => 588_409,
+ # Viimsi -> Tallinn
+ 587_629 => 588_409,
+
+ # Germany
+ # Bezirk Tempelhof-Schöneberg -> Berlin
+ 3_336_297 => 2_950_159,
+ # Bezirk Mitte -> Berlin
+ 2_870_912 => 2_950_159,
+ # Bezirk Charlottenburg-Wilmersdorf -> Berlin
+ 3_336_294 => 2_950_159,
+ # Bezirk Friedrichshain-Kreuzberg -> Berlin
+ 3_336_295 => 2_950_159,
+ # Moosach -> Munich
+ 8_351_447 => 2_867_714,
+ # Schwabing-Freimann -> Munich
+ 8_351_448 => 2_867_714,
+ # Stadtbezirk 06 -> Düsseldorf
+ 6_947_276 => 2_934_246,
+ # Stadtbezirk 04 -> Düsseldorf
+ 6_947_274 => 2_934_246,
+ # Köln-Ehrenfeld -> Köln
+ 6_947_479 => 2_886_242,
+ # Köln-Lindenthal- -> Köln
+ 6_947_481 => 2_886_242,
+ # Beuel -> Bonn
+ 2_949_619 => 2_946_447,
+ # Innenstadt I -> Frankfurt am Main
+ 6_946_225 => 2_925_533,
+
+ # India
+ # Navi Mumbai -> Mumbai
+ 6_619_347 => 1_275_339,
+
+ # Mexico
+ # Miguel Hidalgo Villa Olímpica -> Mexico city
+ 11_561_026 => 3_530_597,
+ # Zedec Santa Fe -> Mexico city
+ 3_517_471 => 3_530_597,
+ # Fuentes del Pedregal-> Mexico city
+ 11_562_596 => 3_530_597,
+ # Centro -> Mexico city
+ 9_179_691 => 3_530_597,
+ # Cuauhtémoc-> Mexico city
+ 12_266_959 => 3_530_597,
+
+ # Netherlands
+ # Schiphol-Rijk -> Amsterdam
+ 10_173_838 => 2_759_794,
+ # Westpoort -> Amsterdam
+ 11_525_047 => 2_759_794,
+ # Amsterdam-Zuidoost -> Amsterdam
+ 6_544_881 => 2_759_794,
+ # Loosduinen -> The Hague
+ 11_525_037 => 2_747_373,
+ # Laak -> The Hague
+ 11_525_042 => 2_747_373,
+
+ # Norway
+ # Nordre Aker District -> Oslo
+ 6_940_981 => 3_143_244,
+
+ # Romania
+ # Sector 1 -> Bucharest,
+ 11_055_041 => 683_506,
+ # Sector 2 -> Bucharest
+ 11_055_040 => 683_506,
+ # Sector 3 -> Bucharest
+ 11_055_044 => 683_506,
+ # Sector 4 -> Bucharest
+ 11_055_042 => 683_506,
+ # Sector 5 -> Bucharest
+ 11_055_043 => 683_506,
+ # Sector 6 -> Bucharest
+ 11_055_039 => 683_506,
+ # Bucuresti -> Bucharest
+ 6_691_781 => 683_506,
+
+ # Slovakia
+ # Bratislava -> Bratislava
+ 3_343_955 => 3_060_972,
+
+ # Sweden
+ # Södermalm -> Stockholm
+ 2_676_209 => 2_673_730,
+
+ # Switzerland
+ # Vorstädte -> Basel
+ 11_789_440 => 2_661_604,
+ # Zürich (Kreis 11) / Oerlikon -> Zürich
+ 2_659_310 => 2_657_896,
+ # Zürich (Kreis 3) / Alt-Wiedikon -> Zürich
+ 2_658_007 => 2_657_896,
+ # Zürich (Kreis 5) -> Zürich
+ 6_295_521 => 2_657_896,
+ # Zürich (Kreis 1) / Hochschulen -> Zürich
+ 6_295_489 => 2_657_896,
+
+ # UK
+ # Shadwell -> London
+ 6_690_595 => 2_643_743,
+ # City of London -> London
+ 2_643_741 => 2_643_743,
+ # South Bank -> London
+ 6_545_251 => 2_643_743,
+ # Soho -> London
+ 6_545_173 => 2_643_743,
+ # Whitechapel -> London
+ 2_634_112 => 2_643_743,
+ # King's Cross -> London
+ 6_690_589 => 2_643_743,
+ # Poplar -> London
+ 2_640_091 => 2_643_743,
+ # Hackney -> London
+ 2_647_694 => 2_643_743
+ }
+ def get, do: @overrides
+end
diff --git a/lib/plausible/ingestion/event.ex b/lib/plausible/ingestion/event.ex
new file mode 100644
index 0000000000..f49d10fe15
--- /dev/null
+++ b/lib/plausible/ingestion/event.ex
@@ -0,0 +1,344 @@
+defmodule Plausible.Ingestion.Event do
+ require OpenTelemetry.Tracer, as: Tracer
+ alias Plausible.Ingestion.{Request, CityOverrides}
+
+ @spec build_and_buffer(Request.t()) :: :ok | :skip | {:error, Ecto.Changeset.t()}
+ @doc """
+ Builds events from %Plausible.Ingestion.Request{} and adds them to Plausible.Event.WriteBuffer.
+ This function reads geolocation data and parses the user agent string. Returns :skip if the
+ request is identified as spam, or blocked.
+ """
+ def build_and_buffer(%Request{} = request) do
+ with :ok <- spam_or_blocked?(request),
+ salts <- Plausible.Session.Salts.fetch(),
+ event <- Map.new(),
+ %{} = event <- put_user_agent(event, request),
+ %{} = event <- put_basic_info(event, request),
+ %{} = event <- put_referrer(event, request),
+ %{} = event <- put_geolocation(event, request),
+ %{} = event <- put_screen_size(event, request),
+ %{} = event <- put_props(event, request),
+ events when is_list(events) <- map_domains(event, request),
+ events when is_list(events) <- put_user_id(events, request, salts),
+ {:ok, events} <- validate_events(events),
+ events when is_list(events) <- register_session(events, request, salts) do
+ Enum.each(events, &Plausible.Event.WriteBuffer.insert/1)
+ end
+ end
+
+ defp put_basic_info(%{} = event, %Request{} = request) do
+ uri = request.url && URI.parse(request.url)
+ host = if uri && uri.host == "", do: "(none)", else: uri && uri.host
+
+ event
+ |> Map.put(:timestamp, NaiveDateTime.utc_now() |> NaiveDateTime.truncate(:second))
+ |> Map.put(:name, request.event_name)
+ |> Map.put(:hostname, strip_www(host))
+ |> Map.put(:pathname, get_pathname(uri, request.hash_mode))
+ end
+
+ defp get_pathname(_uri = nil, _hash_mode), do: "/"
+
+ defp get_pathname(uri, hash_mode) do
+ pathname =
+ (uri.path || "/")
+ |> URI.decode()
+ |> String.trim_trailing()
+
+ if hash_mode == 1 && uri.fragment do
+ pathname <> "#" <> URI.decode(uri.fragment)
+ else
+ pathname
+ end
+ end
+
+ defp put_props(%{} = event, %Request{} = request) do
+ if is_map(request.props) do
+ event
+ |> Map.put(:"meta.key", Map.keys(request.props))
+ |> Map.put(:"meta.value", Map.values(request.props) |> Enum.map(&to_string/1))
+ else
+ event
+ end
+ end
+
+ defp put_referrer(%{} = event, %Request{} = request) do
+ uri = request.url && URI.parse(request.url)
+ ref = parse_referrer(uri, request.referrer)
+
+ event
+ |> Map.put(:utm_medium, request.query_params["utm_medium"])
+ |> Map.put(:utm_source, request.query_params["utm_source"])
+ |> Map.put(:utm_campaign, request.query_params["utm_campaign"])
+ |> Map.put(:utm_content, request.query_params["utm_content"])
+ |> Map.put(:utm_term, request.query_params["utm_term"])
+ |> Map.put(:referrer_source, get_referrer_source(request, ref))
+ |> Map.put(:referrer, clean_referrer(ref))
+ end
+
+ defp parse_referrer(_uri, _referrer_str = nil), do: nil
+
+ defp parse_referrer(uri, referrer_str) do
+ referrer_uri = URI.parse(referrer_str)
+
+ if strip_www(referrer_uri.host) !== strip_www(uri.host) && referrer_uri.host !== "localhost" do
+ RefInspector.parse(referrer_str)
+ end
+ end
+
+ defp get_referrer_source(request, ref) do
+ source =
+ request.query_params["utm_source"] ||
+ request.query_params["source"] ||
+ request.query_params["ref"]
+
+ source || PlausibleWeb.RefInspector.parse(ref)
+ end
+
+ defp clean_referrer(nil), do: nil
+
+ defp clean_referrer(ref) do
+ uri = URI.parse(ref.referer)
+
+ if PlausibleWeb.RefInspector.right_uri?(uri) do
+ host = String.replace_prefix(uri.host, "www.", "")
+ path = uri.path || ""
+ host <> String.trim_trailing(path, "/")
+ end
+ end
+
+ defp put_user_agent(%{} = event, %Request{} = request) do
+ case parse_user_agent(request) do
+ %UAInspector.Result{client: %UAInspector.Result.Client{name: "Headless Chrome"}} ->
+ :skip
+
+ %UAInspector.Result.Bot{} ->
+ :skip
+
+ %UAInspector.Result{} = user_agent ->
+ event
+ |> Map.put(:operating_system, os_name(user_agent))
+ |> Map.put(:operating_system_version, os_version(user_agent))
+ |> Map.put(:browser, browser_name(user_agent))
+ |> Map.put(:browser_version, browser_version(user_agent))
+
+ _any ->
+ event
+ end
+ end
+
+ defp parse_user_agent(%Request{user_agent: user_agent}) when is_binary(user_agent) do
+ Tracer.with_span "parse_user_agent" do
+ case Cachex.fetch(:user_agents, user_agent, &UAInspector.parse/1) do
+ {:ok, user_agent} -> user_agent
+ {:commit, user_agent} -> user_agent
+ _ -> nil
+ end
+ end
+ end
+
+ defp parse_user_agent(request), do: request
+
+ defp browser_name(ua) do
+ case ua.client do
+ :unknown -> ""
+ %UAInspector.Result.Client{name: "Mobile Safari"} -> "Safari"
+ %UAInspector.Result.Client{name: "Chrome Mobile"} -> "Chrome"
+ %UAInspector.Result.Client{name: "Chrome Mobile iOS"} -> "Chrome"
+ %UAInspector.Result.Client{name: "Firefox Mobile"} -> "Firefox"
+ %UAInspector.Result.Client{name: "Firefox Mobile iOS"} -> "Firefox"
+ %UAInspector.Result.Client{name: "Opera Mobile"} -> "Opera"
+ %UAInspector.Result.Client{name: "Opera Mini"} -> "Opera"
+ %UAInspector.Result.Client{name: "Opera Mini iOS"} -> "Opera"
+ %UAInspector.Result.Client{name: "Yandex Browser Lite"} -> "Yandex Browser"
+ %UAInspector.Result.Client{name: "Chrome Webview"} -> "Mobile App"
+ %UAInspector.Result.Client{type: "mobile app"} -> "Mobile App"
+ client -> client.name
+ end
+ end
+
+ defp browser_version(ua) do
+ case ua.client do
+ :unknown -> ""
+ %UAInspector.Result.Client{type: "mobile app"} -> ""
+ client -> major_minor(client.version)
+ end
+ end
+
+ defp os_name(ua) do
+ case ua.os do
+ :unknown -> ""
+ os -> os.name
+ end
+ end
+
+ defp os_version(ua) do
+ case ua.os do
+ :unknown -> ""
+ os -> major_minor(os.version)
+ end
+ end
+
+ defp major_minor(version) do
+ case version do
+ :unknown ->
+ ""
+
+ version ->
+ version
+ |> String.split(".")
+ |> Enum.take(2)
+ |> Enum.join(".")
+ end
+ end
+
+ defp put_screen_size(%{} = event, %Request{} = request) do
+ screen_width =
+ case request.screen_width do
+ nil -> nil
+ width when width < 576 -> "Mobile"
+ width when width < 992 -> "Tablet"
+ width when width < 1440 -> "Laptop"
+ width when width >= 1440 -> "Desktop"
+ end
+
+ Map.put(event, :screen_size, screen_width)
+ end
+
+ defp put_geolocation(%{} = event, %Request{} = request) do
+ Tracer.with_span "parse_visitor_location" do
+ result = Geolix.lookup(request.remote_ip, where: :geolocation)
+
+ country_code =
+ get_in(result, [:country, :iso_code])
+ |> ignore_unknown_country()
+
+ city_geoname_id = get_in(result, [:city, :geoname_id])
+ city_geoname_id = Map.get(CityOverrides.get(), city_geoname_id, city_geoname_id)
+
+ subdivision1_code =
+ case result do
+ %{subdivisions: [%{iso_code: iso_code} | _rest]} ->
+ country_code <> "-" <> iso_code
+
+ _ ->
+ ""
+ end
+
+ subdivision2_code =
+ case result do
+ %{subdivisions: [_first, %{iso_code: iso_code} | _rest]} ->
+ country_code <> "-" <> iso_code
+
+ _ ->
+ ""
+ end
+
+ event
+ |> Map.put(:country_code, country_code)
+ |> Map.put(:subdivision1_code, subdivision1_code)
+ |> Map.put(:subdivision2_code, subdivision2_code)
+ |> Map.put(:city_geoname_id, city_geoname_id)
+ end
+ end
+
+ defp ignore_unknown_country("ZZ"), do: nil
+ defp ignore_unknown_country(country), do: country
+
+ defp map_domains(%{} = event, %Request{} = request) do
+ domains =
+ if request.domain do
+ String.split(request.domain, ",")
+ |> Enum.map(&String.trim/1)
+ |> Enum.map(&strip_www/1)
+ else
+ uri = request.url && URI.parse(request.url)
+ [strip_www(uri && uri.host)]
+ end
+
+ for domain <- domains, do: Map.put(event, :domain, domain)
+ end
+
+ defp put_user_id(events, %Request{} = request, salts) do
+ for %{} = event <- events do
+ user_id = generate_user_id(request, event.domain, event.hostname, salts.current)
+ Map.put(event, :user_id, user_id)
+ end
+ end
+
+ defp register_session(events, %Request{} = request, salts) do
+ for %Plausible.ClickhouseEvent{} = event <- events do
+ previous_user_id = generate_user_id(request, event.domain, event.hostname, salts.previous)
+
+ session_id =
+ Tracer.with_span "cache_store_event" do
+ Plausible.Session.CacheStore.on_event(event, previous_user_id)
+ end
+
+ Map.put(event, :session_id, session_id)
+ end
+ end
+
+ defp generate_user_id(request, domain, hostname, salt) do
+ cond do
+ is_nil(salt) ->
+ nil
+
+ is_nil(domain) ->
+ nil
+
+ true ->
+ user_agent = request.user_agent || ""
+ root_domain = get_root_domain(hostname)
+
+ SipHash.hash!(salt, user_agent <> request.remote_ip <> domain <> root_domain)
+ end
+ end
+
+ defp get_root_domain(nil), do: "(none)"
+
+ defp get_root_domain(hostname) do
+ case PublicSuffix.registrable_domain(hostname) do
+ domain when is_binary(domain) -> domain
+ _any -> hostname
+ end
+ end
+
+ defp spam_or_blocked?(%Request{} = request) do
+ cond do
+ request.domain in Application.get_env(:plausible, :domain_blacklist) ->
+ :skip
+
+ FunWithFlags.enabled?(:block_event_ingest, for: request.domain) ->
+ Tracer.set_attribute("blocked_by_flag", true)
+ :skip
+
+ request.referrer &&
+ URI.parse(request.referrer).host |> strip_www() |> ReferrerBlocklist.is_spammer?() ->
+ :skip
+
+ true ->
+ :ok
+ end
+ end
+
+ defp validate_events(events) do
+ Enum.reduce_while(events, {:ok, []}, fn %{} = attrs, {:ok, acc} ->
+ attrs
+ |> Plausible.ClickhouseEvent.new()
+ |> Ecto.Changeset.apply_action(nil)
+ |> case do
+ {:ok, event} -> {:cont, {:ok, [event | acc]}}
+ {:error, changeset} -> {:halt, {:error, changeset}}
+ end
+ end)
+ end
+
+ defp strip_www(hostname) do
+ if hostname do
+ String.replace_prefix(hostname, "www.", "")
+ else
+ nil
+ end
+ end
+end
diff --git a/lib/plausible/ingestion/ingestion.ex b/lib/plausible/ingestion/ingestion.ex
deleted file mode 100644
index b97e7020de..0000000000
--- a/lib/plausible/ingestion/ingestion.ex
+++ /dev/null
@@ -1,496 +0,0 @@
-defmodule Plausible.Ingestion do
- require OpenTelemetry.Tracer, as: Tracer
-
- @no_domain_error {:error, %{domain: ["can't be blank"]}}
-
- def add_to_buffer(%Plausible.Ingestion.Request{} = request) do
- ua =
- Tracer.with_span "parse_user_agent" do
- parse_user_agent(request)
- end
-
- blacklist_domain = request.params.domain in Application.get_env(:plausible, :domain_blacklist)
-
- if blacklist_domain || is_bot?(ua) || is_spammer?(request.params.referrer) ||
- blocked_via_flag?(request.params.domain) do
- :ok
- else
- uri = request.params.url && URI.parse(request.params.url)
- host = if uri && uri.host == "", do: "(none)", else: uri && uri.host
-
- ref = parse_referrer(uri, request.params.referrer)
-
- location_details =
- Tracer.with_span "parse_visitor_location" do
- visitor_location_details(request)
- end
-
- salts = Plausible.Session.Salts.fetch()
-
- event_attrs = %{
- timestamp: NaiveDateTime.utc_now() |> NaiveDateTime.truncate(:second),
- name: request.params.name,
- hostname: strip_www(host),
- pathname: get_pathname(uri, request.params.hash_mode),
- referrer_source: get_referrer_source(request, ref),
- referrer: clean_referrer(ref),
- utm_medium: request.query_params["utm_medium"],
- utm_source: request.query_params["utm_source"],
- utm_campaign: request.query_params["utm_campaign"],
- utm_content: request.query_params["utm_content"],
- utm_term: request.query_params["utm_term"],
- country_code: location_details[:country_code],
- country_geoname_id: location_details[:country_geoname_id],
- subdivision1_code: location_details[:subdivision1_code],
- subdivision2_code: location_details[:subdivision2_code],
- city_geoname_id: location_details[:city_geoname_id],
- operating_system: ua && os_name(ua),
- operating_system_version: ua && os_version(ua),
- browser: ua && browser_name(ua),
- browser_version: ua && browser_version(ua),
- screen_size: calculate_screen_size(request.params.screen_width),
- "meta.key": Map.keys(request.params.meta),
- "meta.value": Map.values(request.params.meta) |> Enum.map(&Kernel.to_string/1)
- }
-
- Enum.reduce_while(get_domains(request, uri), @no_domain_error, fn domain, _res ->
- user_id = generate_user_id(request, domain, event_attrs[:hostname], salts[:current])
-
- previous_user_id =
- salts[:previous] &&
- generate_user_id(request, domain, event_attrs[:hostname], salts[:previous])
-
- changeset =
- event_attrs
- |> Map.merge(%{domain: domain, user_id: user_id})
- |> Plausible.ClickhouseEvent.new()
-
- if changeset.valid? do
- event = Ecto.Changeset.apply_changes(changeset)
-
- session_id =
- Tracer.with_span "cache_store_event" do
- Plausible.Session.CacheStore.on_event(event, previous_user_id)
- end
-
- event
- |> Map.put(:session_id, session_id)
- |> Plausible.Event.WriteBuffer.insert()
-
- {:cont, :ok}
- else
- errors = Ecto.Changeset.traverse_errors(changeset, &encode_error/1)
- {:halt, {:error, errors}}
- end
- end)
- end
- end
-
- defp blocked_via_flag?(domain) do
- blocked? = FunWithFlags.enabled?(:block_event_ingest, for: domain)
- Tracer.set_attribute("blocked_by_flag", blocked?)
- blocked?
- end
-
- # https://hexdocs.pm/ecto/Ecto.Changeset.html#traverse_errors/2-examples
- defp encode_error({msg, opts}) do
- Regex.replace(~r"%{(\w+)}", msg, fn _, key ->
- opts |> Keyword.get(String.to_existing_atom(key), key) |> to_string()
- end)
- end
-
- defp is_bot?(%UAInspector.Result.Bot{}), do: true
-
- defp is_bot?(%UAInspector.Result{client: %UAInspector.Result.Client{name: "Headless Chrome"}}),
- do: true
-
- defp is_bot?(_), do: false
-
- defp is_spammer?(nil), do: false
-
- defp is_spammer?(referrer_str) do
- uri = URI.parse(referrer_str)
- ReferrerBlocklist.is_spammer?(strip_www(uri.host))
- end
-
- defp get_domains(request, uri) do
- if request.params.domain do
- String.split(request.params.domain, ",")
- |> Enum.map(&String.trim/1)
- |> Enum.map(&strip_www/1)
- else
- List.wrap(strip_www(uri && uri.host))
- end
- end
-
- defp get_pathname(nil, _), do: "/"
-
- defp get_pathname(uri, hash_mode) do
- pathname =
- (uri.path || "/")
- |> URI.decode()
- |> String.trim_trailing()
-
- if hash_mode == 1 && uri.fragment do
- pathname <> "#" <> URI.decode(uri.fragment)
- else
- pathname
- end
- end
-
- @city_overrides %{
- # Austria
- # Gemeindebezirk Floridsdorf -> Vienna
- 2_779_467 => 2_761_369,
- # Gemeindebezirk Leopoldstadt -> Vienna
- 2_772_614 => 2_761_369,
- # Gemeindebezirk Landstrasse -> Vienna
- 2_773_040 => 2_761_369,
- # Gemeindebezirk Donaustadt -> Vienna
- 2_780_851 => 2_761_369,
- # Gemeindebezirk Favoriten -> Vienna
- 2_779_776 => 2_761_369,
- # Gemeindebezirk Währing -> Vienna
- 2_762_091 => 2_761_369,
- # Gemeindebezirk Wieden -> Vienna
- 2_761_393 => 2_761_369,
- # Gemeindebezirk Innere Stadt -> Vienna
- 2_775_259 => 2_761_369,
- # Gemeindebezirk Alsergrund -> Vienna
- 2_782_729 => 2_761_369,
- # Gemeindebezirk Liesing -> Vienna
- 2_772_484 => 2_761_369,
- # Urfahr -> Linz
- 2_762_518 => 2_772_400,
-
- # Canada
- # Old Toronto -> Toronto
- 8_436_019 => 6_167_865,
- # Etobicoke -> Toronto
- 5_950_267 => 6_167_865,
- # East York -> Toronto
- 5_946_235 => 6_167_865,
- # Scarborough -> Toronto
- 6_948_711 => 6_167_865,
- # North York -> Toronto
- 6_091_104 => 6_167_865,
-
- # Czech republic
- # Praha 5 -> Prague
- 11_951_220 => 3_067_696,
- # Praha 4 -> Prague
- 11_951_218 => 3_067_696,
- # Praha 11 -> Prague
- 11_951_232 => 3_067_696,
- # Praha 10 -> Prague
- 11_951_210 => 3_067_696,
- # Praha 4 -> Prague
- 8_378_772 => 3_067_696,
-
- # Denmark
- # København SV -> Copenhagen
- 11_747_123 => 2_618_425,
- # København NV -> Copenhagen
- 11_746_894 => 2_618_425,
- # Odense S -> Odense
- 11_746_825 => 2_615_876,
- # Odense M -> Odense
- 11_746_974 => 2_615_876,
- # Odense SØ -> Odense
- 11_746_888 => 2_615_876,
- # Aarhus C -> Aarhus
- 11_746_746 => 2_624_652,
- # Aarhus N -> Aarhus
- 11_746_890 => 2_624_652,
-
- # Estonia
- # Kristiine linnaosa -> Tallinn
- 11_050_530 => 588_409,
- # Kesklinna linnaosa -> Tallinn
- 11_053_706 => 588_409,
- # Lasnamäe linnaosa -> Tallinn
- 11_050_526 => 588_409,
- # Põhja-Tallinna linnaosa -> Tallinn
- 11_049_594 => 588_409,
- # Mustamäe linnaosa -> Tallinn
- 11_050_531 => 588_409,
- # Haabersti linnaosa -> Tallinn
- 11_053_707 => 588_409,
- # Viimsi -> Tallinn
- 587_629 => 588_409,
-
- # Germany
- # Bezirk Tempelhof-Schöneberg -> Berlin
- 3_336_297 => 2_950_159,
- # Bezirk Mitte -> Berlin
- 2_870_912 => 2_950_159,
- # Bezirk Charlottenburg-Wilmersdorf -> Berlin
- 3_336_294 => 2_950_159,
- # Bezirk Friedrichshain-Kreuzberg -> Berlin
- 3_336_295 => 2_950_159,
- # Moosach -> Munich
- 8_351_447 => 2_867_714,
- # Schwabing-Freimann -> Munich
- 8_351_448 => 2_867_714,
- # Stadtbezirk 06 -> Düsseldorf
- 6_947_276 => 2_934_246,
- # Stadtbezirk 04 -> Düsseldorf
- 6_947_274 => 2_934_246,
- # Köln-Ehrenfeld -> Köln
- 6_947_479 => 2_886_242,
- # Köln-Lindenthal- -> Köln
- 6_947_481 => 2_886_242,
- # Beuel -> Bonn
- 2_949_619 => 2_946_447,
- # Innenstadt I -> Frankfurt am Main
- 6_946_225 => 2_925_533,
-
- # India
- # Navi Mumbai -> Mumbai
- 6_619_347 => 1_275_339,
-
- # Mexico
- # Miguel Hidalgo Villa Olímpica -> Mexico city
- 11_561_026 => 3_530_597,
- # Zedec Santa Fe -> Mexico city
- 3_517_471 => 3_530_597,
- # Fuentes del Pedregal-> Mexico city
- 11_562_596 => 3_530_597,
- # Centro -> Mexico city
- 9_179_691 => 3_530_597,
- # Cuauhtémoc-> Mexico city
- 12_266_959 => 3_530_597,
-
- # Netherlands
- # Schiphol-Rijk -> Amsterdam
- 10_173_838 => 2_759_794,
- # Westpoort -> Amsterdam
- 11_525_047 => 2_759_794,
- # Amsterdam-Zuidoost -> Amsterdam
- 6_544_881 => 2_759_794,
- # Loosduinen -> The Hague
- 11_525_037 => 2_747_373,
- # Laak -> The Hague
- 11_525_042 => 2_747_373,
-
- # Norway
- # Nordre Aker District -> Oslo
- 6_940_981 => 3_143_244,
-
- # Romania
- # Sector 1 -> Bucharest,
- 11_055_041 => 683_506,
- # Sector 2 -> Bucharest
- 11_055_040 => 683_506,
- # Sector 3 -> Bucharest
- 11_055_044 => 683_506,
- # Sector 4 -> Bucharest
- 11_055_042 => 683_506,
- # Sector 5 -> Bucharest
- 11_055_043 => 683_506,
- # Sector 6 -> Bucharest
- 11_055_039 => 683_506,
- # Bucuresti -> Bucharest
- 6_691_781 => 683_506,
-
- # Slovakia
- # Bratislava -> Bratislava
- 3_343_955 => 3_060_972,
-
- # Sweden
- # Södermalm -> Stockholm
- 2_676_209 => 2_673_730,
-
- # Switzerland
- # Vorstädte -> Basel
- 11_789_440 => 2_661_604,
- # Zürich (Kreis 11) / Oerlikon -> Zürich
- 2_659_310 => 2_657_896,
- # Zürich (Kreis 3) / Alt-Wiedikon -> Zürich
- 2_658_007 => 2_657_896,
- # Zürich (Kreis 5) -> Zürich
- 6_295_521 => 2_657_896,
- # Zürich (Kreis 1) / Hochschulen -> Zürich
- 6_295_489 => 2_657_896,
-
- # UK
- # Shadwell -> London
- 6_690_595 => 2_643_743,
- # City of London -> London
- 2_643_741 => 2_643_743,
- # South Bank -> London
- 6_545_251 => 2_643_743,
- # Soho -> London
- 6_545_173 => 2_643_743,
- # Whitechapel -> London
- 2_634_112 => 2_643_743,
- # King's Cross -> London
- 6_690_589 => 2_643_743,
- # Poplar -> London
- 2_640_091 => 2_643_743,
- # Hackney -> London
- 2_647_694 => 2_643_743
- }
-
- defp visitor_location_details(request) do
- result = Geolix.lookup(request.remote_ip, where: :geolocation)
-
- country_code =
- get_in(result, [:country, :iso_code])
- |> ignore_unknown_country()
-
- city_geoname_id = get_in(result, [:city, :geoname_id])
-
- subdivision1_code =
- case result do
- %{subdivisions: [%{iso_code: iso_code} | _rest]} ->
- country_code <> "-" <> iso_code
-
- _ ->
- ""
- end
-
- subdivision2_code =
- case result do
- %{subdivisions: [_first, %{iso_code: iso_code} | _rest]} ->
- country_code <> "-" <> iso_code
-
- _ ->
- ""
- end
-
- %{
- country_code: country_code,
- subdivision1_code: subdivision1_code,
- subdivision2_code: subdivision2_code,
- city_geoname_id: Map.get(@city_overrides, city_geoname_id, city_geoname_id)
- }
- end
-
- defp ignore_unknown_country("ZZ"), do: nil
- defp ignore_unknown_country(country), do: country
-
- defp parse_referrer(_, nil), do: nil
-
- defp parse_referrer(uri, referrer_str) do
- referrer_uri = URI.parse(referrer_str)
-
- if strip_www(referrer_uri.host) !== strip_www(uri.host) && referrer_uri.host !== "localhost" do
- RefInspector.parse(referrer_str)
- end
- end
-
- defp generate_user_id(request, domain, hostname, salt) do
- user_agent = request.headers["user-agent"] || ""
- root_domain = get_root_domain(hostname)
-
- if domain && root_domain do
- SipHash.hash!(salt, user_agent <> request.remote_ip <> domain <> root_domain)
- end
- end
-
- defp get_root_domain(nil), do: "(none)"
-
- defp get_root_domain(hostname) do
- case PublicSuffix.registrable_domain(hostname) do
- domain when is_binary(domain) -> domain
- _ -> hostname
- end
- end
-
- defp calculate_screen_size(nil), do: nil
- defp calculate_screen_size(width) when width < 576, do: "Mobile"
- defp calculate_screen_size(width) when width < 992, do: "Tablet"
- defp calculate_screen_size(width) when width < 1440, do: "Laptop"
- defp calculate_screen_size(width) when width >= 1440, do: "Desktop"
-
- defp clean_referrer(nil), do: nil
-
- defp clean_referrer(ref) do
- uri = URI.parse(ref.referer)
-
- if PlausibleWeb.RefInspector.right_uri?(uri) do
- host = String.replace_prefix(uri.host, "www.", "")
- path = uri.path || ""
- host <> String.trim_trailing(path, "/")
- end
- end
-
- defp strip_www(nil), do: nil
-
- defp strip_www(hostname) do
- String.replace_prefix(hostname, "www.", "")
- end
-
- defp browser_name(ua) do
- case ua.client do
- :unknown -> ""
- %UAInspector.Result.Client{name: "Mobile Safari"} -> "Safari"
- %UAInspector.Result.Client{name: "Chrome Mobile"} -> "Chrome"
- %UAInspector.Result.Client{name: "Chrome Mobile iOS"} -> "Chrome"
- %UAInspector.Result.Client{name: "Firefox Mobile"} -> "Firefox"
- %UAInspector.Result.Client{name: "Firefox Mobile iOS"} -> "Firefox"
- %UAInspector.Result.Client{name: "Opera Mobile"} -> "Opera"
- %UAInspector.Result.Client{name: "Opera Mini"} -> "Opera"
- %UAInspector.Result.Client{name: "Opera Mini iOS"} -> "Opera"
- %UAInspector.Result.Client{name: "Yandex Browser Lite"} -> "Yandex Browser"
- %UAInspector.Result.Client{name: "Chrome Webview"} -> "Mobile App"
- %UAInspector.Result.Client{type: "mobile app"} -> "Mobile App"
- client -> client.name
- end
- end
-
- defp major_minor(:unknown), do: ""
-
- defp major_minor(version) do
- version
- |> String.split(".")
- |> Enum.take(2)
- |> Enum.join(".")
- end
-
- defp browser_version(ua) do
- case ua.client do
- :unknown -> ""
- %UAInspector.Result.Client{type: "mobile app"} -> ""
- client -> major_minor(client.version)
- end
- end
-
- defp os_name(ua) do
- case ua.os do
- :unknown -> ""
- os -> os.name
- end
- end
-
- defp os_version(ua) do
- case ua.os do
- :unknown -> ""
- os -> major_minor(os.version)
- end
- end
-
- defp get_referrer_source(request, ref) do
- source =
- request.query_params["utm_source"] || request.query_params["source"] ||
- request.query_params["ref"]
-
- source || PlausibleWeb.RefInspector.parse(ref)
- end
-
- defp parse_user_agent(%Plausible.Ingestion.Request{} = request) do
- if user_agent = request.headers["user-agent"] do
- res =
- Cachex.fetch(:user_agents, user_agent, fn ua ->
- UAInspector.parse(ua)
- end)
-
- case res do
- {:ok, user_agent} -> user_agent
- {:commit, user_agent} -> user_agent
- _ -> nil
- end
- end
- end
-end
diff --git a/lib/plausible/ingestion/request.ex b/lib/plausible/ingestion/request.ex
index 164aeba43b..02e69bcccc 100644
--- a/lib/plausible/ingestion/request.ex
+++ b/lib/plausible/ingestion/request.ex
@@ -1,33 +1,46 @@
defmodule Plausible.Ingestion.Request do
- defstruct ~w(remote_ip params query_params headers)a
+ @moduledoc """
+ The %Plausible.Ingestion.Request{} struct stores all needed fields to create an event downstream.
+ """
+
+ defstruct [
+ :remote_ip,
+ :user_agent,
+ :event_name,
+ :url,
+ :referrer,
+ :domain,
+ :screen_width,
+ :hash_mode,
+ props: %{},
+ query_params: %{}
+ ]
@type t() :: %__MODULE__{
remote_ip: String.t() | nil,
- params: map(),
- query_params: map(),
- headers: map()
+ user_agent: String.t() | nil,
+ event_name: term(),
+ url: term(),
+ referrer: term(),
+ domain: term(),
+ screen_width: term(),
+ hash_mode: term(),
+ props: map(),
+ query_params: map()
}
- @allowed_query_params ~w(utm_medium utm_source utm_campaign utm_content utm_term utm_source source ref)
- @allowed_headers ~w(user-agent)
-
@spec build(Plug.Conn.t()) :: {:ok, t()} | {:error, :invalid_json}
@doc """
Builds a %Plausible.Ingestion.Request{} struct from %Plug.Conn{}.
"""
def build(%Plug.Conn{} = conn) do
- with {:ok, body} <- parse_body(conn),
- %{} = params <- build_params(body),
- %{} = query_params <- decode_query_params(params),
- %{} = headers <- build_headers(conn),
- remote_ip <- PlausibleWeb.RemoteIp.get(conn) do
- {:ok,
- %__MODULE__{
- remote_ip: remote_ip,
- params: params,
- query_params: query_params,
- headers: headers
- }}
+ with {:ok, request_body} <- parse_body(conn) do
+ %__MODULE__{}
+ |> Map.put(:remote_ip, PlausibleWeb.RemoteIp.get(conn))
+ |> put_user_agent(conn)
+ |> put_request_params(request_body)
+ |> put_query_params()
+ |> then(&{:ok, &1})
end
end
@@ -46,33 +59,30 @@ defmodule Plausible.Ingestion.Request do
end
end
- defp build_params(body) do
- %{
- name: body["n"] || body["name"],
- url: body["u"] || body["url"],
- referrer: body["r"] || body["referrer"],
- domain: body["d"] || body["domain"],
- screen_width: body["w"] || body["screen_width"],
- hash_mode: body["h"] || body["hashMode"],
- meta: parse_meta(body)
+ defp put_request_params(%__MODULE__{} = request, %{} = request_body) do
+ %__MODULE__{
+ request
+ | event_name: request_body["n"] || request_body["name"],
+ url: request_body["u"] || request_body["url"],
+ referrer: request_body["r"] || request_body["referrer"],
+ domain: request_body["d"] || request_body["domain"],
+ screen_width: request_body["w"] || request_body["screen_width"],
+ hash_mode: request_body["h"] || request_body["hashMode"],
+ props: parse_props(request_body)
}
end
- defp parse_meta(params) do
- raw_meta = params["m"] || params["meta"] || params["p"] || params["props"]
+ defp parse_props(%{} = request_body) do
+ raw_props =
+ request_body["m"] || request_body["meta"] || request_body["p"] || request_body["props"]
- case decode_raw_props(raw_meta) do
+ case decode_raw_props(raw_props) do
{:ok, parsed_json} ->
- Enum.filter(parsed_json, fn
- {_, ""} -> false
- {_, nil} -> false
- {_, val} when is_list(val) -> false
- {_, val} when is_map(val) -> false
- _ -> true
- end)
+ parsed_json
+ |> Enum.filter(&valid_prop_value?/1)
|> Map.new()
- _ ->
+ _error ->
%{}
end
end
@@ -91,32 +101,32 @@ defmodule Plausible.Ingestion.Request do
defp decode_raw_props(_), do: :bad_format
- defp decode_query_params(params) do
- with url when is_binary(url) <- params.url,
- %URI{query: query} when is_binary(query) <- URI.parse(url) do
- do_decode_query_params(query)
- else
- _any -> %{}
+ defp valid_prop_value?({key, value}) do
+ case {key, value} do
+ {_key, ""} -> false
+ {_key, nil} -> false
+ {_key, value} when is_list(value) -> false
+ {_key, value} when is_map(value) -> false
+ {_key, _value} -> true
end
end
- defp do_decode_query_params(query) do
- try do
- query
- |> URI.query_decoder()
- |> Enum.reduce(%{}, fn
- {key, value}, acc when key in @allowed_query_params -> Map.put(acc, key, value)
- _any, acc -> acc
- end)
- rescue
- _ -> %{}
+ defp put_query_params(%__MODULE__{url: url} = request) do
+ with url when is_binary(url) <- url,
+ %URI{query: query} when is_binary(query) <- URI.parse(url),
+ %{} = query_params <- URI.decode_query(query) do
+ Map.put(request, :query_params, query_params)
+ else
+ _any -> request
end
end
- defp build_headers(conn) do
- Enum.reduce(@allowed_headers, %{}, fn header, acc ->
- value = conn |> Plug.Conn.get_req_header(header) |> List.first()
- if value, do: Map.put(acc, header, value), else: acc
- end)
+ defp put_user_agent(%__MODULE__{} = request, %Plug.Conn{} = conn) do
+ user_agent =
+ conn
+ |> Plug.Conn.get_req_header("user-agent")
+ |> List.first()
+
+ %__MODULE__{request | user_agent: user_agent}
end
end
diff --git a/lib/plausible_web/controllers/api/external_controller.ex b/lib/plausible_web/controllers/api/external_controller.ex
index 70cd875ca8..6a47218610 100644
--- a/lib/plausible_web/controllers/api/external_controller.ex
+++ b/lib/plausible_web/controllers/api/external_controller.ex
@@ -10,16 +10,26 @@ defmodule PlausibleWeb.Api.ExternalController do
def event(conn, _params) do
with {:ok, ingestion_request} <- Plausible.Ingestion.Request.build(conn),
- _ <- Sentry.Context.set_extra_context(%{request: ingestion_request.params}),
- :ok <- Plausible.Ingestion.add_to_buffer(ingestion_request) do
+ _ <- Sentry.Context.set_extra_context(%{request: ingestion_request}),
+ :ok <- Plausible.Ingestion.Event.build_and_buffer(ingestion_request) do
conn |> put_status(202) |> text("ok")
else
+ :skip ->
+ conn |> put_status(202) |> text("ok")
+
{:error, :invalid_json} ->
conn
|> put_status(400)
|> json(%{errors: %{request: "Unable to parse request body as json"}})
- {:error, errors} ->
+ {:error, %Ecto.Changeset{} = changeset} ->
+ errors =
+ Ecto.Changeset.traverse_errors(changeset, fn {msg, opts} ->
+ Regex.replace(~r"%{(\w+)}", msg, fn _, key ->
+ opts |> Keyword.get(String.to_existing_atom(key), key) |> to_string()
+ end)
+ end)
+
conn |> put_status(400) |> json(%{errors: errors})
end
end
diff --git a/test/plausible/ingestion/event_test.exs b/test/plausible/ingestion/event_test.exs
new file mode 100644
index 0000000000..e306bba2a1
--- /dev/null
+++ b/test/plausible/ingestion/event_test.exs
@@ -0,0 +1,74 @@
+defmodule Plausible.Ingestion.EventTest do
+ use Plausible.DataCase
+
+ def get_event(domain) do
+ Plausible.TestUtils.eventually(fn ->
+ Plausible.Event.WriteBuffer.flush()
+
+ event =
+ Plausible.ClickhouseRepo.one(
+ from e in Plausible.ClickhouseEvent, where: e.domain == ^domain
+ )
+
+ {!is_nil(event), event}
+ end)
+ end
+
+ @valid_request %Plausible.Ingestion.Request{
+ remote_ip: "2.2.2.2",
+ user_agent:
+ "Mozilla/5.0 (iPad; U; CPU OS 3_2_1 like Mac OS X; en-us) AppleWebKit/531.21.10 (KHTML, like Gecko) Mobile/7B405",
+ event_name: "pageview",
+ url: "http://skywalker.test",
+ referrer: "http://m.facebook.test/",
+ screen_width: 1440,
+ hash_mode: nil,
+ query_params: %{
+ "utm_medium" => "utm_medium",
+ "utm_source" => "utm_source",
+ "utm_campaign" => "utm_campaign",
+ "utm_content" => "utm_content",
+ "utm_term" => "utm_term",
+ "source" => "source",
+ "ref" => "ref"
+ }
+ }
+
+ test "build_and_buffer/3 creates an event" do
+ assert :ok ==
+ @valid_request
+ |> Map.put(:domain, "plausible-ingestion-event-basic.test")
+ |> Plausible.Ingestion.Event.build_and_buffer()
+
+ assert %Plausible.ClickhouseEvent{
+ session_id: session_id,
+ user_id: user_id,
+ domain: "plausible-ingestion-event-basic.test",
+ browser: "Safari",
+ browser_version: "",
+ city_geoname_id: 2_988_507,
+ country_code: "FR",
+ hostname: "skywalker.test",
+ "meta.key": [],
+ "meta.value": [],
+ name: "pageview",
+ operating_system: "iOS",
+ operating_system_version: "3.2",
+ pathname: "/",
+ referrer: "m.facebook.test",
+ referrer_source: "utm_source",
+ screen_size: "Desktop",
+ subdivision1_code: "FR-IDF",
+ subdivision2_code: "FR-75",
+ transferred_from: "",
+ utm_campaign: "utm_campaign",
+ utm_content: "utm_content",
+ utm_medium: "utm_medium",
+ utm_source: "utm_source",
+ utm_term: "utm_term"
+ } = get_event("plausible-ingestion-event-basic.test")
+
+ assert is_integer(session_id)
+ assert is_integer(user_id)
+ end
+end
diff --git a/test/plausible_web/controllers/api/external_controller_test.exs b/test/plausible_web/controllers/api/external_controller_test.exs
index 8a1b777f0f..0f9487757f 100644
--- a/test/plausible_web/controllers/api/external_controller_test.exs
+++ b/test/plausible_web/controllers/api/external_controller_test.exs
@@ -927,7 +927,9 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
assert json_response(conn, 400) == %{
"errors" => %{
- "domain" => ["can't be blank"]
+ "domain" => ["can't be blank"],
+ "hostname" => ["can't be blank"],
+ "user_id" => ["can't be blank"]
}
}
end
diff --git a/test/support/test_utils.ex b/test/support/test_utils.ex
index 350e1398b9..d2977f61df 100644
--- a/test/support/test_utils.ex
+++ b/test/support/test_utils.ex
@@ -143,4 +143,17 @@ defmodule Plausible.TestUtils do
|> Timex.shift(shifts)
|> NaiveDateTime.truncate(:second)
end
+
+ def eventually(expectation, wait_time_ms \\ 50, retries \\ 10) do
+ Enum.reduce_while(1..retries, nil, fn _i, _acc ->
+ case expectation.() do
+ {true, result} ->
+ {:halt, result}
+
+ {false, _} ->
+ Process.sleep(wait_time_ms)
+ {:cont, nil}
+ end
+ end)
+ end
end