Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.
TypeScript analyticsanalytics-dashboardclickhouseelixirgoogle-analyticsmarketingopen-source-analyticsphoenixplausible-analyticspostgresqlprivacyprivacy-friendlyself-hostedsimple-analyticsstatisticstailwindcssweb-analyticswebsite-analyticswebsite-statswebsite-tracking
Refactor event struct creation function (#2098)
Vinicius Brasila10d44a0d78e
diff --git a/lib/plausible/ingestion/city_overrides.ex b/lib/plausible/ingestion/city_overrides.exnew file mode 100644index 0000000000..781c7913dc--- /dev/null+++ b/lib/plausible/ingestion/city_overrides.ex@@ -0,0 +1,198 @@+defmodule Plausible.Ingestion.CityOverrides do+ @moduledoc false++ @overrides %{+ # Austria+ # Gemeindebezirk Floridsdorf -> Vienna+ 2_779_467 => 2_761_369,+ # Gemeindebezirk Leopoldstadt -> Vienna+ 2_772_614 => 2_761_369,+ # Gemeindebezirk Landstrasse -> Vienna+ 2_773_040 => 2_761_369,+ # Gemeindebezirk Donaustadt -> Vienna+ 2_780_851 => 2_761_369,+ # Gemeindebezirk Favoriten -> Vienna+ 2_779_776 => 2_761_369,+ # Gemeindebezirk Währing -> Vienna+ 2_762_091 => 2_761_369,+ # Gemeindebezirk Wieden -> Vienna+ 2_761_393 => 2_761_369,+ # Gemeindebezirk Innere Stadt -> Vienna+ 2_775_259 => 2_761_369,+ # Gemeindebezirk Alsergrund -> Vienna+ 2_782_729 => 2_761_369,+ # Gemeindebezirk Liesing -> Vienna+ 2_772_484 => 2_761_369,+ # Urfahr -> Linz+ 2_762_518 => 2_772_400,++ # Canada+ # Old Toronto -> Toronto+ 8_436_019 => 6_167_865,+ # Etobicoke -> Toronto+ 5_950_267 => 6_167_865,+ # East York -> Toronto+ 5_946_235 => 6_167_865,+ # Scarborough -> Toronto+ 6_948_711 => 6_167_865,+ # North York -> Toronto+ 6_091_104 => 6_167_865,++ # Czech republic+ # Praha 5 -> Prague+ 11_951_220 => 3_067_696,+ # Praha 4 -> Prague+ 11_951_218 => 3_067_696,+ # Praha 11 -> Prague+ 11_951_232 => 3_067_696,+ # Praha 10 -> Prague+ 11_951_210 => 3_067_696,+ # Praha 4 -> Prague+ 8_378_772 => 3_067_696,++ # Denmark+ # København SV -> Copenhagen+ 11_747_123 => 2_618_425,+ # København NV -> Copenhagen+ 11_746_894 => 2_618_425,+ # Odense S -> Odense+ 11_746_825 => 2_615_876,+ # Odense M -> Odense+ 11_746_974 => 2_615_876,+ # Odense SØ -> Odense+ 11_746_888 => 2_615_876,+ # Aarhus C -> Aarhus+ 11_746_746 => 2_624_652,+ # Aarhus N -> Aarhus+ 11_746_890 => 2_624_652,++ # Estonia+ # Kristiine linnaosa -> Tallinn+ 11_050_530 => 588_409,+ # Kesklinna linnaosa -> Tallinn+ 11_053_706 => 588_409,+ # Lasnamäe linnaosa -> Tallinn+ 11_050_526 => 588_409,+ # Põhja-Tallinna linnaosa -> Tallinn+ 11_049_594 => 588_409,+ # Mustamäe linnaosa -> Tallinn+ 11_050_531 => 588_409,+ # Haabersti linnaosa -> Tallinn+ 11_053_707 => 588_409,+ # Viimsi -> Tallinn+ 587_629 => 588_409,++ # Germany+ # Bezirk Tempelhof-Schöneberg -> Berlin+ 3_336_297 => 2_950_159,+ # Bezirk Mitte -> Berlin+ 2_870_912 => 2_950_159,+ # Bezirk Charlottenburg-Wilmersdorf -> Berlin+ 3_336_294 => 2_950_159,+ # Bezirk Friedrichshain-Kreuzberg -> Berlin+ 3_336_295 => 2_950_159,+ # Moosach -> Munich+ 8_351_447 => 2_867_714,+ # Schwabing-Freimann -> Munich+ 8_351_448 => 2_867_714,+ # Stadtbezirk 06 -> Düsseldorf+ 6_947_276 => 2_934_246,+ # Stadtbezirk 04 -> Düsseldorf+ 6_947_274 => 2_934_246,+ # Köln-Ehrenfeld -> Köln+ 6_947_479 => 2_886_242,+ # Köln-Lindenthal- -> Köln+ 6_947_481 => 2_886_242,+ # Beuel -> Bonn+ 2_949_619 => 2_946_447,+ # Innenstadt I -> Frankfurt am Main+ 6_946_225 => 2_925_533,++ # India+ # Navi Mumbai -> Mumbai+ 6_619_347 => 1_275_339,++ # Mexico+ # Miguel Hidalgo Villa Olímpica -> Mexico city+ 11_561_026 => 3_530_597,+ # Zedec Santa Fe -> Mexico city+ 3_517_471 => 3_530_597,+ # Fuentes del Pedregal-> Mexico city+ 11_562_596 => 3_530_597,+ # Centro -> Mexico city+ 9_179_691 => 3_530_597,+ # Cuauhtémoc-> Mexico city+ 12_266_959 => 3_530_597,++ # Netherlands+ # Schiphol-Rijk -> Amsterdam+ 10_173_838 => 2_759_794,+ # Westpoort -> Amsterdam+ 11_525_047 => 2_759_794,+ # Amsterdam-Zuidoost -> Amsterdam+ 6_544_881 => 2_759_794,+ # Loosduinen -> The Hague+ 11_525_037 => 2_747_373,+ # Laak -> The Hague+ 11_525_042 => 2_747_373,++ # Norway+ # Nordre Aker District -> Oslo+ 6_940_981 => 3_143_244,++ # Romania+ # Sector 1 -> Bucharest,+ 11_055_041 => 683_506,+ # Sector 2 -> Bucharest+ 11_055_040 => 683_506,+ # Sector 3 -> Bucharest+ 11_055_044 => 683_506,+ # Sector 4 -> Bucharest+ 11_055_042 => 683_506,+ # Sector 5 -> Bucharest+ 11_055_043 => 683_506,+ # Sector 6 -> Bucharest+ 11_055_039 => 683_506,+ # Bucuresti -> Bucharest+ 6_691_781 => 683_506,++ # Slovakia+ # Bratislava -> Bratislava+ 3_343_955 => 3_060_972,++ # Sweden+ # Södermalm -> Stockholm+ 2_676_209 => 2_673_730,++ # Switzerland+ # Vorstädte -> Basel+ 11_789_440 => 2_661_604,+ # Zürich (Kreis 11) / Oerlikon -> Zürich+ 2_659_310 => 2_657_896,+ # Zürich (Kreis 3) / Alt-Wiedikon -> Zürich+ 2_658_007 => 2_657_896,+ # Zürich (Kreis 5) -> Zürich+ 6_295_521 => 2_657_896,+ # Zürich (Kreis 1) / Hochschulen -> Zürich+ 6_295_489 => 2_657_896,++ # UK+ # Shadwell -> London+ 6_690_595 => 2_643_743,+ # City of London -> London+ 2_643_741 => 2_643_743,+ # South Bank -> London+ 6_545_251 => 2_643_743,+ # Soho -> London+ 6_545_173 => 2_643_743,+ # Whitechapel -> London+ 2_634_112 => 2_643_743,+ # King's Cross -> London+ 6_690_589 => 2_643_743,+ # Poplar -> London+ 2_640_091 => 2_643_743,+ # Hackney -> London+ 2_647_694 => 2_643_743+ }+ def get, do: @overrides+enddiff --git a/lib/plausible/ingestion/event.ex b/lib/plausible/ingestion/event.exnew file mode 100644index 0000000000..f49d10fe15--- /dev/null+++ b/lib/plausible/ingestion/event.ex@@ -0,0 +1,344 @@+defmodule Plausible.Ingestion.Event do+ require OpenTelemetry.Tracer, as: Tracer+ alias Plausible.Ingestion.{Request, CityOverrides}++ @spec build_and_buffer(Request.t()) :: :ok | :skip | {:error, Ecto.Changeset.t()}+ @doc """+ Builds events from %Plausible.Ingestion.Request{} and adds them to Plausible.Event.WriteBuffer.+ This function reads geolocation data and parses the user agent string. Returns :skip if the+ request is identified as spam, or blocked.+ """+ def build_and_buffer(%Request{} = request) do+ with :ok <- spam_or_blocked?(request),+ salts <- Plausible.Session.Salts.fetch(),+ event <- Map.new(),+ %{} = event <- put_user_agent(event, request),+ %{} = event <- put_basic_info(event, request),+ %{} = event <- put_referrer(event, request),+ %{} = event <- put_geolocation(event, request),+ %{} = event <- put_screen_size(event, request),+ %{} = event <- put_props(event, request),+ events when is_list(events) <- map_domains(event, request),+ events when is_list(events) <- put_user_id(events, request, salts),+ {:ok, events} <- validate_events(events),+ events when is_list(events) <- register_session(events, request, salts) do+ Enum.each(events, &Plausible.Event.WriteBuffer.insert/1)+ end+ end++ defp put_basic_info(%{} = event, %Request{} = request) do+ uri = request.url && URI.parse(request.url)+ host = if uri && uri.host == "", do: "(none)", else: uri && uri.host++ event+ |> Map.put(:timestamp, NaiveDateTime.utc_now() |> NaiveDateTime.truncate(:second))+ |> Map.put(:name, request.event_name)+ |> Map.put(:hostname, strip_www(host))+ |> Map.put(:pathname, get_pathname(uri, request.hash_mode))+ end++ defp get_pathname(_uri = nil, _hash_mode), do: "/"++ defp get_pathname(uri, hash_mode) do+ pathname =+ (uri.path || "/")+ |> URI.decode()+ |> String.trim_trailing()++ if hash_mode == 1 && uri.fragment do+ pathname <> "#" <> URI.decode(uri.fragment)+ else+ pathname+ end+ end++ defp put_props(%{} = event, %Request{} = request) do+ if is_map(request.props) do+ event+ |> Map.put(:"meta.key", Map.keys(request.props))+ |> Map.put(:"meta.value", Map.values(request.props) |> Enum.map(&to_string/1))+ else+ event+ end+ end++ defp put_referrer(%{} = event, %Request{} = request) do+ uri = request.url && URI.parse(request.url)+ ref = parse_referrer(uri, request.referrer)++ event+ |> Map.put(:utm_medium, request.query_params["utm_medium"])+ |> Map.put(:utm_source, request.query_params["utm_source"])+ |> Map.put(:utm_campaign, request.query_params["utm_campaign"])+ |> Map.put(:utm_content, request.query_params["utm_content"])+ |> Map.put(:utm_term, request.query_params["utm_term"])+ |> Map.put(:referrer_source, get_referrer_source(request, ref))+ |> Map.put(:referrer, clean_referrer(ref))+ end++ defp parse_referrer(_uri, _referrer_str = nil), do: nil++ defp parse_referrer(uri, referrer_str) do+ referrer_uri = URI.parse(referrer_str)++ if strip_www(referrer_uri.host) !== strip_www(uri.host) && referrer_uri.host !== "localhost" do+ RefInspector.parse(referrer_str)+ end+ end++ defp get_referrer_source(request, ref) do+ source =+ request.query_params["utm_source"] ||+ request.query_params["source"] ||+ request.query_params["ref"]++ source || PlausibleWeb.RefInspector.parse(ref)+ end++ defp clean_referrer(nil), do: nil++ defp clean_referrer(ref) do+ uri = URI.parse(ref.referer)++ if PlausibleWeb.RefInspector.right_uri?(uri) do+ host = String.replace_prefix(uri.host, "www.", "")+ path = uri.path || ""+ host <> String.trim_trailing(path, "/")+ end+ end++ defp put_user_agent(%{} = event, %Request{} = request) do+ case parse_user_agent(request) do+ %UAInspector.Result{client: %UAInspector.Result.Client{name: "Headless Chrome"}} ->+ :skip++ %UAInspector.Result.Bot{} ->+ :skip++ %UAInspector.Result{} = user_agent ->+ event+ |> Map.put(:operating_system, os_name(user_agent))+ |> Map.put(:operating_system_version, os_version(user_agent))+ |> Map.put(:browser, browser_name(user_agent))+ |> Map.put(:browser_version, browser_version(user_agent))++ _any ->+ event+ end+ end++ defp parse_user_agent(%Request{user_agent: user_agent}) when is_binary(user_agent) do+ Tracer.with_span "parse_user_agent" do+ case Cachex.fetch(:user_agents, user_agent, &UAInspector.parse/1) do+ {:ok, user_agent} -> user_agent+ {:commit, user_agent} -> user_agent+ _ -> nil+ end+ end+ end++ defp parse_user_agent(request), do: request++ defp browser_name(ua) do+ case ua.client do+ :unknown -> ""+ %UAInspector.Result.Client{name: "Mobile Safari"} -> "Safari"+ %UAInspector.Result.Client{name: "Chrome Mobile"} -> "Chrome"+ %UAInspector.Result.Client{name: "Chrome Mobile iOS"} -> "Chrome"+ %UAInspector.Result.Client{name: "Firefox Mobile"} -> "Firefox"+ %UAInspector.Result.Client{name: "Firefox Mobile iOS"} -> "Firefox"+ %UAInspector.Result.Client{name: "Opera Mobile"} -> "Opera"+ %UAInspector.Result.Client{name: "Opera Mini"} -> "Opera"+ %UAInspector.Result.Client{name: "Opera Mini iOS"} -> "Opera"+ %UAInspector.Result.Client{name: "Yandex Browser Lite"} -> "Yandex Browser"+ %UAInspector.Result.Client{name: "Chrome Webview"} -> "Mobile App"+ %UAInspector.Result.Client{type: "mobile app"} -> "Mobile App"+ client -> client.name+ end+ end++ defp browser_version(ua) do+ case ua.client do+ :unknown -> ""+ %UAInspector.Result.Client{type: "mobile app"} -> ""+ client -> major_minor(client.version)+ end+ end++ defp os_name(ua) do+ case ua.os do+ :unknown -> ""+ os -> os.name+ end+ end++ defp os_version(ua) do+ case ua.os do+ :unknown -> ""+ os -> major_minor(os.version)+ end+ end++ defp major_minor(version) do+ case version do+ :unknown ->+ ""++ version ->+ version+ |> String.split(".")+ |> Enum.take(2)+ |> Enum.join(".")+ end+ end++ defp put_screen_size(%{} = event, %Request{} = request) do+ screen_width =+ case request.screen_width do+ nil -> nil+ width when width < 576 -> "Mobile"+ width when width < 992 -> "Tablet"+ width when width < 1440 -> "Laptop"+ width when width >= 1440 -> "Desktop"+ end++ Map.put(event, :screen_size, screen_width)+ end++ defp put_geolocation(%{} = event, %Request{} = request) do+ Tracer.with_span "parse_visitor_location" do+ result = Geolix.lookup(request.remote_ip, where: :geolocation)++ country_code =+ get_in(result, [:country, :iso_code])+ |> ignore_unknown_country()++ city_geoname_id = get_in(result, [:city, :geoname_id])+ city_geoname_id = Map.get(CityOverrides.get(), city_geoname_id, city_geoname_id)++ subdivision1_code =+ case result do+ %{subdivisions: [%{iso_code: iso_code} | _rest]} ->+ country_code <> "-" <> iso_code++ _ ->+ ""+ end++ subdivision2_code =+ case result do+ %{subdivisions: [_first, %{iso_code: iso_code} | _rest]} ->+ country_code <> "-" <> iso_code++ _ ->+ ""+ end++ event+ |> Map.put(:country_code, country_code)+ |> Map.put(:subdivision1_code, subdivision1_code)+ |> Map.put(:subdivision2_code, subdivision2_code)+ |> Map.put(:city_geoname_id, city_geoname_id)+ end+ end++ defp ignore_unknown_country("ZZ"), do: nil+ defp ignore_unknown_country(country), do: country++ defp map_domains(%{} = event, %Request{} = request) do+ domains =+ if request.domain do+ String.split(request.domain, ",")+ |> Enum.map(&String.trim/1)+ |> Enum.map(&strip_www/1)+ else+ uri = request.url && URI.parse(request.url)+ [strip_www(uri && uri.host)]+ end++ for domain <- domains, do: Map.put(event, :domain, domain)+ end++ defp put_user_id(events, %Request{} = request, salts) do+ for %{} = event <- events do+ user_id = generate_user_id(request, event.domain, event.hostname, salts.current)+ Map.put(event, :user_id, user_id)+ end+ end++ defp register_session(events, %Request{} = request, salts) do+ for %Plausible.ClickhouseEvent{} = event <- events do+ previous_user_id = generate_user_id(request, event.domain, event.hostname, salts.previous)++ session_id =+ Tracer.with_span "cache_store_event" do+ Plausible.Session.CacheStore.on_event(event, previous_user_id)+ end++ Map.put(event, :session_id, session_id)+ end+ end++ defp generate_user_id(request, domain, hostname, salt) do+ cond do+ is_nil(salt) ->+ nil++ is_nil(domain) ->+ nil++ true ->+ user_agent = request.user_agent || ""+ root_domain = get_root_domain(hostname)++ SipHash.hash!(salt, user_agent <> request.remote_ip <> domain <> root_domain)+ end+ end++ defp get_root_domain(nil), do: "(none)"++ defp get_root_domain(hostname) do+ case PublicSuffix.registrable_domain(hostname) do+ domain when is_binary(domain) -> domain+ _any -> hostname+ end+ end++ defp spam_or_blocked?(%Request{} = request) do+ cond do+ request.domain in Application.get_env(:plausible, :domain_blacklist) ->+ :skip++ FunWithFlags.enabled?(:block_event_ingest, for: request.domain) ->+ Tracer.set_attribute("blocked_by_flag", true)+ :skip++ request.referrer &&+ URI.parse(request.referrer).host |> strip_www() |> ReferrerBlocklist.is_spammer?() ->+ :skip++ true ->+ :ok+ end+ end++ defp validate_events(events) do+ Enum.reduce_while(events, {:ok, []}, fn %{} = attrs, {:ok, acc} ->+ attrs+ |> Plausible.ClickhouseEvent.new()+ |> Ecto.Changeset.apply_action(nil)+ |> case do+ {:ok, event} -> {:cont, {:ok, [event | acc]}}+ {:error, changeset} -> {:halt, {:error, changeset}}+ end+ end)+ end++ defp strip_www(hostname) do+ if hostname do+ String.replace_prefix(hostname, "www.", "")+ else+ nil+ end+ end+enddiff --git a/lib/plausible/ingestion/ingestion.ex b/lib/plausible/ingestion/ingestion.exdeleted file mode 100644index b97e7020de..0000000000--- a/lib/plausible/ingestion/ingestion.ex+++ /dev/null@@ -1,496 +0,0 @@-defmodule Plausible.Ingestion do- require OpenTelemetry.Tracer, as: Tracer-- @no_domain_error {:error, %{domain: ["can't be blank"]}}-- def add_to_buffer(%Plausible.Ingestion.Request{} = request) do- ua =- Tracer.with_span "parse_user_agent" do- parse_user_agent(request)- end-- blacklist_domain = request.params.domain in Application.get_env(:plausible, :domain_blacklist)-- if blacklist_domain || is_bot?(ua) || is_spammer?(request.params.referrer) ||- blocked_via_flag?(request.params.domain) do- :ok- else- uri = request.params.url && URI.parse(request.params.url)- host = if uri && uri.host == "", do: "(none)", else: uri && uri.host-- ref = parse_referrer(uri, request.params.referrer)-- location_details =- Tracer.with_span "parse_visitor_location" do- visitor_location_details(request)- end-- salts = Plausible.Session.Salts.fetch()-- event_attrs = %{- timestamp: NaiveDateTime.utc_now() |> NaiveDateTime.truncate(:second),- name: request.params.name,- hostname: strip_www(host),- pathname: get_pathname(uri, request.params.hash_mode),- referrer_source: get_referrer_source(request, ref),- referrer: clean_referrer(ref),- utm_medium: request.query_params["utm_medium"],- utm_source: request.query_params["utm_source"],- utm_campaign: request.query_params["utm_campaign"],- utm_content: request.query_params["utm_content"],- utm_term: request.query_params["utm_term"],- country_code: location_details[:country_code],- country_geoname_id: location_details[:country_geoname_id],- subdivision1_code: location_details[:subdivision1_code],- subdivision2_code: location_details[:subdivision2_code],- city_geoname_id: location_details[:city_geoname_id],- operating_system: ua && os_name(ua),- operating_system_version: ua && os_version(ua),- browser: ua && browser_name(ua),- browser_version: ua && browser_version(ua),- screen_size: calculate_screen_size(request.params.screen_width),- "meta.key": Map.keys(request.params.meta),- "meta.value": Map.values(request.params.meta) |> Enum.map(&Kernel.to_string/1)- }-- Enum.reduce_while(get_domains(request, uri), @no_domain_error, fn domain, _res ->- user_id = generate_user_id(request, domain, event_attrs[:hostname], salts[:current])-- previous_user_id =- salts[:previous] &&- generate_user_id(request, domain, event_attrs[:hostname], salts[:previous])-- changeset =- event_attrs- |> Map.merge(%{domain: domain, user_id: user_id})- |> Plausible.ClickhouseEvent.new()-- if changeset.valid? do- event = Ecto.Changeset.apply_changes(changeset)-- session_id =- Tracer.with_span "cache_store_event" do- Plausible.Session.CacheStore.on_event(event, previous_user_id)- end-- event- |> Map.put(:session_id, session_id)- |> Plausible.Event.WriteBuffer.insert()-- {:cont, :ok}- else- errors = Ecto.Changeset.traverse_errors(changeset, &encode_error/1)- {:halt, {:error, errors}}- end- end)- end- end-- defp blocked_via_flag?(domain) do- blocked? = FunWithFlags.enabled?(:block_event_ingest, for: domain)- Tracer.set_attribute("blocked_by_flag", blocked?)- blocked?- end-- # https://hexdocs.pm/ecto/Ecto.Changeset.html#traverse_errors/2-examples- defp encode_error({msg, opts}) do- Regex.replace(~r"%{(\w+)}", msg, fn _, key ->- opts |> Keyword.get(String.to_existing_atom(key), key) |> to_string()- end)- end-- defp is_bot?(%UAInspector.Result.Bot{}), do: true-- defp is_bot?(%UAInspector.Result{client: %UAInspector.Result.Client{name: "Headless Chrome"}}),- do: true-- defp is_bot?(_), do: false-- defp is_spammer?(nil), do: false-- defp is_spammer?(referrer_str) do- uri = URI.parse(referrer_str)- ReferrerBlocklist.is_spammer?(strip_www(uri.host))- end-- defp get_domains(request, uri) do- if request.params.domain do- String.split(request.params.domain, ",")- |> Enum.map(&String.trim/1)- |> Enum.map(&strip_www/1)- else- List.wrap(strip_www(uri && uri.host))- end- end-- defp get_pathname(nil, _), do: "/"-- defp get_pathname(uri, hash_mode) do- pathname =- (uri.path || "/")- |> URI.decode()- |> String.trim_trailing()-- if hash_mode == 1 && uri.fragment do- pathname <> "#" <> URI.decode(uri.fragment)- else- pathname- end- end-- @city_overrides %{- # Austria- # Gemeindebezirk Floridsdorf -> Vienna- 2_779_467 => 2_761_369,- # Gemeindebezirk Leopoldstadt -> Vienna- 2_772_614 => 2_761_369,- # Gemeindebezirk Landstrasse -> Vienna- 2_773_040 => 2_761_369,- # Gemeindebezirk Donaustadt -> Vienna- 2_780_851 => 2_761_369,- # Gemeindebezirk Favoriten -> Vienna- 2_779_776 => 2_761_369,- # Gemeindebezirk Währing -> Vienna- 2_762_091 => 2_761_369,- # Gemeindebezirk Wieden -> Vienna- 2_761_393 => 2_761_369,- # Gemeindebezirk Innere Stadt -> Vienna- 2_775_259 => 2_761_369,- # Gemeindebezirk Alsergrund -> Vienna- 2_782_729 => 2_761_369,- # Gemeindebezirk Liesing -> Vienna- 2_772_484 => 2_761_369,- # Urfahr -> Linz- 2_762_518 => 2_772_400,-- # Canada- # Old Toronto -> Toronto- 8_436_019 => 6_167_865,- # Etobicoke -> Toronto- 5_950_267 => 6_167_865,- # East York -> Toronto- 5_946_235 => 6_167_865,- # Scarborough -> Toronto- 6_948_711 => 6_167_865,- # North York -> Toronto- 6_091_104 => 6_167_865,-- # Czech republic- # Praha 5 -> Prague- 11_951_220 => 3_067_696,- # Praha 4 -> Prague- 11_951_218 => 3_067_696,- # Praha 11 -> Prague- 11_951_232 => 3_067_696,- # Praha 10 -> Prague- 11_951_210 => 3_067_696,- # Praha 4 -> Prague- 8_378_772 => 3_067_696,-- # Denmark- # København SV -> Copenhagen- 11_747_123 => 2_618_425,- # København NV -> Copenhagen- 11_746_894 => 2_618_425,- # Odense S -> Odense- 11_746_825 => 2_615_876,- # Odense M -> Odense- 11_746_974 => 2_615_876,- # Odense SØ -> Odense- 11_746_888 => 2_615_876,- # Aarhus C -> Aarhus- 11_746_746 => 2_624_652,- # Aarhus N -> Aarhus- 11_746_890 => 2_624_652,-- # Estonia- # Kristiine linnaosa -> Tallinn- 11_050_530 => 588_409,- # Kesklinna linnaosa -> Tallinn- 11_053_706 => 588_409,- # Lasnamäe linnaosa -> Tallinn- 11_050_526 => 588_409,- # Põhja-Tallinna linnaosa -> Tallinn- 11_049_594 => 588_409,- # Mustamäe linnaosa -> Tallinn- 11_050_531 => 588_409,- # Haabersti linnaosa -> Tallinn- 11_053_707 => 588_409,- # Viimsi -> Tallinn- 587_629 => 588_409,-- # Germany- # Bezirk Tempelhof-Schöneberg -> Berlin- 3_336_297 => 2_950_159,- # Bezirk Mitte -> Berlin- 2_870_912 => 2_950_159,- # Bezirk Charlottenburg-Wilmersdorf -> Berlin- 3_336_294 => 2_950_159,- # Bezirk Friedrichshain-Kreuzberg -> Berlin- 3_336_295 => 2_950_159,- # Moosach -> Munich- 8_351_447 => 2_867_714,- # Schwabing-Freimann -> Munich- 8_351_448 => 2_867_714,- # Stadtbezirk 06 -> Düsseldorf- 6_947_276 => 2_934_246,- # Stadtbezirk 04 -> Düsseldorf- 6_947_274 => 2_934_246,- # Köln-Ehrenfeld -> Köln- 6_947_479 => 2_886_242,- # Köln-Lindenthal- -> Köln- 6_947_481 => 2_886_242,- # Beuel -> Bonn- 2_949_619 => 2_946_447,- # Innenstadt I -> Frankfurt am Main- 6_946_225 => 2_925_533,-- # India- # Navi Mumbai -> Mumbai- 6_619_347 => 1_275_339,-- # Mexico- # Miguel Hidalgo Villa Olímpica -> Mexico city- 11_561_026 => 3_530_597,- # Zedec Santa Fe -> Mexico city- 3_517_471 => 3_530_597,- # Fuentes del Pedregal-> Mexico city- 11_562_596 => 3_530_597,- # Centro -> Mexico city- 9_179_691 => 3_530_597,- # Cuauhtémoc-> Mexico city- 12_266_959 => 3_530_597,-- # Netherlands- # Schiphol-Rijk -> Amsterdam- 10_173_838 => 2_759_794,- # Westpoort -> Amsterdam- 11_525_047 => 2_759_794,- # Amsterdam-Zuidoost -> Amsterdam- 6_544_881 => 2_759_794,- # Loosduinen -> The Hague- 11_525_037 => 2_747_373,- # Laak -> The Hague- 11_525_042 => 2_747_373,-- # Norway- # Nordre Aker District -> Oslo- 6_940_981 => 3_143_244,-- # Romania- # Sector 1 -> Bucharest,- 11_055_041 => 683_506,- # Sector 2 -> Bucharest- 11_055_040 => 683_506,- # Sector 3 -> Bucharest- 11_055_044 => 683_506,- # Sector 4 -> Bucharest- 11_055_042 => 683_506,- # Sector 5 -> Bucharest- 11_055_043 => 683_506,- # Sector 6 -> Bucharest- 11_055_039 => 683_506,- # Bucuresti -> Bucharest- 6_691_781 => 683_506,-- # Slovakia- # Bratislava -> Bratislava- 3_343_955 => 3_060_972,-- # Sweden- # Södermalm -> Stockholm- 2_676_209 => 2_673_730,-- # Switzerland- # Vorstädte -> Basel- 11_789_440 => 2_661_604,- # Zürich (Kreis 11) / Oerlikon -> Zürich- 2_659_310 => 2_657_896,- # Zürich (Kreis 3) / Alt-Wiedikon -> Zürich- 2_658_007 => 2_657_896,- # Zürich (Kreis 5) -> Zürich- 6_295_521 => 2_657_896,- # Zürich (Kreis 1) / Hochschulen -> Zürich- 6_295_489 => 2_657_896,-- # UK- # Shadwell -> London- 6_690_595 => 2_643_743,- # City of London -> London- 2_643_741 => 2_643_743,- # South Bank -> London- 6_545_251 => 2_643_743,- # Soho -> London- 6_545_173 => 2_643_743,- # Whitechapel -> London- 2_634_112 => 2_643_743,- # King's Cross -> London- 6_690_589 => 2_643_743,- # Poplar -> London- 2_640_091 => 2_643_743,- # Hackney -> London- 2_647_694 => 2_643_743- }-- defp visitor_location_details(request) do- result = Geolix.lookup(request.remote_ip, where: :geolocation)-- country_code =- get_in(result, [:country, :iso_code])- |> ignore_unknown_country()-- city_geoname_id = get_in(result, [:city, :geoname_id])-- subdivision1_code =- case result do- %{subdivisions: [%{iso_code: iso_code} | _rest]} ->- country_code <> "-" <> iso_code-- _ ->- ""- end-- subdivision2_code =- case result do- %{subdivisions: [_first, %{iso_code: iso_code} | _rest]} ->- country_code <> "-" <> iso_code-- _ ->- ""- end-- %{- country_code: country_code,- subdivision1_code: subdivision1_code,- subdivision2_code: subdivision2_code,- city_geoname_id: Map.get(@city_overrides, city_geoname_id, city_geoname_id)- }- end-- defp ignore_unknown_country("ZZ"), do: nil- defp ignore_unknown_country(country), do: country-- defp parse_referrer(_, nil), do: nil-- defp parse_referrer(uri, referrer_str) do- referrer_uri = URI.parse(referrer_str)-- if strip_www(referrer_uri.host) !== strip_www(uri.host) && referrer_uri.host !== "localhost" do- RefInspector.parse(referrer_str)- end- end-- defp generate_user_id(request, domain, hostname, salt) do- user_agent = request.headers["user-agent"] || ""- root_domain = get_root_domain(hostname)-- if domain && root_domain do- SipHash.hash!(salt, user_agent <> request.remote_ip <> domain <> root_domain)- end- end-- defp get_root_domain(nil), do: "(none)"-- defp get_root_domain(hostname) do- case PublicSuffix.registrable_domain(hostname) do- domain when is_binary(domain) -> domain- _ -> hostname- end- end-- defp calculate_screen_size(nil), do: nil- defp calculate_screen_size(width) when width < 576, do: "Mobile"- defp calculate_screen_size(width) when width < 992, do: "Tablet"- defp calculate_screen_size(width) when width < 1440, do: "Laptop"- defp calculate_screen_size(width) when width >= 1440, do: "Desktop"-- defp clean_referrer(nil), do: nil-- defp clean_referrer(ref) do- uri = URI.parse(ref.referer)-- if PlausibleWeb.RefInspector.right_uri?(uri) do- host = String.replace_prefix(uri.host, "www.", "")- path = uri.path || ""- host <> String.trim_trailing(path, "/")- end- end-- defp strip_www(nil), do: nil-- defp strip_www(hostname) do- String.replace_prefix(hostname, "www.", "")- end-- defp browser_name(ua) do- case ua.client do- :unknown -> ""- %UAInspector.Result.Client{name: "Mobile Safari"} -> "Safari"- %UAInspector.Result.Client{name: "Chrome Mobile"} -> "Chrome"- %UAInspector.Result.Client{name: "Chrome Mobile iOS"} -> "Chrome"- %UAInspector.Result.Client{name: "Firefox Mobile"} -> "Firefox"- %UAInspector.Result.Client{name: "Firefox Mobile iOS"} -> "Firefox"- %UAInspector.Result.Client{name: "Opera Mobile"} -> "Opera"- %UAInspector.Result.Client{name: "Opera Mini"} -> "Opera"- %UAInspector.Result.Client{name: "Opera Mini iOS"} -> "Opera"- %UAInspector.Result.Client{name: "Yandex Browser Lite"} -> "Yandex Browser"- %UAInspector.Result.Client{name: "Chrome Webview"} -> "Mobile App"- %UAInspector.Result.Client{type: "mobile app"} -> "Mobile App"- client -> client.name- end- end-- defp major_minor(:unknown), do: ""-- defp major_minor(version) do- version- |> String.split(".")- |> Enum.take(2)- |> Enum.join(".")- end-- defp browser_version(ua) do- case ua.client do- :unknown -> ""- %UAInspector.Result.Client{type: "mobile app"} -> ""- client -> major_minor(client.version)- end- end-- defp os_name(ua) do- case ua.os do- :unknown -> ""- os -> os.name- end- end-- defp os_version(ua) do- case ua.os do- :unknown -> ""- os -> major_minor(os.version)- end- end-- defp get_referrer_source(request, ref) do- source =- request.query_params["utm_source"] || request.query_params["source"] ||- request.query_params["ref"]-- source || PlausibleWeb.RefInspector.parse(ref)- end-- defp parse_user_agent(%Plausible.Ingestion.Request{} = request) do- if user_agent = request.headers["user-agent"] do- res =- Cachex.fetch(:user_agents, user_agent, fn ua ->- UAInspector.parse(ua)- end)-- case res do- {:ok, user_agent} -> user_agent- {:commit, user_agent} -> user_agent- _ -> nil- end- end- end-enddiff --git a/lib/plausible/ingestion/request.ex b/lib/plausible/ingestion/request.exindex 164aeba43b..02e69bcccc 100644--- a/lib/plausible/ingestion/request.ex+++ b/lib/plausible/ingestion/request.ex@@ -1,33 +1,46 @@defmodule Plausible.Ingestion.Request do- defstruct ~w(remote_ip params query_params headers)a+ @moduledoc """+ The %Plausible.Ingestion.Request{} struct stores all needed fields to create an event downstream.+ """++ defstruct [+ :remote_ip,+ :user_agent,+ :event_name,+ :url,+ :referrer,+ :domain,+ :screen_width,+ :hash_mode,+ props: %{},+ query_params: %{}+ ]@type t() :: %__MODULE__{remote_ip: String.t() | nil,- params: map(),- query_params: map(),- headers: map()+ user_agent: String.t() | nil,+ event_name: term(),+ url: term(),+ referrer: term(),+ domain: term(),+ screen_width: term(),+ hash_mode: term(),+ props: map(),+ query_params: map()}- @allowed_query_params ~w(utm_medium utm_source utm_campaign utm_content utm_term utm_source source ref)- @allowed_headers ~w(user-agent)-@spec build(Plug.Conn.t()) :: {:ok, t()} | {:error, :invalid_json}@doc """Builds a %Plausible.Ingestion.Request{} struct from %Plug.Conn{}."""def build(%Plug.Conn{} = conn) do- with {:ok, body} <- parse_body(conn),- %{} = params <- build_params(body),- %{} = query_params <- decode_query_params(params),- %{} = headers <- build_headers(conn),- remote_ip <- PlausibleWeb.RemoteIp.get(conn) do- {:ok,- %__MODULE__{- remote_ip: remote_ip,- params: params,- query_params: query_params,- headers: headers- }}+ with {:ok, request_body} <- parse_body(conn) do+ %__MODULE__{}+ |> Map.put(:remote_ip, PlausibleWeb.RemoteIp.get(conn))+ |> put_user_agent(conn)+ |> put_request_params(request_body)+ |> put_query_params()+ |> then(&{:ok, &1})endend@@ -46,33 +59,30 @@ defmodule Plausible.Ingestion.Request doendend- defp build_params(body) do- %{- name: body["n"] || body["name"],- url: body["u"] || body["url"],- referrer: body["r"] || body["referrer"],- domain: body["d"] || body["domain"],- screen_width: body["w"] || body["screen_width"],- hash_mode: body["h"] || body["hashMode"],- meta: parse_meta(body)+ defp put_request_params(%__MODULE__{} = request, %{} = request_body) do+ %__MODULE__{+ request+ | event_name: request_body["n"] || request_body["name"],+ url: request_body["u"] || request_body["url"],+ referrer: request_body["r"] || request_body["referrer"],+ domain: request_body["d"] || request_body["domain"],+ screen_width: request_body["w"] || request_body["screen_width"],+ hash_mode: request_body["h"] || request_body["hashMode"],+ props: parse_props(request_body)}end- defp parse_meta(params) do- raw_meta = params["m"] || params["meta"] || params["p"] || params["props"]+ defp parse_props(%{} = request_body) do+ raw_props =+ request_body["m"] || request_body["meta"] || request_body["p"] || request_body["props"]- case decode_raw_props(raw_meta) do+ case decode_raw_props(raw_props) do{:ok, parsed_json} ->- Enum.filter(parsed_json, fn- {_, ""} -> false- {_, nil} -> false- {_, val} when is_list(val) -> false- {_, val} when is_map(val) -> false- _ -> true- end)+ parsed_json+ |> Enum.filter(&valid_prop_value?/1)|> Map.new()- _ ->+ _error ->%{}endend@@ -91,32 +101,32 @@ defmodule Plausible.Ingestion.Request dodefp decode_raw_props(_), do: :bad_format- defp decode_query_params(params) do- with url when is_binary(url) <- params.url,- %URI{query: query} when is_binary(query) <- URI.parse(url) do- do_decode_query_params(query)- else- _any -> %{}+ defp valid_prop_value?({key, value}) do+ case {key, value} do+ {_key, ""} -> false+ {_key, nil} -> false+ {_key, value} when is_list(value) -> false+ {_key, value} when is_map(value) -> false+ {_key, _value} -> trueendend- defp do_decode_query_params(query) do- try do- query- |> URI.query_decoder()- |> Enum.reduce(%{}, fn- {key, value}, acc when key in @allowed_query_params -> Map.put(acc, key, value)- _any, acc -> acc- end)- rescue- _ -> %{}+ defp put_query_params(%__MODULE__{url: url} = request) do+ with url when is_binary(url) <- url,+ %URI{query: query} when is_binary(query) <- URI.parse(url),+ %{} = query_params <- URI.decode_query(query) do+ Map.put(request, :query_params, query_params)+ else+ _any -> requestendend- defp build_headers(conn) do- Enum.reduce(@allowed_headers, %{}, fn header, acc ->- value = conn |> Plug.Conn.get_req_header(header) |> List.first()- if value, do: Map.put(acc, header, value), else: acc- end)+ defp put_user_agent(%__MODULE__{} = request, %Plug.Conn{} = conn) do+ user_agent =+ conn+ |> Plug.Conn.get_req_header("user-agent")+ |> List.first()++ %__MODULE__{request | user_agent: user_agent}endenddiff --git a/lib/plausible_web/controllers/api/external_controller.ex b/lib/plausible_web/controllers/api/external_controller.exindex 70cd875ca8..6a47218610 100644--- a/lib/plausible_web/controllers/api/external_controller.ex+++ b/lib/plausible_web/controllers/api/external_controller.ex@@ -10,16 +10,26 @@ defmodule PlausibleWeb.Api.ExternalController dodef event(conn, _params) dowith {:ok, ingestion_request} <- Plausible.Ingestion.Request.build(conn),- _ <- Sentry.Context.set_extra_context(%{request: ingestion_request.params}),- :ok <- Plausible.Ingestion.add_to_buffer(ingestion_request) do+ _ <- Sentry.Context.set_extra_context(%{request: ingestion_request}),+ :ok <- Plausible.Ingestion.Event.build_and_buffer(ingestion_request) doconn |> put_status(202) |> text("ok")else+ :skip ->+ conn |> put_status(202) |> text("ok")+{:error, :invalid_json} ->conn|> put_status(400)|> json(%{errors: %{request: "Unable to parse request body as json"}})- {:error, errors} ->+ {:error, %Ecto.Changeset{} = changeset} ->+ errors =+ Ecto.Changeset.traverse_errors(changeset, fn {msg, opts} ->+ Regex.replace(~r"%{(\w+)}", msg, fn _, key ->+ opts |> Keyword.get(String.to_existing_atom(key), key) |> to_string()+ end)+ end)+conn |> put_status(400) |> json(%{errors: errors})endenddiff --git a/test/plausible/ingestion/event_test.exs b/test/plausible/ingestion/event_test.exsnew file mode 100644index 0000000000..e306bba2a1--- /dev/null+++ b/test/plausible/ingestion/event_test.exs@@ -0,0 +1,74 @@+defmodule Plausible.Ingestion.EventTest do+ use Plausible.DataCase++ def get_event(domain) do+ Plausible.TestUtils.eventually(fn ->+ Plausible.Event.WriteBuffer.flush()++ event =+ Plausible.ClickhouseRepo.one(+ from e in Plausible.ClickhouseEvent, where: e.domain == ^domain+ )++ {!is_nil(event), event}+ end)+ end++ @valid_request %Plausible.Ingestion.Request{+ remote_ip: "2.2.2.2",+ user_agent:+ "Mozilla/5.0 (iPad; U; CPU OS 3_2_1 like Mac OS X; en-us) AppleWebKit/531.21.10 (KHTML, like Gecko) Mobile/7B405",+ event_name: "pageview",+ url: "http://skywalker.test",+ referrer: "http://m.facebook.test/",+ screen_width: 1440,+ hash_mode: nil,+ query_params: %{+ "utm_medium" => "utm_medium",+ "utm_source" => "utm_source",+ "utm_campaign" => "utm_campaign",+ "utm_content" => "utm_content",+ "utm_term" => "utm_term",+ "source" => "source",+ "ref" => "ref"+ }+ }++ test "build_and_buffer/3 creates an event" do+ assert :ok ==+ @valid_request+ |> Map.put(:domain, "plausible-ingestion-event-basic.test")+ |> Plausible.Ingestion.Event.build_and_buffer()++ assert %Plausible.ClickhouseEvent{+ session_id: session_id,+ user_id: user_id,+ domain: "plausible-ingestion-event-basic.test",+ browser: "Safari",+ browser_version: "",+ city_geoname_id: 2_988_507,+ country_code: "FR",+ hostname: "skywalker.test",+ "meta.key": [],+ "meta.value": [],+ name: "pageview",+ operating_system: "iOS",+ operating_system_version: "3.2",+ pathname: "/",+ referrer: "m.facebook.test",+ referrer_source: "utm_source",+ screen_size: "Desktop",+ subdivision1_code: "FR-IDF",+ subdivision2_code: "FR-75",+ transferred_from: "",+ utm_campaign: "utm_campaign",+ utm_content: "utm_content",+ utm_medium: "utm_medium",+ utm_source: "utm_source",+ utm_term: "utm_term"+ } = get_event("plausible-ingestion-event-basic.test")++ assert is_integer(session_id)+ assert is_integer(user_id)+ end+enddiff --git a/test/plausible_web/controllers/api/external_controller_test.exs b/test/plausible_web/controllers/api/external_controller_test.exsindex 8a1b777f0f..0f9487757f 100644--- a/test/plausible_web/controllers/api/external_controller_test.exs+++ b/test/plausible_web/controllers/api/external_controller_test.exs@@ -927,7 +927,9 @@ defmodule PlausibleWeb.Api.ExternalControllerTest doassert json_response(conn, 400) == %{"errors" => %{- "domain" => ["can't be blank"]+ "domain" => ["can't be blank"],+ "hostname" => ["can't be blank"],+ "user_id" => ["can't be blank"]}}enddiff --git a/test/support/test_utils.ex b/test/support/test_utils.exindex 350e1398b9..d2977f61df 100644--- a/test/support/test_utils.ex+++ b/test/support/test_utils.ex@@ -143,4 +143,17 @@ defmodule Plausible.TestUtils do|> Timex.shift(shifts)|> NaiveDateTime.truncate(:second)end++ def eventually(expectation, wait_time_ms \\ 50, retries \\ 10) do+ Enum.reduce_while(1..retries, nil, fn _i, _acc ->+ case expectation.() do+ {true, result} ->+ {:halt, result}++ {false, _} ->+ Process.sleep(wait_time_ms)+ {:cont, nil}+ end+ end)+ endend