Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.
TypeScript analyticsanalytics-dashboardclickhouseelixirgoogle-analyticsmarketingopen-source-analyticsphoenixplausible-analyticspostgresqlprivacyprivacy-friendlyself-hostedsimple-analyticsstatisticstailwindcssweb-analyticswebsite-analyticswebsite-statswebsite-tracking
Limit ingestion payload size (#2904)
Vini Brasil4ca2e26b9c4f
diff --git a/lib/plausible/goal/schema.ex b/lib/plausible/goal/schema.exindex c26954ac9c..7530807d5d 100644--- a/lib/plausible/goal/schema.ex+++ b/lib/plausible/goal/schema.ex@@ -37,6 +37,7 @@ defmodule Plausible.Goal do|> validate_event_name_and_page_path()|> update_change(:event_name, &String.trim/1)|> update_change(:page_path, &String.trim/1)+ |> validate_length(:event_name, max: 120)enddefp validate_event_name_and_page_path(changeset) dodiff --git a/lib/plausible/ingestion/request.ex b/lib/plausible/ingestion/request.exindex 6e03e2aea0..ccf322571a 100644--- a/lib/plausible/ingestion/request.ex+++ b/lib/plausible/ingestion/request.ex@@ -8,11 +8,13 @@ defmodule Plausible.Ingestion.Request douse Ecto.Schemaalias Ecto.Changeset+ @max_url_size 2_000+embedded_schema dofield :remote_ip, :stringfield :user_agent, :stringfield :event_name, :string- field :uri, :string+ field :uri, :mapfield :hostname, :stringfield :referrer, :stringfield :domains, {:array, :string}@@ -47,6 +49,8 @@ defmodule Plausible.Ingestion.Request do|> put_hostname()|> put_user_agent(conn)|> put_request_params(request_body)+ |> put_referrer(request_body)+ |> put_props(request_body)|> put_pathname()|> put_query_params()|> map_domains(request_body)@@ -56,7 +60,7 @@ defmodule Plausible.Ingestion.Request do:pathname,:timestamp])- |> Changeset.validate_length(:pathname, max: 2000)+ |> Changeset.validate_length(:event_name, max: 120)|> Changeset.apply_action(nil){:error, :invalid_json} ->@@ -87,12 +91,21 @@ defmodule Plausible.Ingestion.Request doChangeset.change(changeset,event_name: request_body["n"] || request_body["name"],- referrer: request_body["r"] || request_body["referrer"],- hash_mode: request_body["h"] || request_body["hashMode"],- props: parse_props(request_body)+ hash_mode: request_body["h"] || request_body["hashMode"])end+ defp put_referrer(changeset, %{} = request_body) do+ referrer = request_body["r"] || request_body["referrer"]++ if is_binary(referrer) do+ referrer = String.slice(referrer, 0..(@max_url_size - 1))+ Changeset.put_change(changeset, :referrer, referrer)+ else+ changeset+ end+ end+defp put_pathname(changeset) douri = Changeset.get_field(changeset, :uri)hash_mode = Changeset.get_field(changeset, :hash_mode)@@ -130,12 +143,13 @@ defmodule Plausible.Ingestion.Request do@disallowed_schemes ~w(data)defp put_uri(changeset, %{} = request_body) dowith url when is_binary(url) <- request_body["u"] || request_body["url"],+ url when byte_size(url) <= @max_url_size <- url,%URI{} = uri when uri.scheme not in @disallowed_schemes <- URI.parse(url) doChangeset.put_change(changeset, :uri, uri)elsenil -> Changeset.add_error(changeset, :url, "is required")%URI{} -> Changeset.add_error(changeset, :url, "scheme is not allowed")- _ -> Changeset.add_error(changeset, :url, "must be a string")+ _ -> Changeset.add_error(changeset, :url, "must be a valid url")endend@@ -149,42 +163,49 @@ defmodule Plausible.Ingestion.Request doChangeset.put_change(changeset, :hostname, sanitize_hostname(host))end- defp parse_props(%{} = request_body) do- raw_props =- request_body["m"] || request_body["meta"] || request_body["p"] || request_body["props"]+ defp put_props(changeset, %{} = request_body) do+ props =+ (request_body["m"] || request_body["meta"] || request_body["p"] || request_body["props"])+ |> decode_props_or_fallback()+ |> Enum.reject(fn {_k, v} -> is_nil(v) || is_list(v) || is_map(v) || v == "" end)+ |> Map.new()- case decode_raw_props(raw_props) do- {:ok, parsed_json} ->- parsed_json- |> Enum.filter(&valid_prop_value?/1)- |> Map.new()-- _error ->- %{}- end+ changeset+ |> Changeset.put_change(:props, props)+ |> validate_props()end- defp decode_raw_props(props) when is_map(props), do: {:ok, props}-- defp decode_raw_props(raw_json) when is_binary(raw_json) do- case Jason.decode(raw_json) do- {:ok, parsed_props} when is_map(parsed_props) ->- {:ok, parsed_props}-- _ ->- :not_a_map+ defp decode_props_or_fallback(raw) do+ with raw when is_binary(raw) <- raw,+ {:ok, %{} = decoded} <- Jason.decode(raw) do+ decoded+ else+ already_a_map when is_map(already_a_map) -> already_a_map+ {:ok, _list_or_other} -> %{}+ {:error, _decode_error} -> %{}+ _any -> %{}endend- defp decode_raw_props(_), do: :bad_format-- defp valid_prop_value?({key, value}) do- case {key, value} do- {_key, ""} -> false- {_key, nil} -> false- {_key, value} when is_list(value) -> false- {_key, value} when is_map(value) -> false- {_key, _value} -> true+ @max_prop_key_length 300+ @max_prop_value_length 2000+ defp validate_props(changeset) do+ case Changeset.get_field(changeset, :props) do+ props ->+ Enum.reduce_while(props, changeset, fn+ {key, value}, changeset+ when byte_size(key) > @max_prop_key_length or+ byte_size(value) > @max_prop_value_length ->+ {:halt,+ Changeset.add_error(+ changeset,+ :props,+ "keys should have at most #{@max_prop_key_length} bytes and values #{@max_prop_value_length} bytes"+ )}++ _, changeset ->+ {:cont, changeset}+ end)endenddiff --git a/test/plausible/goals_test.exs b/test/plausible/goals_test.exsindex 93838a257a..4e3f20dcda 100644--- a/test/plausible/goals_test.exs+++ b/test/plausible/goals_test.exs@@ -12,6 +12,12 @@ defmodule Plausible.GoalsTest doassert goal.event_name == "some event name"end+ test "create/2 validates goal name is at most 120 chars" do+ site = insert(:site)+ assert {:error, changeset} = Goals.create(site, %{"event_name" => String.duplicate("a", 130)})+ assert {"should be at most %{count} character(s)", _} = changeset.errors[:event_name]+ end+test "for_site2 returns trimmed input even if it was saved with trailing whitespace" dosite = insert(:site)insert(:goal, %{site: site, event_name: " Signup "})diff --git a/test/plausible/ingestion/request_test.exs b/test/plausible/ingestion/request_test.exsindex 14051aa7b6..7de51d1eeb 100644--- a/test/plausible/ingestion/request_test.exs+++ b/test/plausible/ingestion/request_test.exs@@ -213,18 +213,84 @@ defmodule Plausible.Ingestion.RequestTest doassert {"scheme is not allowed", _} = changeset.errors[:url]end- test "returns validation error when pathname is too long" do- long_string = for _ <- 1..5000, do: "a", into: ""+ test "returns validation error when url is too long" do+ payload = %{+ name: "pageview",+ domain: "dummy.site",+ url: "https://dummy.site/#{String.duplicate("a", 5000)}"+ }++ conn = build_conn(:post, "/api/events", payload)+ assert {:error, changeset} = Request.build(conn)+ assert {"must be a valid url", _} = changeset.errors[:url]+ end+ test "returns validation error when event name is too long" do+ payload = %{+ name: String.duplicate("a", 500),+ domain: "dummy.site",+ url: "https://dummy.site/"+ }++ conn = build_conn(:post, "/api/events", payload)+ assert {:error, changeset} = Request.build(conn)+ assert {"should be at most %{count} character(s)", _} = changeset.errors[:event_name]+ end++ test "truncates referrer when too long" dopayload = %{name: "pageview",domain: "dummy.site",- url: "https://dummy.site/#{long_string}"+ url: "https://dummy.site/",+ referrer: String.duplicate("a", 2500)+ }++ conn = build_conn(:post, "/api/events", payload)+ assert {:ok, request} = Request.build(conn)+ assert request.referrer == String.duplicate("a", 2000)+ end++ test "returns validation error when props keys are too long" do+ payload = %{+ name: "pageview",+ domain: "dummy.site",+ url: "https://dummy.site/",+ props: %{String.duplicate("a", 500) => "abc"}+ }++ conn = build_conn(:post, "/api/events", payload)+ assert {:error, changeset} = Request.build(conn)++ assert {"keys should have at most 300 bytes and values 2000 bytes", _} =+ changeset.errors[:props]+ end++ test "returns validation error when props values are too long" do+ payload = %{+ name: "pageview",+ domain: "dummy.site",+ url: "https://dummy.site/",+ props: %{"abc" => String.duplicate("a", 2500)}}conn = build_conn(:post, "/api/events", payload)assert {:error, changeset} = Request.build(conn)- assert {"should be at most %{count} character(s)", _} = changeset.errors[:pathname]++ assert {"keys should have at most 300 bytes and values 2000 bytes", _} =+ changeset.errors[:props]+ end++ test "does not fail when sending many props" do+ payload = %{+ name: "pageview",+ domain: "dummy.site",+ url: "https://dummy.site/",+ props: for(i <- 1..100, do: {"key_#{i}", "value"}, into: %{})+ }++ conn = build_conn(:post, "/api/events", payload)+ assert {:ok, request} = Request.build(conn)+ assert map_size(request.props) == 100endtest "malicious input, technically valid json" do