Skip to content

Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.

Limit ingestion payload size (#2904)

Vini Brasil4ca2e26b9c4f
diff --git a/lib/plausible/goal/schema.ex b/lib/plausible/goal/schema.ex
index c26954ac9c..7530807d5d 100644
--- a/lib/plausible/goal/schema.ex
+++ b/lib/plausible/goal/schema.ex
@@ -37,6 +37,7 @@ defmodule Plausible.Goal do
|> validate_event_name_and_page_path()
|> update_change(:event_name, &String.trim/1)
|> update_change(:page_path, &String.trim/1)
+ |> validate_length(:event_name, max: 120)
end
defp validate_event_name_and_page_path(changeset) do
diff --git a/lib/plausible/ingestion/request.ex b/lib/plausible/ingestion/request.ex
index 6e03e2aea0..ccf322571a 100644
--- a/lib/plausible/ingestion/request.ex
+++ b/lib/plausible/ingestion/request.ex
@@ -8,11 +8,13 @@ defmodule Plausible.Ingestion.Request do
use Ecto.Schema
alias Ecto.Changeset
+ @max_url_size 2_000
+
embedded_schema do
field :remote_ip, :string
field :user_agent, :string
field :event_name, :string
- field :uri, :string
+ field :uri, :map
field :hostname, :string
field :referrer, :string
field :domains, {:array, :string}
@@ -47,6 +49,8 @@ defmodule Plausible.Ingestion.Request do
|> put_hostname()
|> put_user_agent(conn)
|> put_request_params(request_body)
+ |> put_referrer(request_body)
+ |> put_props(request_body)
|> put_pathname()
|> put_query_params()
|> map_domains(request_body)
@@ -56,7 +60,7 @@ defmodule Plausible.Ingestion.Request do
:pathname,
:timestamp
])
- |> Changeset.validate_length(:pathname, max: 2000)
+ |> Changeset.validate_length(:event_name, max: 120)
|> Changeset.apply_action(nil)
{:error, :invalid_json} ->
@@ -87,12 +91,21 @@ defmodule Plausible.Ingestion.Request do
Changeset.change(
changeset,
event_name: request_body["n"] || request_body["name"],
- referrer: request_body["r"] || request_body["referrer"],
- hash_mode: request_body["h"] || request_body["hashMode"],
- props: parse_props(request_body)
+ hash_mode: request_body["h"] || request_body["hashMode"]
)
end
+ defp put_referrer(changeset, %{} = request_body) do
+ referrer = request_body["r"] || request_body["referrer"]
+
+ if is_binary(referrer) do
+ referrer = String.slice(referrer, 0..(@max_url_size - 1))
+ Changeset.put_change(changeset, :referrer, referrer)
+ else
+ changeset
+ end
+ end
+
defp put_pathname(changeset) do
uri = Changeset.get_field(changeset, :uri)
hash_mode = Changeset.get_field(changeset, :hash_mode)
@@ -130,12 +143,13 @@ defmodule Plausible.Ingestion.Request do
@disallowed_schemes ~w(data)
defp put_uri(changeset, %{} = request_body) do
with url when is_binary(url) <- request_body["u"] || request_body["url"],
+ url when byte_size(url) <= @max_url_size <- url,
%URI{} = uri when uri.scheme not in @disallowed_schemes <- URI.parse(url) do
Changeset.put_change(changeset, :uri, uri)
else
nil -> Changeset.add_error(changeset, :url, "is required")
%URI{} -> Changeset.add_error(changeset, :url, "scheme is not allowed")
- _ -> Changeset.add_error(changeset, :url, "must be a string")
+ _ -> Changeset.add_error(changeset, :url, "must be a valid url")
end
end
@@ -149,42 +163,49 @@ defmodule Plausible.Ingestion.Request do
Changeset.put_change(changeset, :hostname, sanitize_hostname(host))
end
- defp parse_props(%{} = request_body) do
- raw_props =
- request_body["m"] || request_body["meta"] || request_body["p"] || request_body["props"]
+ defp put_props(changeset, %{} = request_body) do
+ props =
+ (request_body["m"] || request_body["meta"] || request_body["p"] || request_body["props"])
+ |> decode_props_or_fallback()
+ |> Enum.reject(fn {_k, v} -> is_nil(v) || is_list(v) || is_map(v) || v == "" end)
+ |> Map.new()
- case decode_raw_props(raw_props) do
- {:ok, parsed_json} ->
- parsed_json
- |> Enum.filter(&valid_prop_value?/1)
- |> Map.new()
-
- _error ->
- %{}
- end
+ changeset
+ |> Changeset.put_change(:props, props)
+ |> validate_props()
end
- defp decode_raw_props(props) when is_map(props), do: {:ok, props}
-
- defp decode_raw_props(raw_json) when is_binary(raw_json) do
- case Jason.decode(raw_json) do
- {:ok, parsed_props} when is_map(parsed_props) ->
- {:ok, parsed_props}
-
- _ ->
- :not_a_map
+ defp decode_props_or_fallback(raw) do
+ with raw when is_binary(raw) <- raw,
+ {:ok, %{} = decoded} <- Jason.decode(raw) do
+ decoded
+ else
+ already_a_map when is_map(already_a_map) -> already_a_map
+ {:ok, _list_or_other} -> %{}
+ {:error, _decode_error} -> %{}
+ _any -> %{}
end
end
- defp decode_raw_props(_), do: :bad_format
-
- defp valid_prop_value?({key, value}) do
- case {key, value} do
- {_key, ""} -> false
- {_key, nil} -> false
- {_key, value} when is_list(value) -> false
- {_key, value} when is_map(value) -> false
- {_key, _value} -> true
+ @max_prop_key_length 300
+ @max_prop_value_length 2000
+ defp validate_props(changeset) do
+ case Changeset.get_field(changeset, :props) do
+ props ->
+ Enum.reduce_while(props, changeset, fn
+ {key, value}, changeset
+ when byte_size(key) > @max_prop_key_length or
+ byte_size(value) > @max_prop_value_length ->
+ {:halt,
+ Changeset.add_error(
+ changeset,
+ :props,
+ "keys should have at most #{@max_prop_key_length} bytes and values #{@max_prop_value_length} bytes"
+ )}
+
+ _, changeset ->
+ {:cont, changeset}
+ end)
end
end
diff --git a/test/plausible/goals_test.exs b/test/plausible/goals_test.exs
index 93838a257a..4e3f20dcda 100644
--- a/test/plausible/goals_test.exs
+++ b/test/plausible/goals_test.exs
@@ -12,6 +12,12 @@ defmodule Plausible.GoalsTest do
assert goal.event_name == "some event name"
end
+ test "create/2 validates goal name is at most 120 chars" do
+ site = insert(:site)
+ assert {:error, changeset} = Goals.create(site, %{"event_name" => String.duplicate("a", 130)})
+ assert {"should be at most %{count} character(s)", _} = changeset.errors[:event_name]
+ end
+
test "for_site2 returns trimmed input even if it was saved with trailing whitespace" do
site = insert(:site)
insert(:goal, %{site: site, event_name: " Signup "})
diff --git a/test/plausible/ingestion/request_test.exs b/test/plausible/ingestion/request_test.exs
index 14051aa7b6..7de51d1eeb 100644
--- a/test/plausible/ingestion/request_test.exs
+++ b/test/plausible/ingestion/request_test.exs
@@ -213,18 +213,84 @@ defmodule Plausible.Ingestion.RequestTest do
assert {"scheme is not allowed", _} = changeset.errors[:url]
end
- test "returns validation error when pathname is too long" do
- long_string = for _ <- 1..5000, do: "a", into: ""
+ test "returns validation error when url is too long" do
+ payload = %{
+ name: "pageview",
+ domain: "dummy.site",
+ url: "https://dummy.site/#{String.duplicate("a", 5000)}"
+ }
+
+ conn = build_conn(:post, "/api/events", payload)
+ assert {:error, changeset} = Request.build(conn)
+ assert {"must be a valid url", _} = changeset.errors[:url]
+ end
+ test "returns validation error when event name is too long" do
+ payload = %{
+ name: String.duplicate("a", 500),
+ domain: "dummy.site",
+ url: "https://dummy.site/"
+ }
+
+ conn = build_conn(:post, "/api/events", payload)
+ assert {:error, changeset} = Request.build(conn)
+ assert {"should be at most %{count} character(s)", _} = changeset.errors[:event_name]
+ end
+
+ test "truncates referrer when too long" do
payload = %{
name: "pageview",
domain: "dummy.site",
- url: "https://dummy.site/#{long_string}"
+ url: "https://dummy.site/",
+ referrer: String.duplicate("a", 2500)
+ }
+
+ conn = build_conn(:post, "/api/events", payload)
+ assert {:ok, request} = Request.build(conn)
+ assert request.referrer == String.duplicate("a", 2000)
+ end
+
+ test "returns validation error when props keys are too long" do
+ payload = %{
+ name: "pageview",
+ domain: "dummy.site",
+ url: "https://dummy.site/",
+ props: %{String.duplicate("a", 500) => "abc"}
+ }
+
+ conn = build_conn(:post, "/api/events", payload)
+ assert {:error, changeset} = Request.build(conn)
+
+ assert {"keys should have at most 300 bytes and values 2000 bytes", _} =
+ changeset.errors[:props]
+ end
+
+ test "returns validation error when props values are too long" do
+ payload = %{
+ name: "pageview",
+ domain: "dummy.site",
+ url: "https://dummy.site/",
+ props: %{"abc" => String.duplicate("a", 2500)}
}
conn = build_conn(:post, "/api/events", payload)
assert {:error, changeset} = Request.build(conn)
- assert {"should be at most %{count} character(s)", _} = changeset.errors[:pathname]
+
+ assert {"keys should have at most 300 bytes and values 2000 bytes", _} =
+ changeset.errors[:props]
+ end
+
+ test "does not fail when sending many props" do
+ payload = %{
+ name: "pageview",
+ domain: "dummy.site",
+ url: "https://dummy.site/",
+ props: for(i <- 1..100, do: {"key_#{i}", "value"}, into: %{})
+ }
+
+ conn = build_conn(:post, "/api/events", payload)
+ assert {:ok, request} = Request.build(conn)
+ assert map_size(request.props) == 100
end
test "malicious input, technically valid json" do