Skip to content

Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.

Ingestion: Reject blank engagements with 400 (#5253)

RobertJoonas487e3d81b4b6
diff --git a/lib/plausible/ingestion/event.ex b/lib/plausible/ingestion/event.ex
index dd87587b1b..bed0d239bc 100644
--- a/lib/plausible/ingestion/event.ex
+++ b/lib/plausible/ingestion/event.ex
@@ -36,7 +36,6 @@ defmodule Plausible.Ingestion.Event do
| :verification_agent
| :lock_timeout
| :no_session_for_engagement
- | :blank_engagement
@type t() :: %__MODULE__{
domain: String.t() | nil,
@@ -128,7 +127,6 @@ defmodule Plausible.Ingestion.Event do
defp pipeline() do
[
- drop_blank_engagement: &drop_blank_engagement/2,
drop_verification_agent: &drop_verification_agent/2,
drop_datacenter_ip: &drop_datacenter_ip/2,
drop_shield_rule_hostname: &drop_shield_rule_hostname/2,
@@ -193,16 +191,6 @@ defmodule Plausible.Ingestion.Event do
struct!(event, clickhouse_session_attrs: Map.merge(event.clickhouse_session_attrs, attrs))
end
- defp drop_blank_engagement(%__MODULE__{} = event, _context) do
- case event.request do
- %{event_name: "engagement", scroll_depth: 255, engagement_time: 0} ->
- drop(event, :blank_engagement)
-
- _ ->
- event
- end
- end
-
defp drop_verification_agent(%__MODULE__{} = event, _context) do
case event.request.user_agent do
@verification_user_agent ->
diff --git a/lib/plausible/ingestion/request.ex b/lib/plausible/ingestion/request.ex
index 19ec409a6e..c61a68ad5e 100644
--- a/lib/plausible/ingestion/request.ex
+++ b/lib/plausible/ingestion/request.ex
@@ -27,6 +27,17 @@ defmodule Plausible.Ingestion.Request do
alias Ecto.Changeset
@max_url_size 2_000
+ @missing_scroll_depth 255
+ @missing_engagement_time 0
+
+ # :KLUDGE: Old version of tracker script sent huge values for engagement time. Ignore
+ # these while users might still have the old script cached.
+ @too_large_engagement_time :timer.hours(30 * 24)
+
+ @blank_engagement_error_message "engagement event requires a valid integer value for at least one of 'sd' or 'e' fields"
+
+ def too_large_engagement_time(), do: @too_large_engagement_time
+ def blank_engagement_error_message(), do: @blank_engagement_error_message
@primary_key false
embedded_schema do
@@ -80,8 +91,7 @@ defmodule Plausible.Ingestion.Request do
|> put_request_params(request_body)
|> put_referrer(request_body)
|> put_props(request_body)
- |> put_scroll_depth(request_body)
- |> put_engagement_time(request_body)
+ |> put_engagement_fields(request_body)
|> put_pathname()
|> put_query_params()
|> put_revenue_source(request_body)
@@ -253,20 +263,24 @@ defmodule Plausible.Ingestion.Request do
end
end
- defp put_scroll_depth(changeset, %{} = request_body) do
+ defp put_engagement_fields(changeset, %{} = request_body) do
if Changeset.get_field(changeset, :event_name) == "engagement" do
scroll_depth = parse_scroll_depth(request_body["sd"])
- Changeset.put_change(changeset, :scroll_depth, scroll_depth)
- else
- changeset
- end
- end
-
- defp put_engagement_time(changeset, %{} = request_body) do
- if Changeset.get_field(changeset, :event_name) == "engagement" do
engagement_time = parse_engagement_time(request_body["e"])
- Changeset.put_change(changeset, :engagement_time, engagement_time)
+ case {scroll_depth, engagement_time} do
+ {@missing_scroll_depth, @missing_engagement_time} ->
+ changeset
+ |> Changeset.add_error(
+ :event_name,
+ "engagement event requires a valid integer value for at least one of 'sd' or 'e' fields"
+ )
+
+ _ ->
+ changeset
+ |> Changeset.put_change(:scroll_depth, scroll_depth)
+ |> Changeset.put_change(:engagement_time, engagement_time)
+ end
else
changeset
end
@@ -342,8 +356,6 @@ defmodule Plausible.Ingestion.Request do
nil
end
- @missing_scroll_depth 255
-
defp parse_scroll_depth(sd) when is_binary(sd) do
case Integer.parse(sd) do
{sd_int, ""} -> parse_scroll_depth(sd_int)
@@ -355,8 +367,6 @@ defmodule Plausible.Ingestion.Request do
defp parse_scroll_depth(sd) when is_integer(sd) and sd > 100, do: 100
defp parse_scroll_depth(_), do: @missing_scroll_depth
- @missing_engagement_time 0
-
defp parse_engagement_time(et) when is_binary(et) do
case Integer.parse(et) do
{et_int, ""} -> parse_engagement_time(et_int)
@@ -364,10 +374,6 @@ defmodule Plausible.Ingestion.Request do
end
end
- # :KLUDGE: Old version of tracker script sent huge values for engagement time. Ignore
- # these while users might still have the old script cached.
- @too_large_engagement_time :timer.hours(30 * 24)
-
defp parse_engagement_time(et)
when is_integer(et) and et >= 0 and et < @too_large_engagement_time,
do: et
diff --git a/test/plausible/ingestion/event_test.exs b/test/plausible/ingestion/event_test.exs
index a3beb68708..ee943a01ed 100644
--- a/test/plausible/ingestion/event_test.exs
+++ b/test/plausible/ingestion/event_test.exs
@@ -376,22 +376,6 @@ defmodule Plausible.Ingestion.EventTest do
assert dropped.drop_reason == :no_session_for_engagement
end
- test "blank engagements (i.e. both 'sd' and 'e' missing) get dropped" do
- site = new_site()
-
- payload = %{
- name: "engagement",
- url: "https://#{site.domain}/123",
- d: "#{site.domain}"
- }
-
- conn = build_conn(:post, "/api/events", payload)
-
- assert {:ok, request} = Request.build(conn)
- assert {:ok, %{buffered: [], dropped: [dropped]}} = Event.build_and_buffer(request)
- assert dropped.drop_reason == :blank_engagement
- end
-
@tag :ee_only
test "saves revenue amount" do
site = new_site()
diff --git a/test/plausible/ingestion/request_test.exs b/test/plausible/ingestion/request_test.exs
index 6a80c5df44..1abbebf9f7 100644
--- a/test/plausible/ingestion/request_test.exs
+++ b/test/plausible/ingestion/request_test.exs
@@ -499,4 +499,74 @@ defmodule Plausible.Ingestion.RequestTest do
assert %NaiveDateTime{} = NaiveDateTime.from_iso8601!(request["timestamp"])
end
+
+ describe "engagement event" do
+ test "fails validation if no engagement metrics are present or are invalid" do
+ [%{}, %{sd: -1}, %{e: "abc"}, %{sd: "null", e: "null"}]
+ |> Enum.each(fn invalid_param_set ->
+ assert {:error, changeset} = build_engagement_request(invalid_param_set)
+ assert {msg, _} = changeset.errors[:event_name]
+ assert msg == Request.blank_engagement_error_message()
+ end)
+ end
+
+ test "sets valid scroll_depth" do
+ assert {:ok, %Request{scroll_depth: 0}} = build_engagement_request(%{sd: 0})
+ assert {:ok, %Request{scroll_depth: 25}} = build_engagement_request(%{sd: 25})
+ assert {:ok, %Request{scroll_depth: 25}} = build_engagement_request(%{sd: "25"})
+ end
+
+ test "scroll_depth defaults to 100 when given integer is greater than 100" do
+ assert {:ok, %Request{scroll_depth: 100}} = build_engagement_request(%{sd: 101})
+ assert {:ok, %Request{scroll_depth: 100}} = build_engagement_request(%{sd: "101"})
+ end
+
+ test "scroll_depth defaults to 255 when given value is invalid or missing" do
+ assert {:ok, %Request{scroll_depth: 255}} = build_engagement_request(%{sd: -1, e: 1})
+ assert {:ok, %Request{scroll_depth: 255}} = build_engagement_request(%{sd: "abc", e: 1})
+ assert {:ok, %Request{scroll_depth: 255}} = build_engagement_request(%{e: 1})
+ end
+
+ test "sets valid engagement_time" do
+ assert {:ok, %Request{engagement_time: 123}} = build_engagement_request(%{e: 123})
+ assert {:ok, %Request{engagement_time: 123}} = build_engagement_request(%{e: "123"})
+ end
+
+ test "engagement_time defaults to 0 when given value is invalid or missing" do
+ assert {:ok, %Request{engagement_time: 0}} = build_engagement_request(%{sd: 1, e: -1})
+ assert {:ok, %Request{engagement_time: 0}} = build_engagement_request(%{sd: 1, e: "abc"})
+
+ assert {:ok, %Request{engagement_time: 0}} =
+ build_engagement_request(%{sd: 1, e: Request.too_large_engagement_time()})
+ end
+
+ test "sd and e fields are ignored if name is not engagement" do
+ params = %{name: "pageview", domain: "site.com", url: "https://site.com", sd: 25, e: 1000}
+
+ assert {:ok, %Request{engagement_time: nil, scroll_depth: nil}} =
+ build_conn(:post, "/api/events", params)
+ |> Request.build()
+
+ assert {:ok, %Request{engagement_time: nil, scroll_depth: nil}} =
+ build_conn(:post, "/api/events", Map.put(params, :name, "Custom Event"))
+ |> Request.build()
+ end
+
+ test "ingests valid scroll_depth and engagement_time for the same request" do
+ assert {:ok, %Request{scroll_depth: 23, engagement_time: 100}} =
+ build_engagement_request(%{sd: 23, e: 100})
+
+ assert {:ok, %Request{scroll_depth: 23, engagement_time: 100}} =
+ build_engagement_request(%{sd: "23", e: "100"})
+ end
+ end
+
+ defp build_engagement_request(extra_params) do
+ params =
+ %{name: "engagement", domain: "dummy.site", url: "https://dummy.site/"}
+ |> Map.merge(extra_params)
+
+ build_conn(:post, "/api/events", params)
+ |> Request.build()
+ end
end
diff --git a/test/plausible_web/controllers/api/external_controller_test.exs b/test/plausible_web/controllers/api/external_controller_test.exs
index 8874e9f5ed..8a0143154b 100644
--- a/test/plausible_web/controllers/api/external_controller_test.exs
+++ b/test/plausible_web/controllers/api/external_controller_test.exs
@@ -1321,6 +1321,17 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
{:ok, site: site}
end
+ test "rejects engagement when both sd and e fields are missing", %{conn: conn, site: site} do
+ post(conn, "/api/event", %{n: "pageview", u: "https://test.com", d: site.domain})
+
+ conn = post(conn, "/api/event", %{n: "engagement", u: "https://test.com", d: site.domain})
+
+ assert %{"errors" => %{"event_name" => [error_msg]}} = json_response(conn, 400)
+
+ assert error_msg =~
+ "engagement event requires a valid integer value for at least one of 'sd' or 'e' fields"
+ end
+
test "ingests scroll_depth as 255 when not in params", %{conn: conn, site: site} do
post(conn, "/api/event", %{n: "pageview", u: "https://test.com", d: site.domain})
post(conn, "/api/event", %{n: "engagement", u: "https://test.com", d: site.domain, e: 200})