Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.
TypeScript analyticsanalytics-dashboardclickhouseelixirgoogle-analyticsmarketingopen-source-analyticsphoenixplausible-analyticspostgresqlprivacyprivacy-friendlyself-hostedsimple-analyticsstatisticstailwindcssweb-analyticswebsite-analyticswebsite-statswebsite-tracking
Ingestion: Reject blank engagements with 400 (#5253)
RobertJoonas487e3d81b4b6
diff --git a/lib/plausible/ingestion/event.ex b/lib/plausible/ingestion/event.exindex dd87587b1b..bed0d239bc 100644--- a/lib/plausible/ingestion/event.ex+++ b/lib/plausible/ingestion/event.ex@@ -36,7 +36,6 @@ defmodule Plausible.Ingestion.Event do| :verification_agent| :lock_timeout| :no_session_for_engagement- | :blank_engagement@type t() :: %__MODULE__{domain: String.t() | nil,@@ -128,7 +127,6 @@ defmodule Plausible.Ingestion.Event dodefp pipeline() do[- drop_blank_engagement: &drop_blank_engagement/2,drop_verification_agent: &drop_verification_agent/2,drop_datacenter_ip: &drop_datacenter_ip/2,drop_shield_rule_hostname: &drop_shield_rule_hostname/2,@@ -193,16 +191,6 @@ defmodule Plausible.Ingestion.Event dostruct!(event, clickhouse_session_attrs: Map.merge(event.clickhouse_session_attrs, attrs))end- defp drop_blank_engagement(%__MODULE__{} = event, _context) do- case event.request do- %{event_name: "engagement", scroll_depth: 255, engagement_time: 0} ->- drop(event, :blank_engagement)-- _ ->- event- end- end-defp drop_verification_agent(%__MODULE__{} = event, _context) docase event.request.user_agent do@verification_user_agent ->diff --git a/lib/plausible/ingestion/request.ex b/lib/plausible/ingestion/request.exindex 19ec409a6e..c61a68ad5e 100644--- a/lib/plausible/ingestion/request.ex+++ b/lib/plausible/ingestion/request.ex@@ -27,6 +27,17 @@ defmodule Plausible.Ingestion.Request doalias Ecto.Changeset@max_url_size 2_000+ @missing_scroll_depth 255+ @missing_engagement_time 0++ # :KLUDGE: Old version of tracker script sent huge values for engagement time. Ignore+ # these while users might still have the old script cached.+ @too_large_engagement_time :timer.hours(30 * 24)++ @blank_engagement_error_message "engagement event requires a valid integer value for at least one of 'sd' or 'e' fields"++ def too_large_engagement_time(), do: @too_large_engagement_time+ def blank_engagement_error_message(), do: @blank_engagement_error_message@primary_key falseembedded_schema do@@ -80,8 +91,7 @@ defmodule Plausible.Ingestion.Request do|> put_request_params(request_body)|> put_referrer(request_body)|> put_props(request_body)- |> put_scroll_depth(request_body)- |> put_engagement_time(request_body)+ |> put_engagement_fields(request_body)|> put_pathname()|> put_query_params()|> put_revenue_source(request_body)@@ -253,20 +263,24 @@ defmodule Plausible.Ingestion.Request doendend- defp put_scroll_depth(changeset, %{} = request_body) do+ defp put_engagement_fields(changeset, %{} = request_body) doif Changeset.get_field(changeset, :event_name) == "engagement" doscroll_depth = parse_scroll_depth(request_body["sd"])- Changeset.put_change(changeset, :scroll_depth, scroll_depth)- else- changeset- end- end-- defp put_engagement_time(changeset, %{} = request_body) do- if Changeset.get_field(changeset, :event_name) == "engagement" doengagement_time = parse_engagement_time(request_body["e"])- Changeset.put_change(changeset, :engagement_time, engagement_time)+ case {scroll_depth, engagement_time} do+ {@missing_scroll_depth, @missing_engagement_time} ->+ changeset+ |> Changeset.add_error(+ :event_name,+ "engagement event requires a valid integer value for at least one of 'sd' or 'e' fields"+ )++ _ ->+ changeset+ |> Changeset.put_change(:scroll_depth, scroll_depth)+ |> Changeset.put_change(:engagement_time, engagement_time)+ endelsechangesetend@@ -342,8 +356,6 @@ defmodule Plausible.Ingestion.Request donilend- @missing_scroll_depth 255-defp parse_scroll_depth(sd) when is_binary(sd) docase Integer.parse(sd) do{sd_int, ""} -> parse_scroll_depth(sd_int)@@ -355,8 +367,6 @@ defmodule Plausible.Ingestion.Request dodefp parse_scroll_depth(sd) when is_integer(sd) and sd > 100, do: 100defp parse_scroll_depth(_), do: @missing_scroll_depth- @missing_engagement_time 0-defp parse_engagement_time(et) when is_binary(et) docase Integer.parse(et) do{et_int, ""} -> parse_engagement_time(et_int)@@ -364,10 +374,6 @@ defmodule Plausible.Ingestion.Request doendend- # :KLUDGE: Old version of tracker script sent huge values for engagement time. Ignore- # these while users might still have the old script cached.- @too_large_engagement_time :timer.hours(30 * 24)-defp parse_engagement_time(et)when is_integer(et) and et >= 0 and et < @too_large_engagement_time,do: etdiff --git a/test/plausible/ingestion/event_test.exs b/test/plausible/ingestion/event_test.exsindex a3beb68708..ee943a01ed 100644--- a/test/plausible/ingestion/event_test.exs+++ b/test/plausible/ingestion/event_test.exs@@ -376,22 +376,6 @@ defmodule Plausible.Ingestion.EventTest doassert dropped.drop_reason == :no_session_for_engagementend- test "blank engagements (i.e. both 'sd' and 'e' missing) get dropped" do- site = new_site()-- payload = %{- name: "engagement",- url: "https://#{site.domain}/123",- d: "#{site.domain}"- }-- conn = build_conn(:post, "/api/events", payload)-- assert {:ok, request} = Request.build(conn)- assert {:ok, %{buffered: [], dropped: [dropped]}} = Event.build_and_buffer(request)- assert dropped.drop_reason == :blank_engagement- end-@tag :ee_onlytest "saves revenue amount" dosite = new_site()diff --git a/test/plausible/ingestion/request_test.exs b/test/plausible/ingestion/request_test.exsindex 6a80c5df44..1abbebf9f7 100644--- a/test/plausible/ingestion/request_test.exs+++ b/test/plausible/ingestion/request_test.exs@@ -499,4 +499,74 @@ defmodule Plausible.Ingestion.RequestTest doassert %NaiveDateTime{} = NaiveDateTime.from_iso8601!(request["timestamp"])end++ describe "engagement event" do+ test "fails validation if no engagement metrics are present or are invalid" do+ [%{}, %{sd: -1}, %{e: "abc"}, %{sd: "null", e: "null"}]+ |> Enum.each(fn invalid_param_set ->+ assert {:error, changeset} = build_engagement_request(invalid_param_set)+ assert {msg, _} = changeset.errors[:event_name]+ assert msg == Request.blank_engagement_error_message()+ end)+ end++ test "sets valid scroll_depth" do+ assert {:ok, %Request{scroll_depth: 0}} = build_engagement_request(%{sd: 0})+ assert {:ok, %Request{scroll_depth: 25}} = build_engagement_request(%{sd: 25})+ assert {:ok, %Request{scroll_depth: 25}} = build_engagement_request(%{sd: "25"})+ end++ test "scroll_depth defaults to 100 when given integer is greater than 100" do+ assert {:ok, %Request{scroll_depth: 100}} = build_engagement_request(%{sd: 101})+ assert {:ok, %Request{scroll_depth: 100}} = build_engagement_request(%{sd: "101"})+ end++ test "scroll_depth defaults to 255 when given value is invalid or missing" do+ assert {:ok, %Request{scroll_depth: 255}} = build_engagement_request(%{sd: -1, e: 1})+ assert {:ok, %Request{scroll_depth: 255}} = build_engagement_request(%{sd: "abc", e: 1})+ assert {:ok, %Request{scroll_depth: 255}} = build_engagement_request(%{e: 1})+ end++ test "sets valid engagement_time" do+ assert {:ok, %Request{engagement_time: 123}} = build_engagement_request(%{e: 123})+ assert {:ok, %Request{engagement_time: 123}} = build_engagement_request(%{e: "123"})+ end++ test "engagement_time defaults to 0 when given value is invalid or missing" do+ assert {:ok, %Request{engagement_time: 0}} = build_engagement_request(%{sd: 1, e: -1})+ assert {:ok, %Request{engagement_time: 0}} = build_engagement_request(%{sd: 1, e: "abc"})++ assert {:ok, %Request{engagement_time: 0}} =+ build_engagement_request(%{sd: 1, e: Request.too_large_engagement_time()})+ end++ test "sd and e fields are ignored if name is not engagement" do+ params = %{name: "pageview", domain: "site.com", url: "https://site.com", sd: 25, e: 1000}++ assert {:ok, %Request{engagement_time: nil, scroll_depth: nil}} =+ build_conn(:post, "/api/events", params)+ |> Request.build()++ assert {:ok, %Request{engagement_time: nil, scroll_depth: nil}} =+ build_conn(:post, "/api/events", Map.put(params, :name, "Custom Event"))+ |> Request.build()+ end++ test "ingests valid scroll_depth and engagement_time for the same request" do+ assert {:ok, %Request{scroll_depth: 23, engagement_time: 100}} =+ build_engagement_request(%{sd: 23, e: 100})++ assert {:ok, %Request{scroll_depth: 23, engagement_time: 100}} =+ build_engagement_request(%{sd: "23", e: "100"})+ end+ end++ defp build_engagement_request(extra_params) do+ params =+ %{name: "engagement", domain: "dummy.site", url: "https://dummy.site/"}+ |> Map.merge(extra_params)++ build_conn(:post, "/api/events", params)+ |> Request.build()+ endenddiff --git a/test/plausible_web/controllers/api/external_controller_test.exs b/test/plausible_web/controllers/api/external_controller_test.exsindex 8874e9f5ed..8a0143154b 100644--- a/test/plausible_web/controllers/api/external_controller_test.exs+++ b/test/plausible_web/controllers/api/external_controller_test.exs@@ -1321,6 +1321,17 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do{:ok, site: site}end+ test "rejects engagement when both sd and e fields are missing", %{conn: conn, site: site} do+ post(conn, "/api/event", %{n: "pageview", u: "https://test.com", d: site.domain})++ conn = post(conn, "/api/event", %{n: "engagement", u: "https://test.com", d: site.domain})++ assert %{"errors" => %{"event_name" => [error_msg]}} = json_response(conn, 400)++ assert error_msg =~+ "engagement event requires a valid integer value for at least one of 'sd' or 'e' fields"+ end+test "ingests scroll_depth as 255 when not in params", %{conn: conn, site: site} dopost(conn, "/api/event", %{n: "pageview", u: "https://test.com", d: site.domain})post(conn, "/api/event", %{n: "engagement", u: "https://test.com", d: site.domain, e: 200})