Skip to content

Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.

Ingest scroll depth and engagment time from string + drop blanks (#5182)

RobertJoonas37813a160509
diff --git a/lib/plausible/ingestion/event.ex b/lib/plausible/ingestion/event.ex
index d08e8d4c26..acc3a73eb3 100644
--- a/lib/plausible/ingestion/event.ex
+++ b/lib/plausible/ingestion/event.ex
@@ -36,7 +36,7 @@ defmodule Plausible.Ingestion.Event do
| :verification_agent
| :lock_timeout
| :no_session_for_engagement
- | :engagement_ff_throttle
+ | :blank_engagement
@type t() :: %__MODULE__{
domain: String.t() | nil,
@@ -58,15 +58,16 @@ defmodule Plausible.Ingestion.Event do
for domain <- domains, do: drop(new(domain, request), :spam_referrer)
else
Enum.reduce(domains, [], fn domain, acc ->
- with {:allow, site} <- GateKeeper.check(domain),
- :ok <- check_engagements_flag(domain, request) do
- processed =
- domain
- |> new(site, request)
- |> process_unless_dropped(pipeline(), context)
-
- [processed | acc]
- else
+ # credo:disable-for-next-line Credo.Check.Refactor.Nesting
+ case GateKeeper.check(domain) do
+ {:allow, site} ->
+ processed =
+ domain
+ |> new(site, request)
+ |> process_unless_dropped(pipeline(), context)
+
+ [processed | acc]
+
{:deny, reason} ->
[drop(new(domain, request), reason) | acc]
end
@@ -77,16 +78,6 @@ defmodule Plausible.Ingestion.Event do
{:ok, %{dropped: dropped, buffered: buffered}}
end
- defp check_engagements_flag(domain, %{event_name: "engagement"}) do
- if FunWithFlags.enabled?(:engagements, for: %Plausible.Site{domain: domain}) do
- :ok
- else
- {:deny, :engagement_ff_throttle}
- end
- end
-
- defp check_engagements_flag(_, _), do: :ok
-
@spec telemetry_event_buffered() :: [atom()]
def telemetry_event_buffered() do
[:plausible, :ingest, :event, :buffered]
@@ -124,6 +115,7 @@ defmodule Plausible.Ingestion.Event do
defp pipeline() do
[
+ drop_blank_engagement: &drop_blank_engagement/2,
drop_verification_agent: &drop_verification_agent/2,
drop_datacenter_ip: &drop_datacenter_ip/2,
drop_shield_rule_hostname: &drop_shield_rule_hostname/2,
@@ -188,6 +180,16 @@ defmodule Plausible.Ingestion.Event do
struct!(event, clickhouse_session_attrs: Map.merge(event.clickhouse_session_attrs, attrs))
end
+ defp drop_blank_engagement(%__MODULE__{} = event, _context) do
+ case event.request do
+ %{event_name: "engagement", scroll_depth: 255, engagement_time: 0} ->
+ drop(event, :blank_engagement)
+
+ _ ->
+ event
+ end
+ end
+
defp drop_verification_agent(%__MODULE__{} = event, _context) do
case event.request.user_agent do
@verification_user_agent ->
diff --git a/lib/plausible/ingestion/request.ex b/lib/plausible/ingestion/request.ex
index 0baf752de9..5807f5adf2 100644
--- a/lib/plausible/ingestion/request.ex
+++ b/lib/plausible/ingestion/request.ex
@@ -253,13 +253,7 @@ defmodule Plausible.Ingestion.Request do
defp put_scroll_depth(changeset, %{} = request_body) do
if Changeset.get_field(changeset, :event_name) == "engagement" do
- scroll_depth =
- case request_body["sd"] do
- sd when is_integer(sd) and sd >= 0 and sd <= 100 -> sd
- sd when is_integer(sd) and sd > 100 -> 100
- _ -> 255
- end
-
+ scroll_depth = parse_scroll_depth(request_body["sd"])
Changeset.put_change(changeset, :scroll_depth, scroll_depth)
else
changeset
@@ -268,11 +262,7 @@ defmodule Plausible.Ingestion.Request do
defp put_engagement_time(changeset, %{} = request_body) do
if Changeset.get_field(changeset, :event_name) == "engagement" do
- engagement_time =
- case request_body["e"] do
- e when is_integer(e) and e >= 0 -> e
- _ -> 0
- end
+ engagement_time = parse_engagement_time(request_body["e"])
Changeset.put_change(changeset, :engagement_time, engagement_time)
else
@@ -339,6 +329,31 @@ defmodule Plausible.Ingestion.Request do
def sanitize_hostname(nil) do
nil
end
+
+ @missing_scroll_depth 255
+
+ defp parse_scroll_depth(sd) when is_binary(sd) do
+ case Integer.parse(sd) do
+ {sd_int, ""} -> parse_scroll_depth(sd_int)
+ _ -> @missing_scroll_depth
+ end
+ end
+
+ defp parse_scroll_depth(sd) when is_integer(sd) and sd >= 0 and sd <= 100, do: sd
+ defp parse_scroll_depth(sd) when is_integer(sd) and sd > 100, do: 100
+ defp parse_scroll_depth(_), do: @missing_scroll_depth
+
+ @missing_engagement_time 0
+
+ defp parse_engagement_time(et) when is_binary(et) do
+ case Integer.parse(et) do
+ {et_int, ""} -> parse_engagement_time(et_int)
+ _ -> @missing_engagement_time
+ end
+ end
+
+ defp parse_engagement_time(et) when is_integer(et) and et >= 0, do: et
+ defp parse_engagement_time(_), do: @missing_engagement_time
end
defimpl Jason.Encoder, for: URI do
diff --git a/test/plausible/ingestion/event_test.exs b/test/plausible/ingestion/event_test.exs
index 38e185ebc6..42463baf2f 100644
--- a/test/plausible/ingestion/event_test.exs
+++ b/test/plausible/ingestion/event_test.exs
@@ -329,7 +329,9 @@ defmodule Plausible.Ingestion.EventTest do
payload = %{
name: "engagement",
url: "https://#{site.domain}/123",
- d: "#{site.domain}"
+ d: "#{site.domain}",
+ sd: 25,
+ et: 1000
}
conn = build_conn(:post, "/api/events", payload)
@@ -339,11 +341,9 @@ defmodule Plausible.Ingestion.EventTest do
assert dropped.drop_reason == :no_session_for_engagement
end
- test "TEMPORARY: drops engagement if FF not enabled for site" do
+ test "blank engagements (i.e. both 'sd' and 'e' missing) get dropped" do
site = new_site()
- FunWithFlags.disable(:engagements, for_actor: "site:#{site.domain}")
-
payload = %{
name: "engagement",
url: "https://#{site.domain}/123",
@@ -354,7 +354,7 @@ defmodule Plausible.Ingestion.EventTest do
assert {:ok, request} = Request.build(conn)
assert {:ok, %{buffered: [], dropped: [dropped]}} = Event.build_and_buffer(request)
- assert dropped.drop_reason == :engagement_ff_throttle
+ assert dropped.drop_reason == :blank_engagement
end
@tag :ee_only
diff --git a/test/plausible/stats/query_test.exs b/test/plausible/stats/query_test.exs
index 1461aaab7e..02d58a32a8 100644
--- a/test/plausible/stats/query_test.exs
+++ b/test/plausible/stats/query_test.exs
@@ -167,7 +167,10 @@ defmodule Plausible.Stats.QueryTest do
end
test "defaults to 30 days format", %{site: site} do
- assert Query.from(site, %{}) == Query.from(site, %{"period" => "30d"})
+ query_default = Query.from(site, %{}) |> Map.delete(:now)
+ query_30d = Query.from(site, %{"period" => "30d"}) |> Map.delete(:now)
+
+ assert query_default == query_30d
end
test "parses custom format", %{site: site} do
diff --git a/test/plausible_web/controllers/api/external_controller_test.exs b/test/plausible_web/controllers/api/external_controller_test.exs
index 0a5dba7cae..eba5946bf3 100644
--- a/test/plausible_web/controllers/api/external_controller_test.exs
+++ b/test/plausible_web/controllers/api/external_controller_test.exs
@@ -660,6 +660,8 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
name: "engagement",
url: "http://ab.cd/",
domain: site.domain,
+ sd: 50,
+ e: 1000,
props: %{
bool_test: true,
number_test: 12
@@ -1319,13 +1321,21 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
{:ok, site: site}
end
- test "ingests scroll_depth and engagement_time when not in params", %{conn: conn, site: site} do
+ test "ingests scroll_depth as 255 when not in params", %{conn: conn, site: site} do
post(conn, "/api/event", %{n: "pageview", u: "https://test.com", d: site.domain})
- post(conn, "/api/event", %{n: "engagement", u: "https://test.com", d: site.domain})
+ post(conn, "/api/event", %{n: "engagement", u: "https://test.com", d: site.domain, e: 200})
engagement = get_events(site) |> Enum.find(&(&1.name == "engagement"))
assert engagement.scroll_depth == 255
+ end
+
+ test "ingests engagement_time as 0 when not in params", %{conn: conn, site: site} do
+ post(conn, "/api/event", %{n: "pageview", u: "https://test.com", d: site.domain})
+ post(conn, "/api/event", %{n: "engagement", u: "https://test.com", d: site.domain, sd: 50})
+
+ engagement = get_events(site) |> Enum.find(&(&1.name == "engagement"))
+
assert engagement.engagement_time == 0
end
@@ -1379,37 +1389,73 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
assert engagement.scroll_depth == 100
end
- test "ingests scroll_depth as 255 when sd is a string", %{conn: conn, site: site} do
+ test "parses scroll_depth from a string", %{conn: conn, site: site} do
post(conn, "/api/event", %{n: "pageview", u: "https://test.com", d: site.domain})
post(conn, "/api/event", %{n: "engagement", u: "https://test.com", d: site.domain, sd: "1"})
engagement = get_events(site) |> Enum.find(&(&1.name == "engagement"))
- assert engagement.scroll_depth == 255
+ assert engagement.scroll_depth == 1
end
test "ingests scroll_depth as 255 when sd is a negative integer", %{conn: conn, site: site} do
post(conn, "/api/event", %{n: "pageview", u: "https://test.com", d: site.domain})
- post(conn, "/api/event", %{n: "engagement", u: "https://test.com", d: site.domain, sd: -1})
+
+ post(conn, "/api/event", %{
+ n: "engagement",
+ u: "https://test.com",
+ d: site.domain,
+ sd: -1,
+ e: 100
+ })
engagement = get_events(site) |> Enum.find(&(&1.name == "engagement"))
assert engagement.scroll_depth == 255
end
- test "ingests engagement_time as 0 when e is a string", %{conn: conn, site: site} do
+ test "ingests scroll_depth as 255 when sd is a non-number string", %{conn: conn, site: site} do
post(conn, "/api/event", %{n: "pageview", u: "https://test.com", d: site.domain})
- post(conn, "/api/event", %{n: "engagement", u: "https://test.com", d: site.domain, e: "789"})
+ post(conn, "/api/event", %{
+ n: "engagement",
+ u: "https://test.com",
+ d: site.domain,
+ sd: "12asd",
+ e: 100
+ })
engagement = get_events(site) |> Enum.find(&(&1.name == "engagement"))
- assert engagement.engagement_time == 0
+ assert engagement.scroll_depth == 255
+ end
+
+ test "ingests engagement_time from a string", %{conn: conn, site: site} do
+ post(conn, "/api/event", %{n: "pageview", u: "https://test.com", d: site.domain})
+
+ post(conn, "/api/event", %{
+ n: "engagement",
+ u: "https://test.com",
+ d: site.domain,
+ e: "789",
+ sd: 40
+ })
+
+ engagement = get_events(site) |> Enum.find(&(&1.name == "engagement"))
+
+ assert engagement.engagement_time == 789
end
test "ingests engagement_time as 0 when e is a negative integer", %{conn: conn, site: site} do
post(conn, "/api/event", %{n: "pageview", u: "https://test.com", d: site.domain})
- post(conn, "/api/event", %{n: "engagement", u: "https://test.com", d: site.domain, e: -100})
+
+ post(conn, "/api/event", %{
+ n: "engagement",
+ u: "https://test.com",
+ d: site.domain,
+ e: -100,
+ sd: 50
+ })
engagement = get_events(site) |> Enum.find(&(&1.name == "engagement"))
diff --git a/test/test_helper.exs b/test/test_helper.exs
index da85677609..f2114ce256 100644
--- a/test/test_helper.exs
+++ b/test/test_helper.exs
@@ -8,7 +8,6 @@ Application.ensure_all_started(:double)
FunWithFlags.enable(:channels)
FunWithFlags.enable(:scroll_depth)
-FunWithFlags.enable(:engagements)
FunWithFlags.enable(:teams)
FunWithFlags.enable(:saved_segments)