Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.
TypeScript analyticsanalytics-dashboardclickhouseelixirgoogle-analyticsmarketingopen-source-analyticsphoenixplausible-analyticspostgresqlprivacyprivacy-friendlyself-hostedsimple-analyticsstatisticstailwindcssweb-analyticswebsite-analyticswebsite-statswebsite-tracking
Ingest scroll depth and engagment time from string + drop blanks (#5182)
RobertJoonas37813a160509
diff --git a/lib/plausible/ingestion/event.ex b/lib/plausible/ingestion/event.exindex d08e8d4c26..acc3a73eb3 100644--- a/lib/plausible/ingestion/event.ex+++ b/lib/plausible/ingestion/event.ex@@ -36,7 +36,7 @@ defmodule Plausible.Ingestion.Event do| :verification_agent| :lock_timeout| :no_session_for_engagement- | :engagement_ff_throttle+ | :blank_engagement@type t() :: %__MODULE__{domain: String.t() | nil,@@ -58,15 +58,16 @@ defmodule Plausible.Ingestion.Event dofor domain <- domains, do: drop(new(domain, request), :spam_referrer)elseEnum.reduce(domains, [], fn domain, acc ->- with {:allow, site} <- GateKeeper.check(domain),- :ok <- check_engagements_flag(domain, request) do- processed =- domain- |> new(site, request)- |> process_unless_dropped(pipeline(), context)-- [processed | acc]- else+ # credo:disable-for-next-line Credo.Check.Refactor.Nesting+ case GateKeeper.check(domain) do+ {:allow, site} ->+ processed =+ domain+ |> new(site, request)+ |> process_unless_dropped(pipeline(), context)++ [processed | acc]+{:deny, reason} ->[drop(new(domain, request), reason) | acc]end@@ -77,16 +78,6 @@ defmodule Plausible.Ingestion.Event do{:ok, %{dropped: dropped, buffered: buffered}}end- defp check_engagements_flag(domain, %{event_name: "engagement"}) do- if FunWithFlags.enabled?(:engagements, for: %Plausible.Site{domain: domain}) do- :ok- else- {:deny, :engagement_ff_throttle}- end- end-- defp check_engagements_flag(_, _), do: :ok-@spec telemetry_event_buffered() :: [atom()]def telemetry_event_buffered() do[:plausible, :ingest, :event, :buffered]@@ -124,6 +115,7 @@ defmodule Plausible.Ingestion.Event dodefp pipeline() do[+ drop_blank_engagement: &drop_blank_engagement/2,drop_verification_agent: &drop_verification_agent/2,drop_datacenter_ip: &drop_datacenter_ip/2,drop_shield_rule_hostname: &drop_shield_rule_hostname/2,@@ -188,6 +180,16 @@ defmodule Plausible.Ingestion.Event dostruct!(event, clickhouse_session_attrs: Map.merge(event.clickhouse_session_attrs, attrs))end+ defp drop_blank_engagement(%__MODULE__{} = event, _context) do+ case event.request do+ %{event_name: "engagement", scroll_depth: 255, engagement_time: 0} ->+ drop(event, :blank_engagement)++ _ ->+ event+ end+ end+defp drop_verification_agent(%__MODULE__{} = event, _context) docase event.request.user_agent do@verification_user_agent ->diff --git a/lib/plausible/ingestion/request.ex b/lib/plausible/ingestion/request.exindex 0baf752de9..5807f5adf2 100644--- a/lib/plausible/ingestion/request.ex+++ b/lib/plausible/ingestion/request.ex@@ -253,13 +253,7 @@ defmodule Plausible.Ingestion.Request dodefp put_scroll_depth(changeset, %{} = request_body) doif Changeset.get_field(changeset, :event_name) == "engagement" do- scroll_depth =- case request_body["sd"] do- sd when is_integer(sd) and sd >= 0 and sd <= 100 -> sd- sd when is_integer(sd) and sd > 100 -> 100- _ -> 255- end-+ scroll_depth = parse_scroll_depth(request_body["sd"])Changeset.put_change(changeset, :scroll_depth, scroll_depth)elsechangeset@@ -268,11 +262,7 @@ defmodule Plausible.Ingestion.Request dodefp put_engagement_time(changeset, %{} = request_body) doif Changeset.get_field(changeset, :event_name) == "engagement" do- engagement_time =- case request_body["e"] do- e when is_integer(e) and e >= 0 -> e- _ -> 0- end+ engagement_time = parse_engagement_time(request_body["e"])Changeset.put_change(changeset, :engagement_time, engagement_time)else@@ -339,6 +329,31 @@ defmodule Plausible.Ingestion.Request dodef sanitize_hostname(nil) donilend++ @missing_scroll_depth 255++ defp parse_scroll_depth(sd) when is_binary(sd) do+ case Integer.parse(sd) do+ {sd_int, ""} -> parse_scroll_depth(sd_int)+ _ -> @missing_scroll_depth+ end+ end++ defp parse_scroll_depth(sd) when is_integer(sd) and sd >= 0 and sd <= 100, do: sd+ defp parse_scroll_depth(sd) when is_integer(sd) and sd > 100, do: 100+ defp parse_scroll_depth(_), do: @missing_scroll_depth++ @missing_engagement_time 0++ defp parse_engagement_time(et) when is_binary(et) do+ case Integer.parse(et) do+ {et_int, ""} -> parse_engagement_time(et_int)+ _ -> @missing_engagement_time+ end+ end++ defp parse_engagement_time(et) when is_integer(et) and et >= 0, do: et+ defp parse_engagement_time(_), do: @missing_engagement_timeenddefimpl Jason.Encoder, for: URI dodiff --git a/test/plausible/ingestion/event_test.exs b/test/plausible/ingestion/event_test.exsindex 38e185ebc6..42463baf2f 100644--- a/test/plausible/ingestion/event_test.exs+++ b/test/plausible/ingestion/event_test.exs@@ -329,7 +329,9 @@ defmodule Plausible.Ingestion.EventTest dopayload = %{name: "engagement",url: "https://#{site.domain}/123",- d: "#{site.domain}"+ d: "#{site.domain}",+ sd: 25,+ et: 1000}conn = build_conn(:post, "/api/events", payload)@@ -339,11 +341,9 @@ defmodule Plausible.Ingestion.EventTest doassert dropped.drop_reason == :no_session_for_engagementend- test "TEMPORARY: drops engagement if FF not enabled for site" do+ test "blank engagements (i.e. both 'sd' and 'e' missing) get dropped" dosite = new_site()- FunWithFlags.disable(:engagements, for_actor: "site:#{site.domain}")-payload = %{name: "engagement",url: "https://#{site.domain}/123",@@ -354,7 +354,7 @@ defmodule Plausible.Ingestion.EventTest doassert {:ok, request} = Request.build(conn)assert {:ok, %{buffered: [], dropped: [dropped]}} = Event.build_and_buffer(request)- assert dropped.drop_reason == :engagement_ff_throttle+ assert dropped.drop_reason == :blank_engagementend@tag :ee_onlydiff --git a/test/plausible/stats/query_test.exs b/test/plausible/stats/query_test.exsindex 1461aaab7e..02d58a32a8 100644--- a/test/plausible/stats/query_test.exs+++ b/test/plausible/stats/query_test.exs@@ -167,7 +167,10 @@ defmodule Plausible.Stats.QueryTest doendtest "defaults to 30 days format", %{site: site} do- assert Query.from(site, %{}) == Query.from(site, %{"period" => "30d"})+ query_default = Query.from(site, %{}) |> Map.delete(:now)+ query_30d = Query.from(site, %{"period" => "30d"}) |> Map.delete(:now)++ assert query_default == query_30dendtest "parses custom format", %{site: site} dodiff --git a/test/plausible_web/controllers/api/external_controller_test.exs b/test/plausible_web/controllers/api/external_controller_test.exsindex 0a5dba7cae..eba5946bf3 100644--- a/test/plausible_web/controllers/api/external_controller_test.exs+++ b/test/plausible_web/controllers/api/external_controller_test.exs@@ -660,6 +660,8 @@ defmodule PlausibleWeb.Api.ExternalControllerTest doname: "engagement",url: "http://ab.cd/",domain: site.domain,+ sd: 50,+ e: 1000,props: %{bool_test: true,number_test: 12@@ -1319,13 +1321,21 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do{:ok, site: site}end- test "ingests scroll_depth and engagement_time when not in params", %{conn: conn, site: site} do+ test "ingests scroll_depth as 255 when not in params", %{conn: conn, site: site} dopost(conn, "/api/event", %{n: "pageview", u: "https://test.com", d: site.domain})- post(conn, "/api/event", %{n: "engagement", u: "https://test.com", d: site.domain})+ post(conn, "/api/event", %{n: "engagement", u: "https://test.com", d: site.domain, e: 200})engagement = get_events(site) |> Enum.find(&(&1.name == "engagement"))assert engagement.scroll_depth == 255+ end++ test "ingests engagement_time as 0 when not in params", %{conn: conn, site: site} do+ post(conn, "/api/event", %{n: "pageview", u: "https://test.com", d: site.domain})+ post(conn, "/api/event", %{n: "engagement", u: "https://test.com", d: site.domain, sd: 50})++ engagement = get_events(site) |> Enum.find(&(&1.name == "engagement"))+assert engagement.engagement_time == 0end@@ -1379,37 +1389,73 @@ defmodule PlausibleWeb.Api.ExternalControllerTest doassert engagement.scroll_depth == 100end- test "ingests scroll_depth as 255 when sd is a string", %{conn: conn, site: site} do+ test "parses scroll_depth from a string", %{conn: conn, site: site} dopost(conn, "/api/event", %{n: "pageview", u: "https://test.com", d: site.domain})post(conn, "/api/event", %{n: "engagement", u: "https://test.com", d: site.domain, sd: "1"})engagement = get_events(site) |> Enum.find(&(&1.name == "engagement"))- assert engagement.scroll_depth == 255+ assert engagement.scroll_depth == 1endtest "ingests scroll_depth as 255 when sd is a negative integer", %{conn: conn, site: site} dopost(conn, "/api/event", %{n: "pageview", u: "https://test.com", d: site.domain})- post(conn, "/api/event", %{n: "engagement", u: "https://test.com", d: site.domain, sd: -1})++ post(conn, "/api/event", %{+ n: "engagement",+ u: "https://test.com",+ d: site.domain,+ sd: -1,+ e: 100+ })engagement = get_events(site) |> Enum.find(&(&1.name == "engagement"))assert engagement.scroll_depth == 255end- test "ingests engagement_time as 0 when e is a string", %{conn: conn, site: site} do+ test "ingests scroll_depth as 255 when sd is a non-number string", %{conn: conn, site: site} dopost(conn, "/api/event", %{n: "pageview", u: "https://test.com", d: site.domain})- post(conn, "/api/event", %{n: "engagement", u: "https://test.com", d: site.domain, e: "789"})+ post(conn, "/api/event", %{+ n: "engagement",+ u: "https://test.com",+ d: site.domain,+ sd: "12asd",+ e: 100+ })engagement = get_events(site) |> Enum.find(&(&1.name == "engagement"))- assert engagement.engagement_time == 0+ assert engagement.scroll_depth == 255+ end++ test "ingests engagement_time from a string", %{conn: conn, site: site} do+ post(conn, "/api/event", %{n: "pageview", u: "https://test.com", d: site.domain})++ post(conn, "/api/event", %{+ n: "engagement",+ u: "https://test.com",+ d: site.domain,+ e: "789",+ sd: 40+ })++ engagement = get_events(site) |> Enum.find(&(&1.name == "engagement"))++ assert engagement.engagement_time == 789endtest "ingests engagement_time as 0 when e is a negative integer", %{conn: conn, site: site} dopost(conn, "/api/event", %{n: "pageview", u: "https://test.com", d: site.domain})- post(conn, "/api/event", %{n: "engagement", u: "https://test.com", d: site.domain, e: -100})++ post(conn, "/api/event", %{+ n: "engagement",+ u: "https://test.com",+ d: site.domain,+ e: -100,+ sd: 50+ })engagement = get_events(site) |> Enum.find(&(&1.name == "engagement"))diff --git a/test/test_helper.exs b/test/test_helper.exsindex da85677609..f2114ce256 100644--- a/test/test_helper.exs+++ b/test/test_helper.exs@@ -8,7 +8,6 @@ Application.ensure_all_started(:double)FunWithFlags.enable(:channels)FunWithFlags.enable(:scroll_depth)-FunWithFlags.enable(:engagements)FunWithFlags.enable(:teams)FunWithFlags.enable(:saved_segments)