Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.
TypeScript analyticsanalytics-dashboardclickhouseelixirgoogle-analyticsmarketingopen-source-analyticsphoenixplausible-analyticspostgresqlprivacyprivacy-friendlyself-hostedsimple-analyticsstatisticstailwindcssweb-analyticswebsite-analyticswebsite-statswebsite-tracking
Ingestion logic for engagement_time (#5100)
Karl-Aksel Puulmann4c0c9e8411d3
diff --git a/lib/plausible/clickhouse_event_v2.ex b/lib/plausible/clickhouse_event_v2.exindex 3cda98dfcd..3366195576 100644--- a/lib/plausible/clickhouse_event_v2.ex+++ b/lib/plausible/clickhouse_event_v2.ex@@ -18,6 +18,7 @@ defmodule Plausible.ClickhouseEventV2 dofield :"meta.key", {:array, :string}field :"meta.value", {:array, :string}field :scroll_depth, Ch, type: "UInt8"+ field :engagement_time, Ch, type: "UInt32"field :revenue_source_amount, Ch, type: "Nullable(Decimal64(3))"field :revenue_source_currency, Ch, type: "FixedString(3)"@@ -62,6 +63,7 @@ defmodule Plausible.ClickhouseEventV2 do:"meta.key",:"meta.value",:scroll_depth,+ :engagement_time,:revenue_source_amount,:revenue_source_currency,:revenue_reporting_amount,diff --git a/lib/plausible/ingestion/event.ex b/lib/plausible/ingestion/event.exindex adbd488f82..592507b0fa 100644--- a/lib/plausible/ingestion/event.ex+++ b/lib/plausible/ingestion/event.ex@@ -251,7 +251,8 @@ defmodule Plausible.Ingestion.Event doname: event.request.event_name,hostname: event.request.hostname,pathname: event.request.pathname,- scroll_depth: event.request.scroll_depth+ scroll_depth: event.request.scroll_depth,+ engagement_time: event.request.engagement_time})enddiff --git a/lib/plausible/ingestion/request.ex b/lib/plausible/ingestion/request.exindex 7ae04ac0a4..0baf752de9 100644--- a/lib/plausible/ingestion/request.ex+++ b/lib/plausible/ingestion/request.ex@@ -42,6 +42,7 @@ defmodule Plausible.Ingestion.Request dofield :pathname, :stringfield :props, :mapfield :scroll_depth, :integer+ field :engagement_time, :integeron_ee dofield :revenue_source, :map@@ -79,6 +80,7 @@ defmodule Plausible.Ingestion.Request do|> put_referrer(request_body)|> put_props(request_body)|> put_scroll_depth(request_body)+ |> put_engagement_time(request_body)|> put_pathname()|> put_query_params()|> put_revenue_source(request_body)@@ -264,6 +266,20 @@ defmodule Plausible.Ingestion.Request doendend+ defp put_engagement_time(changeset, %{} = request_body) do+ if Changeset.get_field(changeset, :event_name) == "engagement" do+ engagement_time =+ case request_body["e"] do+ e when is_integer(e) and e >= 0 -> e+ _ -> 0+ end++ Changeset.put_change(changeset, :engagement_time, engagement_time)+ else+ changeset+ end+ end+defp put_query_params(changeset) docase Changeset.get_field(changeset, :uri) do%{query: query} when is_binary(query) ->diff --git a/test/plausible/ingestion/request_test.exs b/test/plausible/ingestion/request_test.exsindex f4c02fa865..ac102a42a8 100644--- a/test/plausible/ingestion/request_test.exs+++ b/test/plausible/ingestion/request_test.exs@@ -432,7 +432,7 @@ defmodule Plausible.Ingestion.RequestTest dotest "long body length" dopayload = """{- "name": "pageview",+ "name": "pageview","domain": "dummy.site","url": "#{:binary.copy("a", 1_000)}"}@@ -490,7 +490,8 @@ defmodule Plausible.Ingestion.RequestTest do"uri" => "https://dummy.site/pictures/index.html?foo=bar&baz=bam","user_agent" => "Mozilla","ip_classification" => nil,- "scroll_depth" => nil+ "scroll_depth" => nil,+ "engagement_time" => nil}assert %NaiveDateTime{} = NaiveDateTime.from_iso8601!(request["timestamp"])diff --git a/test/plausible_web/controllers/api/external_controller_test.exs b/test/plausible_web/controllers/api/external_controller_test.exsindex 0597939495..0a5dba7cae 100644--- a/test/plausible_web/controllers/api/external_controller_test.exs+++ b/test/plausible_web/controllers/api/external_controller_test.exs@@ -1313,35 +1313,61 @@ defmodule PlausibleWeb.Api.ExternalControllerTest doendend- describe "scroll depth tests" do+ describe "engagement event tests" dosetup dosite = new_site(){:ok, site: site}end- test "ingests scroll_depth as 255 when sd not in params", %{conn: conn, site: site} do+ test "ingests scroll_depth and engagement_time when not in params", %{conn: conn, site: site} dopost(conn, "/api/event", %{n: "pageview", u: "https://test.com", d: site.domain})post(conn, "/api/event", %{n: "engagement", u: "https://test.com", d: site.domain})engagement = get_events(site) |> Enum.find(&(&1.name == "engagement"))assert engagement.scroll_depth == 255+ assert engagement.engagement_time == 0end- test "sd field is ignored if name is not engagement", %{conn: conn, site: site} do- post(conn, "/api/event", %{n: "pageview", u: "https://test.com", d: site.domain, sd: 10})- post(conn, "/api/event", %{n: "custom_e", u: "https://test.com", d: site.domain, sd: 10})+ test "sd and e fields are ignored if name is not engagement", %{conn: conn, site: site} do+ post(conn, "/api/event", %{+ n: "pageview",+ u: "https://test.com",+ d: site.domain,+ sd: 10,+ e: 789+ })++ post(conn, "/api/event", %{+ n: "custom_e",+ u: "https://test.com",+ d: site.domain,+ sd: 10,+ e: 789+ })- assert [%{scroll_depth: 0}, %{scroll_depth: 0}] = get_events(site)+ assert [%{scroll_depth: 0, engagement_time: 0}, %{scroll_depth: 0, engagement_time: 0}] =+ get_events(site)end- test "ingests valid scroll_depth for a engagement", %{conn: conn, site: site} do+ test "ingests valid scroll_depth and engagement_time for a engagement", %{+ conn: conn,+ site: site+ } dopost(conn, "/api/event", %{n: "pageview", u: "https://test.com", d: site.domain})- post(conn, "/api/event", %{n: "engagement", u: "https://test.com", d: site.domain, sd: 25})++ post(conn, "/api/event", %{+ n: "engagement",+ u: "https://test.com",+ d: site.domain,+ sd: 25,+ e: 789+ })engagement = get_events(site) |> Enum.find(&(&1.name == "engagement"))assert engagement.scroll_depth == 25+ assert engagement.engagement_time == 789endtest "ingests scroll_depth as 100 when sd > 100", %{conn: conn, site: site} do@@ -1371,13 +1397,23 @@ defmodule PlausibleWeb.Api.ExternalControllerTest doassert engagement.scroll_depth == 255end- test "ingests valid scroll_depth for a engagement event", %{conn: conn, site: site} do+ test "ingests engagement_time as 0 when e is a string", %{conn: conn, site: site} dopost(conn, "/api/event", %{n: "pageview", u: "https://test.com", d: site.domain})- post(conn, "/api/event", %{n: "engagement", u: "https://test.com", d: site.domain, sd: 25})- event = get_events(site) |> Enum.find(&(&1.name == "engagement"))+ post(conn, "/api/event", %{n: "engagement", u: "https://test.com", d: site.domain, e: "789"})++ engagement = get_events(site) |> Enum.find(&(&1.name == "engagement"))++ assert engagement.engagement_time == 0+ end++ test "ingests engagement_time as 0 when e is a negative integer", %{conn: conn, site: site} do+ post(conn, "/api/event", %{n: "pageview", u: "https://test.com", d: site.domain})+ post(conn, "/api/event", %{n: "engagement", u: "https://test.com", d: site.domain, e: -100})++ engagement = get_events(site) |> Enum.find(&(&1.name == "engagement"))- assert event.scroll_depth == 25+ assert engagement.engagement_time == 0endend