Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.
TypeScript analyticsanalytics-dashboardclickhouseelixirgoogle-analyticsmarketingopen-source-analyticsphoenixplausible-analyticspostgresqlprivacyprivacy-friendlyself-hostedsimple-analyticsstatisticstailwindcssweb-analyticswebsite-analyticswebsite-statswebsite-tracking
Average Scroll Depth Metric: handle missing data better (#4889)
RobertJoonasd21d48558a5e
diff --git a/lib/plausible/data_migration/missing_scroll_depth.ex b/lib/plausible/data_migration/missing_scroll_depth.exnew file mode 100644index 0000000000..533287a1dd--- /dev/null+++ b/lib/plausible/data_migration/missing_scroll_depth.ex@@ -0,0 +1,18 @@+defmodule Plausible.DataMigration.MissingScrollDepth do+ @moduledoc """+ Set scroll_depth to 255 (max UInt8) for all pageleave events where it's 0.+ """++ def run() do+ Plausible.IngestRepo.query!(+ """+ ALTER TABLE events_v2+ #{Plausible.MigrationUtils.on_cluster_statement("events_v2")}+ UPDATE scroll_depth = 255+ WHERE name = 'pageleave' AND scroll_depth = 0+ """,+ [],+ timeout: 60_000+ )+ end+enddiff --git a/lib/plausible/ingestion/request.ex b/lib/plausible/ingestion/request.exindex 1cc18dd358..a04b682304 100644--- a/lib/plausible/ingestion/request.ex+++ b/lib/plausible/ingestion/request.ex@@ -255,7 +255,7 @@ defmodule Plausible.Ingestion.Request docase request_body["sd"] dosd when is_integer(sd) and sd >= 0 and sd <= 100 -> sdsd when is_integer(sd) and sd > 100 -> 100- _ -> 0+ _ -> 255endChangeset.put_change(changeset, :scroll_depth, scroll_depth)diff --git a/lib/plausible/stats/query_runner.ex b/lib/plausible/stats/query_runner.exindex 98e6016b73..fadc683466 100644--- a/lib/plausible/stats/query_runner.ex+++ b/lib/plausible/stats/query_runner.ex@@ -271,6 +271,7 @@ defmodule Plausible.Stats.QueryRunner dodo: nilend+ defp empty_metric_value(:scroll_depth), do: nildefp empty_metric_value(_), do: 0defp total_rows([]), do: 0diff --git a/lib/plausible/stats/sql/special_metrics.ex b/lib/plausible/stats/sql/special_metrics.exindex 80fcd0150a..b4a615a1cc 100644--- a/lib/plausible/stats/sql/special_metrics.ex+++ b/lib/plausible/stats/sql/special_metrics.ex@@ -131,7 +131,7 @@ defmodule Plausible.Stats.SQL.SpecialMetrics doif :scroll_depth in query.metrics domax_per_visitor_q =Base.base_event_query(site, query)- |> where([e], e.name == "pageleave")+ |> where([e], e.name == "pageleave" and e.scroll_depth <= 100)|> select([e], %{user_id: e.user_id,max_scroll_depth: max(e.scroll_depth)@@ -153,7 +153,12 @@ defmodule Plausible.Stats.SQL.SpecialMetrics doscroll_depth_q =subquery(max_per_visitor_q)|> select([p], %{- scroll_depth: fragment("toUInt8(round(ifNotFinite(avg(?), 0)))", p.max_scroll_depth)+ scroll_depth:+ fragment(+ "if(isFinite(avg(?)), toUInt8(round(avg(?))), NULL)",+ p.max_scroll_depth,+ p.max_scroll_depth+ )})|> select_merge(^dim_select)|> group_by(^dim_group_by)diff --git a/test/plausible_web/controllers/api/external_controller_test.exs b/test/plausible_web/controllers/api/external_controller_test.exsindex b7679bab76..868adb31aa 100644--- a/test/plausible_web/controllers/api/external_controller_test.exs+++ b/test/plausible_web/controllers/api/external_controller_test.exs@@ -1260,12 +1260,13 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do{:ok, site: site}end- test "ingests scroll_depth as 0 when sd not in params", %{conn: conn, site: site} do+ test "ingests scroll_depth as 255 when sd not in params", %{conn: conn, site: site} dopost(conn, "/api/event", %{n: "pageview", u: "https://test.com", d: site.domain})post(conn, "/api/event", %{n: "pageleave", u: "https://test.com", d: site.domain})- post(conn, "/api/event", %{n: "custom", u: "https://test.com", d: site.domain})- assert [%{scroll_depth: 0}, %{scroll_depth: 0}, %{scroll_depth: 0}] = get_events(site)+ pageleave = get_events(site) |> Enum.find(&(&1.name == "pageleave"))++ assert pageleave.scroll_depth == 255endtest "sd field is ignored if name is not pageleave", %{conn: conn, site: site} do@@ -1293,22 +1294,22 @@ defmodule PlausibleWeb.Api.ExternalControllerTest doassert pageleave.scroll_depth == 100end- test "ingests scroll_depth as 0 when sd is a string", %{conn: conn, site: site} do+ test "ingests scroll_depth as 255 when sd is a string", %{conn: conn, site: site} dopost(conn, "/api/event", %{n: "pageview", u: "https://test.com", d: site.domain})post(conn, "/api/event", %{n: "pageleave", u: "https://test.com", d: site.domain, sd: "1"})pageleave = get_events(site) |> Enum.find(&(&1.name == "pageleave"))- assert pageleave.scroll_depth == 0+ assert pageleave.scroll_depth == 255end- test "ingests scroll_depth as 0 when sd is a negative integer", %{conn: conn, site: site} do+ test "ingests scroll_depth as 255 when sd is a negative integer", %{conn: conn, site: site} dopost(conn, "/api/event", %{n: "pageview", u: "https://test.com", d: site.domain})post(conn, "/api/event", %{n: "pageleave", u: "https://test.com", d: site.domain, sd: -1})pageleave = get_events(site) |> Enum.find(&(&1.name == "pageleave"))- assert pageleave.scroll_depth == 0+ assert pageleave.scroll_depth == 255endenddiff --git a/test/plausible_web/controllers/api/external_stats_controller/query_test.exs b/test/plausible_web/controllers/api/external_stats_controller/query_test.exsindex 1352b29a54..5100a9c311 100644--- a/test/plausible_web/controllers/api/external_stats_controller/query_test.exs+++ b/test/plausible_web/controllers/api/external_stats_controller/query_test.exs@@ -3773,7 +3773,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest do})assert json_response(conn, 200)["results"] == [- %{"metrics" => [1, 0], "dimensions" => []}+ %{"metrics" => [1, nil], "dimensions" => []}]end@@ -3787,7 +3787,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest do})assert json_response(conn, 200)["results"] == [- %{"metrics" => [0], "dimensions" => []}+ %{"metrics" => [nil], "dimensions" => []}]enddiff --git a/test/plausible_web/controllers/api/stats_controller/pages_test.exs b/test/plausible_web/controllers/api/stats_controller/pages_test.exsindex 81980fe26d..9661f1edd9 100644--- a/test/plausible_web/controllers/api/stats_controller/pages_test.exs+++ b/test/plausible_web/controllers/api/stats_controller/pages_test.exs@@ -278,7 +278,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do"pageviews" => 2,"bounce_rate" => 0,"time_on_page" => 600,- "scroll_depth" => 0+ "scroll_depth" => nil},%{"name" => "/blog/john-1",@@ -286,7 +286,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do"pageviews" => 1,"bounce_rate" => 0,"time_on_page" => 60,- "scroll_depth" => 0+ "scroll_depth" => nil}]end@@ -342,7 +342,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do"pageviews" => 2,"bounce_rate" => 0,"time_on_page" => 120.0,- "scroll_depth" => 0+ "scroll_depth" => nil},%{"name" => "/blog/other-post",@@ -350,7 +350,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do"pageviews" => 1,"bounce_rate" => 0,"time_on_page" => nil,- "scroll_depth" => 0+ "scroll_depth" => nil}]end@@ -396,7 +396,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do"pageviews" => 2,"bounce_rate" => 50,"time_on_page" => 60,- "scroll_depth" => 0+ "scroll_depth" => nil},%{"name" => "/blog/other-post",@@ -404,7 +404,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do"pageviews" => 1,"bounce_rate" => 0,"time_on_page" => nil,- "scroll_depth" => 0+ "scroll_depth" => nil}]end@@ -454,7 +454,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do"pageviews" => 2,"bounce_rate" => 100,"time_on_page" => nil,- "scroll_depth" => 0+ "scroll_depth" => nil},%{"name" => "/blog/john-1",@@ -462,7 +462,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do"pageviews" => 1,"bounce_rate" => 0,"time_on_page" => 60,- "scroll_depth" => 0+ "scroll_depth" => nil}]end@@ -598,7 +598,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do"pageviews" => 3,"bounce_rate" => 50,"time_on_page" => 60,- "scroll_depth" => 0+ "scroll_depth" => nil}]end@@ -691,7 +691,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do"pageviews" => 3,"bounce_rate" => 50,"time_on_page" => 60,- "scroll_depth" => 0+ "scroll_depth" => nil},%{"name" => "/about",@@ -699,7 +699,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do"pageviews" => 1,"bounce_rate" => 100,"time_on_page" => nil,- "scroll_depth" => 0+ "scroll_depth" => nil}]end@@ -747,7 +747,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do"pageviews" => 3,"bounce_rate" => 50,"time_on_page" => 60,- "scroll_depth" => 0+ "scroll_depth" => nil}]end@@ -794,7 +794,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do"pageviews" => 2,"bounce_rate" => 100,"time_on_page" => nil,- "scroll_depth" => 0+ "scroll_depth" => nil},%{"name" => "/blog/post-1",@@ -802,7 +802,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do"pageviews" => 1,"bounce_rate" => 0,"time_on_page" => 60,- "scroll_depth" => 0+ "scroll_depth" => nil},%{"name" => "/blog/post-2",@@ -810,7 +810,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do"pageviews" => 1,"bounce_rate" => 0,"time_on_page" => nil,- "scroll_depth" => 0+ "scroll_depth" => nil}]end@@ -849,7 +849,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do"pageviews" => 1,"bounce_rate" => 0,"time_on_page" => 60,- "scroll_depth" => 0+ "scroll_depth" => nil},%{"name" => "/blog/(/post-2",@@ -857,7 +857,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do"pageviews" => 1,"bounce_rate" => 0,"time_on_page" => nil,- "scroll_depth" => 0+ "scroll_depth" => nil}]end@@ -904,7 +904,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do"pageviews" => 2,"bounce_rate" => 50,"time_on_page" => 600,- "scroll_depth" => 0+ "scroll_depth" => nil},%{"name" => "/about",@@ -912,7 +912,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do"pageviews" => 1,"bounce_rate" => 0,"time_on_page" => nil,- "scroll_depth" => 0+ "scroll_depth" => nil}]end@@ -1007,7 +1007,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do"visitors" => 2,"pageviews" => 2,"name" => "/",- "scroll_depth" => 0+ "scroll_depth" => nil},%{"bounce_rate" => 0,@@ -1015,7 +1015,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do"visitors" => 1,"pageviews" => 1,"name" => "/some-other-page",- "scroll_depth" => 0+ "scroll_depth" => nil}]end@@ -1056,7 +1056,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do"pageviews" => 2,"time_on_page" => nil,"visitors" => 2,- "scroll_depth" => 0+ "scroll_depth" => nil}]end@@ -1136,7 +1136,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do"pageviews" => 3,"time_on_page" => 1140.0,"visitors" => 2,- "scroll_depth" => 0+ "scroll_depth" => nil},%{"bounce_rate" => 0,@@ -1144,7 +1144,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do"pageviews" => 1,"time_on_page" => nil,"visitors" => 1,- "scroll_depth" => 0+ "scroll_depth" => nil}]end@@ -1492,20 +1492,20 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do"pageviews" => 0,"time_on_page" => 0,"visitors" => 0,- "scroll_depth" => 0,+ "scroll_depth" => nil,"change" => %{"bounce_rate" => nil,"pageviews" => 100,"time_on_page" => nil,"visitors" => 100,- "scroll_depth" => 0+ "scroll_depth" => nil}},"name" => "/page2","pageviews" => 2,"time_on_page" => nil,"visitors" => 2,- "scroll_depth" => 0+ "scroll_depth" => nil},%{"bounce_rate" => 100,@@ -1513,19 +1513,19 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do"pageviews" => 1,"time_on_page" => nil,"visitors" => 1,- "scroll_depth" => 0,+ "scroll_depth" => nil,"comparison" => %{"bounce_rate" => 100,"pageviews" => 1,"time_on_page" => nil,"visitors" => 1,- "scroll_depth" => 0,+ "scroll_depth" => nil,"change" => %{"bounce_rate" => 0,"pageviews" => 0,"time_on_page" => nil,"visitors" => 0,- "scroll_depth" => 0+ "scroll_depth" => nil}}}