Skip to content

Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.

Average Scroll Depth Metric: handle missing data better (#4889)

RobertJoonasd21d48558a5e
diff --git a/lib/plausible/data_migration/missing_scroll_depth.ex b/lib/plausible/data_migration/missing_scroll_depth.ex
new file mode 100644
index 0000000000..533287a1dd
--- /dev/null
+++ b/lib/plausible/data_migration/missing_scroll_depth.ex
@@ -0,0 +1,18 @@
+defmodule Plausible.DataMigration.MissingScrollDepth do
+ @moduledoc """
+ Set scroll_depth to 255 (max UInt8) for all pageleave events where it's 0.
+ """
+
+ def run() do
+ Plausible.IngestRepo.query!(
+ """
+ ALTER TABLE events_v2
+ #{Plausible.MigrationUtils.on_cluster_statement("events_v2")}
+ UPDATE scroll_depth = 255
+ WHERE name = 'pageleave' AND scroll_depth = 0
+ """,
+ [],
+ timeout: 60_000
+ )
+ end
+end
diff --git a/lib/plausible/ingestion/request.ex b/lib/plausible/ingestion/request.ex
index 1cc18dd358..a04b682304 100644
--- a/lib/plausible/ingestion/request.ex
+++ b/lib/plausible/ingestion/request.ex
@@ -255,7 +255,7 @@ defmodule Plausible.Ingestion.Request do
case request_body["sd"] do
sd when is_integer(sd) and sd >= 0 and sd <= 100 -> sd
sd when is_integer(sd) and sd > 100 -> 100
- _ -> 0
+ _ -> 255
end
Changeset.put_change(changeset, :scroll_depth, scroll_depth)
diff --git a/lib/plausible/stats/query_runner.ex b/lib/plausible/stats/query_runner.ex
index 98e6016b73..fadc683466 100644
--- a/lib/plausible/stats/query_runner.ex
+++ b/lib/plausible/stats/query_runner.ex
@@ -271,6 +271,7 @@ defmodule Plausible.Stats.QueryRunner do
do: nil
end
+ defp empty_metric_value(:scroll_depth), do: nil
defp empty_metric_value(_), do: 0
defp total_rows([]), do: 0
diff --git a/lib/plausible/stats/sql/special_metrics.ex b/lib/plausible/stats/sql/special_metrics.ex
index 80fcd0150a..b4a615a1cc 100644
--- a/lib/plausible/stats/sql/special_metrics.ex
+++ b/lib/plausible/stats/sql/special_metrics.ex
@@ -131,7 +131,7 @@ defmodule Plausible.Stats.SQL.SpecialMetrics do
if :scroll_depth in query.metrics do
max_per_visitor_q =
Base.base_event_query(site, query)
- |> where([e], e.name == "pageleave")
+ |> where([e], e.name == "pageleave" and e.scroll_depth <= 100)
|> select([e], %{
user_id: e.user_id,
max_scroll_depth: max(e.scroll_depth)
@@ -153,7 +153,12 @@ defmodule Plausible.Stats.SQL.SpecialMetrics do
scroll_depth_q =
subquery(max_per_visitor_q)
|> select([p], %{
- scroll_depth: fragment("toUInt8(round(ifNotFinite(avg(?), 0)))", p.max_scroll_depth)
+ scroll_depth:
+ fragment(
+ "if(isFinite(avg(?)), toUInt8(round(avg(?))), NULL)",
+ p.max_scroll_depth,
+ p.max_scroll_depth
+ )
})
|> select_merge(^dim_select)
|> group_by(^dim_group_by)
diff --git a/test/plausible_web/controllers/api/external_controller_test.exs b/test/plausible_web/controllers/api/external_controller_test.exs
index b7679bab76..868adb31aa 100644
--- a/test/plausible_web/controllers/api/external_controller_test.exs
+++ b/test/plausible_web/controllers/api/external_controller_test.exs
@@ -1260,12 +1260,13 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
{:ok, site: site}
end
- test "ingests scroll_depth as 0 when sd not in params", %{conn: conn, site: site} do
+ test "ingests scroll_depth as 255 when sd not in params", %{conn: conn, site: site} do
post(conn, "/api/event", %{n: "pageview", u: "https://test.com", d: site.domain})
post(conn, "/api/event", %{n: "pageleave", u: "https://test.com", d: site.domain})
- post(conn, "/api/event", %{n: "custom", u: "https://test.com", d: site.domain})
- assert [%{scroll_depth: 0}, %{scroll_depth: 0}, %{scroll_depth: 0}] = get_events(site)
+ pageleave = get_events(site) |> Enum.find(&(&1.name == "pageleave"))
+
+ assert pageleave.scroll_depth == 255
end
test "sd field is ignored if name is not pageleave", %{conn: conn, site: site} do
@@ -1293,22 +1294,22 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
assert pageleave.scroll_depth == 100
end
- test "ingests scroll_depth as 0 when sd is a string", %{conn: conn, site: site} do
+ test "ingests scroll_depth as 255 when sd is a string", %{conn: conn, site: site} do
post(conn, "/api/event", %{n: "pageview", u: "https://test.com", d: site.domain})
post(conn, "/api/event", %{n: "pageleave", u: "https://test.com", d: site.domain, sd: "1"})
pageleave = get_events(site) |> Enum.find(&(&1.name == "pageleave"))
- assert pageleave.scroll_depth == 0
+ assert pageleave.scroll_depth == 255
end
- test "ingests scroll_depth as 0 when sd is a negative integer", %{conn: conn, site: site} do
+ test "ingests scroll_depth as 255 when sd is a negative integer", %{conn: conn, site: site} do
post(conn, "/api/event", %{n: "pageview", u: "https://test.com", d: site.domain})
post(conn, "/api/event", %{n: "pageleave", u: "https://test.com", d: site.domain, sd: -1})
pageleave = get_events(site) |> Enum.find(&(&1.name == "pageleave"))
- assert pageleave.scroll_depth == 0
+ assert pageleave.scroll_depth == 255
end
end
diff --git a/test/plausible_web/controllers/api/external_stats_controller/query_test.exs b/test/plausible_web/controllers/api/external_stats_controller/query_test.exs
index 1352b29a54..5100a9c311 100644
--- a/test/plausible_web/controllers/api/external_stats_controller/query_test.exs
+++ b/test/plausible_web/controllers/api/external_stats_controller/query_test.exs
@@ -3773,7 +3773,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest do
})
assert json_response(conn, 200)["results"] == [
- %{"metrics" => [1, 0], "dimensions" => []}
+ %{"metrics" => [1, nil], "dimensions" => []}
]
end
@@ -3787,7 +3787,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest do
})
assert json_response(conn, 200)["results"] == [
- %{"metrics" => [0], "dimensions" => []}
+ %{"metrics" => [nil], "dimensions" => []}
]
end
diff --git a/test/plausible_web/controllers/api/stats_controller/pages_test.exs b/test/plausible_web/controllers/api/stats_controller/pages_test.exs
index 81980fe26d..9661f1edd9 100644
--- a/test/plausible_web/controllers/api/stats_controller/pages_test.exs
+++ b/test/plausible_web/controllers/api/stats_controller/pages_test.exs
@@ -278,7 +278,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
"pageviews" => 2,
"bounce_rate" => 0,
"time_on_page" => 600,
- "scroll_depth" => 0
+ "scroll_depth" => nil
},
%{
"name" => "/blog/john-1",
@@ -286,7 +286,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
"pageviews" => 1,
"bounce_rate" => 0,
"time_on_page" => 60,
- "scroll_depth" => 0
+ "scroll_depth" => nil
}
]
end
@@ -342,7 +342,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
"pageviews" => 2,
"bounce_rate" => 0,
"time_on_page" => 120.0,
- "scroll_depth" => 0
+ "scroll_depth" => nil
},
%{
"name" => "/blog/other-post",
@@ -350,7 +350,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
"pageviews" => 1,
"bounce_rate" => 0,
"time_on_page" => nil,
- "scroll_depth" => 0
+ "scroll_depth" => nil
}
]
end
@@ -396,7 +396,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
"pageviews" => 2,
"bounce_rate" => 50,
"time_on_page" => 60,
- "scroll_depth" => 0
+ "scroll_depth" => nil
},
%{
"name" => "/blog/other-post",
@@ -404,7 +404,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
"pageviews" => 1,
"bounce_rate" => 0,
"time_on_page" => nil,
- "scroll_depth" => 0
+ "scroll_depth" => nil
}
]
end
@@ -454,7 +454,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
"pageviews" => 2,
"bounce_rate" => 100,
"time_on_page" => nil,
- "scroll_depth" => 0
+ "scroll_depth" => nil
},
%{
"name" => "/blog/john-1",
@@ -462,7 +462,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
"pageviews" => 1,
"bounce_rate" => 0,
"time_on_page" => 60,
- "scroll_depth" => 0
+ "scroll_depth" => nil
}
]
end
@@ -598,7 +598,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
"pageviews" => 3,
"bounce_rate" => 50,
"time_on_page" => 60,
- "scroll_depth" => 0
+ "scroll_depth" => nil
}
]
end
@@ -691,7 +691,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
"pageviews" => 3,
"bounce_rate" => 50,
"time_on_page" => 60,
- "scroll_depth" => 0
+ "scroll_depth" => nil
},
%{
"name" => "/about",
@@ -699,7 +699,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
"pageviews" => 1,
"bounce_rate" => 100,
"time_on_page" => nil,
- "scroll_depth" => 0
+ "scroll_depth" => nil
}
]
end
@@ -747,7 +747,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
"pageviews" => 3,
"bounce_rate" => 50,
"time_on_page" => 60,
- "scroll_depth" => 0
+ "scroll_depth" => nil
}
]
end
@@ -794,7 +794,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
"pageviews" => 2,
"bounce_rate" => 100,
"time_on_page" => nil,
- "scroll_depth" => 0
+ "scroll_depth" => nil
},
%{
"name" => "/blog/post-1",
@@ -802,7 +802,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
"pageviews" => 1,
"bounce_rate" => 0,
"time_on_page" => 60,
- "scroll_depth" => 0
+ "scroll_depth" => nil
},
%{
"name" => "/blog/post-2",
@@ -810,7 +810,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
"pageviews" => 1,
"bounce_rate" => 0,
"time_on_page" => nil,
- "scroll_depth" => 0
+ "scroll_depth" => nil
}
]
end
@@ -849,7 +849,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
"pageviews" => 1,
"bounce_rate" => 0,
"time_on_page" => 60,
- "scroll_depth" => 0
+ "scroll_depth" => nil
},
%{
"name" => "/blog/(/post-2",
@@ -857,7 +857,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
"pageviews" => 1,
"bounce_rate" => 0,
"time_on_page" => nil,
- "scroll_depth" => 0
+ "scroll_depth" => nil
}
]
end
@@ -904,7 +904,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
"pageviews" => 2,
"bounce_rate" => 50,
"time_on_page" => 600,
- "scroll_depth" => 0
+ "scroll_depth" => nil
},
%{
"name" => "/about",
@@ -912,7 +912,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
"pageviews" => 1,
"bounce_rate" => 0,
"time_on_page" => nil,
- "scroll_depth" => 0
+ "scroll_depth" => nil
}
]
end
@@ -1007,7 +1007,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
"visitors" => 2,
"pageviews" => 2,
"name" => "/",
- "scroll_depth" => 0
+ "scroll_depth" => nil
},
%{
"bounce_rate" => 0,
@@ -1015,7 +1015,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
"visitors" => 1,
"pageviews" => 1,
"name" => "/some-other-page",
- "scroll_depth" => 0
+ "scroll_depth" => nil
}
]
end
@@ -1056,7 +1056,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
"pageviews" => 2,
"time_on_page" => nil,
"visitors" => 2,
- "scroll_depth" => 0
+ "scroll_depth" => nil
}
]
end
@@ -1136,7 +1136,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
"pageviews" => 3,
"time_on_page" => 1140.0,
"visitors" => 2,
- "scroll_depth" => 0
+ "scroll_depth" => nil
},
%{
"bounce_rate" => 0,
@@ -1144,7 +1144,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
"pageviews" => 1,
"time_on_page" => nil,
"visitors" => 1,
- "scroll_depth" => 0
+ "scroll_depth" => nil
}
]
end
@@ -1492,20 +1492,20 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
"pageviews" => 0,
"time_on_page" => 0,
"visitors" => 0,
- "scroll_depth" => 0,
+ "scroll_depth" => nil,
"change" => %{
"bounce_rate" => nil,
"pageviews" => 100,
"time_on_page" => nil,
"visitors" => 100,
- "scroll_depth" => 0
+ "scroll_depth" => nil
}
},
"name" => "/page2",
"pageviews" => 2,
"time_on_page" => nil,
"visitors" => 2,
- "scroll_depth" => 0
+ "scroll_depth" => nil
},
%{
"bounce_rate" => 100,
@@ -1513,19 +1513,19 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
"pageviews" => 1,
"time_on_page" => nil,
"visitors" => 1,
- "scroll_depth" => 0,
+ "scroll_depth" => nil,
"comparison" => %{
"bounce_rate" => 100,
"pageviews" => 1,
"time_on_page" => nil,
"visitors" => 1,
- "scroll_depth" => 0,
+ "scroll_depth" => nil,
"change" => %{
"bounce_rate" => 0,
"pageviews" => 0,
"time_on_page" => nil,
"visitors" => 0,
- "scroll_depth" => 0
+ "scroll_depth" => nil
}
}
}