Skip to content

Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.

[FF] Average Scroll Depth Metric: dashboard CSV export (#4898)

RobertJoonas7f3e91e1cc13
diff --git a/lib/plausible/stats/timeseries.ex b/lib/plausible/stats/timeseries.ex
index 1b45c95485..5de0d3c93d 100644
--- a/lib/plausible/stats/timeseries.ex
+++ b/lib/plausible/stats/timeseries.ex
@@ -87,7 +87,7 @@ defmodule Plausible.Stats.Timeseries do
:views_per_visit -> Map.merge(row, %{views_per_visit: 0.0})
:conversion_rate -> Map.merge(row, %{conversion_rate: 0.0})
:group_conversion_rate -> Map.merge(row, %{group_conversion_rate: 0.0})
- :scroll_depth -> Map.merge(row, %{scroll_depth: 0})
+ :scroll_depth -> Map.merge(row, %{scroll_depth: nil})
:bounce_rate -> Map.merge(row, %{bounce_rate: 0.0})
:visit_duration -> Map.merge(row, %{visit_duration: nil})
:average_revenue -> Map.merge(row, %{average_revenue: nil})
diff --git a/lib/plausible_web/controllers/api/stats_controller.ex b/lib/plausible_web/controllers/api/stats_controller.ex
index 5fa2372adb..a4261e2adf 100644
--- a/lib/plausible_web/controllers/api/stats_controller.ex
+++ b/lib/plausible_web/controllers/api/stats_controller.ex
@@ -831,9 +831,11 @@ defmodule PlausibleWeb.Api.StatsController do
params = Map.put(params, "property", "event:page")
query = Query.from(site, params, debug_metadata(conn))
+ include_scroll_depth? = !query.include_imported && scroll_depth_enabled?(site, current_user)
+
extra_metrics =
cond do
- params["detailed"] && !query.include_imported && scroll_depth_enabled?(site, current_user) ->
+ params["detailed"] && include_scroll_depth? ->
[:pageviews, :bounce_rate, :time_on_page, :scroll_depth]
params["detailed"] ->
@@ -856,7 +858,12 @@ defmodule PlausibleWeb.Api.StatsController do
|> transform_keys(%{visitors: :conversions})
|> to_csv([:name, :conversions, :conversion_rate])
else
- pages |> to_csv([:name, :visitors, :pageviews, :bounce_rate, :time_on_page])
+ cols = [:name, :visitors, :pageviews, :bounce_rate, :time_on_page]
+
+ # credo:disable-for-next-line Credo.Check.Refactor.Nesting
+ cols = if include_scroll_depth?, do: cols ++ [:scroll_depth], else: cols
+
+ pages |> to_csv(cols)
end
else
json(conn, %{
@@ -1614,7 +1621,7 @@ defmodule PlausibleWeb.Api.StatsController do
defp realtime_period_to_30m(params), do: params
- defp scroll_depth_enabled?(site, user) do
+ def scroll_depth_enabled?(site, user) do
FunWithFlags.enabled?(:scroll_depth, for: user) ||
FunWithFlags.enabled?(:scroll_depth, for: site)
end
diff --git a/lib/plausible_web/controllers/stats_controller.ex b/lib/plausible_web/controllers/stats_controller.ex
index d194107768..686567336f 100644
--- a/lib/plausible_web/controllers/stats_controller.ex
+++ b/lib/plausible_web/controllers/stats_controller.ex
@@ -119,7 +119,7 @@ defmodule PlausibleWeb.StatsController do
limited_params = Map.merge(params, %{"limit" => "100"})
csvs = %{
- ~c"visitors.csv" => fn -> main_graph_csv(site, query) end,
+ ~c"visitors.csv" => fn -> main_graph_csv(site, query, conn.assigns[:current_user]) end,
~c"sources.csv" => fn -> Api.StatsController.sources(conn, params) end,
~c"utm_mediums.csv" => fn -> Api.StatsController.utm_mediums(conn, params) end,
~c"utm_sources.csv" => fn -> Api.StatsController.utm_sources(conn, params) end,
@@ -177,8 +177,8 @@ defmodule PlausibleWeb.StatsController do
end
end
- defp main_graph_csv(site, query) do
- {metrics, column_headers} = csv_graph_metrics(query)
+ defp main_graph_csv(site, query, current_user) do
+ {metrics, column_headers} = csv_graph_metrics(query, site, current_user)
map_bucket_to_row = fn bucket -> Enum.map([:date | metrics], &bucket[&1]) end
prepend_column_headers = fn data -> [column_headers | data] end
@@ -190,7 +190,12 @@ defmodule PlausibleWeb.StatsController do
|> NimbleCSV.RFC4180.dump_to_iodata()
end
- defp csv_graph_metrics(query) do
+ defp csv_graph_metrics(query, site, current_user) do
+ include_scroll_depth? =
+ !query.include_imported &&
+ PlausibleWeb.Api.StatsController.scroll_depth_enabled?(site, current_user) &&
+ Filters.filtering_on_dimension?(query, "event:page")
+
{metrics, column_headers} =
if Filters.filtering_on_dimension?(query, "event:goal") do
{
@@ -207,6 +212,8 @@ defmodule PlausibleWeb.StatsController do
:visit_duration
]
+ metrics = if include_scroll_depth?, do: metrics ++ [:scroll_depth], else: metrics
+
{
metrics,
[:date | metrics]
diff --git a/test/plausible_web/controllers/CSVs/30d-filter-path/pages.csv b/test/plausible_web/controllers/CSVs/30d-filter-path/pages.csv
index 4741db70e8..ae0e05a64a 100644
--- a/test/plausible_web/controllers/CSVs/30d-filter-path/pages.csv
+++ b/test/plausible_web/controllers/CSVs/30d-filter-path/pages.csv
@@ -1,2 +1,2 @@
-name,visitors,pageviews,bounce_rate,time_on_page
-/some-other-page,1,1,0,60.0
+name,visitors,pageviews,bounce_rate,time_on_page,scroll_depth
+/some-other-page,1,1,0,60.0,
diff --git a/test/plausible_web/controllers/CSVs/30d-filter-path/visitors.csv b/test/plausible_web/controllers/CSVs/30d-filter-path/visitors.csv
index fa82414a60..2ac3de20cc 100644
--- a/test/plausible_web/controllers/CSVs/30d-filter-path/visitors.csv
+++ b/test/plausible_web/controllers/CSVs/30d-filter-path/visitors.csv
@@ -1,32 +1,32 @@
-date,visitors,pageviews,visits,views_per_visit,bounce_rate,visit_duration
-2021-09-20,0,0,0,0.0,0.0,
-2021-09-21,0,0,0,0.0,0.0,
-2021-09-22,0,0,0,0.0,0.0,
-2021-09-23,0,0,0,0.0,0.0,
-2021-09-24,0,0,0,0.0,0.0,
-2021-09-25,0,0,0,0.0,0.0,
-2021-09-26,0,0,0,0.0,0.0,
-2021-09-27,0,0,0,0.0,0.0,
-2021-09-28,0,0,0,0.0,0.0,
-2021-09-29,0,0,0,0.0,0.0,
-2021-09-30,0,0,0,0.0,0.0,
-2021-10-01,0,0,0,0.0,0.0,
-2021-10-02,0,0,0,0.0,0.0,
-2021-10-03,0,0,0,0.0,0.0,
-2021-10-04,0,0,0,0.0,0.0,
-2021-10-05,0,0,0,0.0,0.0,
-2021-10-06,0,0,0,0.0,0.0,
-2021-10-07,0,0,0,0.0,0.0,
-2021-10-08,0,0,0,0.0,0.0,
-2021-10-09,0,0,0,0.0,0.0,
-2021-10-10,0,0,0,0.0,0.0,
-2021-10-11,0,0,0,0.0,0.0,
-2021-10-12,0,0,0,0.0,0.0,
-2021-10-13,0,0,0,0.0,0.0,
-2021-10-14,0,0,0,0.0,0.0,
-2021-10-15,0,0,0,0.0,0.0,
-2021-10-16,0,0,0,0.0,0.0,
-2021-10-17,0,0,0,0.0,0.0,
-2021-10-18,0,0,0,0.0,0.0,
-2021-10-19,0,0,0,0.0,0.0,
-2021-10-20,1,1,1,2.0,0,60
+date,visitors,pageviews,visits,views_per_visit,bounce_rate,visit_duration,scroll_depth
+2021-09-20,0,0,0,0.0,0.0,,
+2021-09-21,0,0,0,0.0,0.0,,
+2021-09-22,0,0,0,0.0,0.0,,
+2021-09-23,0,0,0,0.0,0.0,,
+2021-09-24,0,0,0,0.0,0.0,,
+2021-09-25,0,0,0,0.0,0.0,,
+2021-09-26,0,0,0,0.0,0.0,,
+2021-09-27,0,0,0,0.0,0.0,,
+2021-09-28,0,0,0,0.0,0.0,,
+2021-09-29,0,0,0,0.0,0.0,,
+2021-09-30,0,0,0,0.0,0.0,,
+2021-10-01,0,0,0,0.0,0.0,,
+2021-10-02,0,0,0,0.0,0.0,,
+2021-10-03,0,0,0,0.0,0.0,,
+2021-10-04,0,0,0,0.0,0.0,,
+2021-10-05,0,0,0,0.0,0.0,,
+2021-10-06,0,0,0,0.0,0.0,,
+2021-10-07,0,0,0,0.0,0.0,,
+2021-10-08,0,0,0,0.0,0.0,,
+2021-10-09,0,0,0,0.0,0.0,,
+2021-10-10,0,0,0,0.0,0.0,,
+2021-10-11,0,0,0,0.0,0.0,,
+2021-10-12,0,0,0,0.0,0.0,,
+2021-10-13,0,0,0,0.0,0.0,,
+2021-10-14,0,0,0,0.0,0.0,,
+2021-10-15,0,0,0,0.0,0.0,,
+2021-10-16,0,0,0,0.0,0.0,,
+2021-10-17,0,0,0,0.0,0.0,,
+2021-10-18,0,0,0,0.0,0.0,,
+2021-10-19,0,0,0,0.0,0.0,,
+2021-10-20,1,1,1,2.0,0,60,
diff --git a/test/plausible_web/controllers/CSVs/30d/pages.csv b/test/plausible_web/controllers/CSVs/30d/pages.csv
index 62b246c420..c04e0e7d7c 100644
--- a/test/plausible_web/controllers/CSVs/30d/pages.csv
+++ b/test/plausible_web/controllers/CSVs/30d/pages.csv
@@ -1,4 +1,4 @@
-name,visitors,pageviews,bounce_rate,time_on_page
-/,4,3,67,
-/signup,1,1,0,60.0
-/some-other-page,1,1,0,60.0
+name,visitors,pageviews,bounce_rate,time_on_page,scroll_depth
+/,4,3,67,,
+/signup,1,1,0,60.0,
+/some-other-page,1,1,0,60.0,
diff --git a/test/plausible_web/controllers/CSVs/6m/pages.csv b/test/plausible_web/controllers/CSVs/6m/pages.csv
index ed895e036f..294a95100a 100644
--- a/test/plausible_web/controllers/CSVs/6m/pages.csv
+++ b/test/plausible_web/controllers/CSVs/6m/pages.csv
@@ -1,4 +1,4 @@
-name,visitors,pageviews,bounce_rate,time_on_page
-/,5,4,75,
-/signup,1,1,0,60.0
-/some-other-page,1,1,0,60.0
+name,visitors,pageviews,bounce_rate,time_on_page,scroll_depth
+/,5,4,75,,
+/signup,1,1,0,60.0,
+/some-other-page,1,1,0,60.0,
diff --git a/test/plausible_web/controllers/stats_controller_test.exs b/test/plausible_web/controllers/stats_controller_test.exs
index 0aa89f9bc6..586cb5b6b6 100644
--- a/test/plausible_web/controllers/stats_controller_test.exs
+++ b/test/plausible_web/controllers/stats_controller_test.exs
@@ -217,6 +217,39 @@ defmodule PlausibleWeb.StatsControllerTest do
assert ~c"utm_terms.csv" in zip
end
+ test "exports scroll depth metric in pages.csv", %{conn: conn, site: site} do
+ t0 = ~N[2020-01-01 00:00:00]
+ [t1, t2, t3] = for i <- 1..3, do: NaiveDateTime.add(t0, i, :minute)
+
+ populate_stats(site, [
+ build(:pageview, user_id: 12, pathname: "/blog", timestamp: t0),
+ build(:pageleave, user_id: 12, pathname: "/blog", timestamp: t1, scroll_depth: 20),
+ build(:pageview, user_id: 12, pathname: "/another", timestamp: t1),
+ build(:pageleave, user_id: 12, pathname: "/another", timestamp: t2, scroll_depth: 24),
+ build(:pageview, user_id: 34, pathname: "/blog", timestamp: t0),
+ build(:pageleave, user_id: 34, pathname: "/blog", timestamp: t1, scroll_depth: 17),
+ build(:pageview, user_id: 34, pathname: "/another", timestamp: t1),
+ build(:pageleave, user_id: 34, pathname: "/another", timestamp: t2, scroll_depth: 26),
+ build(:pageview, user_id: 34, pathname: "/blog", timestamp: t2),
+ build(:pageleave, user_id: 34, pathname: "/blog", timestamp: t3, scroll_depth: 60),
+ build(:pageview, user_id: 56, pathname: "/blog", timestamp: t0),
+ build(:pageleave, user_id: 56, pathname: "/blog", timestamp: t1, scroll_depth: 100)
+ ])
+
+ pages =
+ conn
+ |> get("/#{site.domain}/export?date=2020-01-01")
+ |> response(200)
+ |> unzip_and_parse_csv(~c"pages.csv")
+
+ assert pages == [
+ ["name", "visitors", "pageviews", "bounce_rate", "time_on_page", "scroll_depth"],
+ ["/blog", "3", "4", "33", "60.0", "60"],
+ ["/another", "2", "2", "0", "60.0", "25"],
+ [""]
+ ]
+ end
+
test "exports only internally used props in custom_props.csv for a growth plan", %{
conn: conn,
site: site
@@ -231,14 +264,13 @@ defmodule PlausibleWeb.StatsControllerTest do
build(:event, name: "File Download", "meta.key": ["url"], "meta.value": ["b"])
])
- conn = get(conn, "/" <> site.domain <> "/export?period=day")
- assert response = response(conn, 200)
- {:ok, zip} = :zip.unzip(response, [:memory])
-
- {_filename, result} =
- Enum.find(zip, fn {filename, _data} -> filename == ~c"custom_props.csv" end)
+ result =
+ conn
+ |> get("/" <> site.domain <> "/export?period=day")
+ |> response(200)
+ |> unzip_and_parse_csv(~c"custom_props.csv")
- assert parse_csv(result) == [
+ assert result == [
["property", "value", "visitors", "events", "percentage"],
["url", "(none)", "1", "1", "50.0"],
["url", "b", "1", "1", "50.0"],
@@ -298,14 +330,13 @@ defmodule PlausibleWeb.StatsControllerTest do
build(:pageview)
])
- conn = get(conn, "/" <> site.domain <> "/export?period=day")
- assert response = response(conn, 200)
- {:ok, zip} = :zip.unzip(response, [:memory])
-
- {_filename, result} =
- Enum.find(zip, fn {filename, _data} -> filename == ~c"custom_props.csv" end)
+ result =
+ conn
+ |> get("/" <> site.domain <> "/export?period=day")
+ |> response(200)
+ |> unzip_and_parse_csv(~c"custom_props.csv")
- assert parse_csv(result) == [
+ assert result == [
["property", "value", "visitors", "events", "percentage"],
["author", "(none)", "3", "4", "50.0"],
["author", "uku", "2", "2", "33.3"],
@@ -318,15 +349,14 @@ defmodule PlausibleWeb.StatsControllerTest do
test "exports data grouped by interval", %{conn: conn, site: site} do
populate_exported_stats(site)
- conn = get(conn, "/" <> site.domain <> "/export?date=2021-10-20&period=30d&interval=week")
- assert response = response(conn, 200)
- {:ok, zip} = :zip.unzip(response, [:memory])
-
- {_filename, visitors} =
- Enum.find(zip, fn {filename, _data} -> filename == ~c"visitors.csv" end)
+ visitors =
+ conn
+ |> get("/" <> site.domain <> "/export?date=2021-10-20&period=30d&interval=week")
+ |> response(200)
+ |> unzip_and_parse_csv(~c"visitors.csv")
- assert parse_csv(visitors) == [
+ assert visitors == [
[
"date",
"visitors",
@@ -361,15 +391,13 @@ defmodule PlausibleWeb.StatsControllerTest do
build(:pageview, operating_system: "Mac", operating_system_version: "13")
])
- conn = get(conn, "/#{site.domain}/export")
-
- assert response = response(conn, 200)
- {:ok, zip} = :zip.unzip(response, [:memory])
-
- {_filename, os_versions} =
- Enum.find(zip, fn {filename, _data} -> filename == ~c"operating_system_versions.csv" end)
+ os_versions =
+ conn
+ |> get("/#{site.domain}/export")
+ |> response(200)
+ |> unzip_and_parse_csv(~c"operating_system_versions.csv")
- assert parse_csv(os_versions) == [
+ assert os_versions == [
["name", "version", "visitors"],
["Mac", "14", "3"],
["Ubuntu", "20.04", "2"],
@@ -632,6 +660,61 @@ defmodule PlausibleWeb.StatsControllerTest do
conn = get(conn, "/#{site.domain}/export?date=2021-10-20&filters=#{filters}")
assert_zip(conn, "30d-filter-path")
end
+
+ test "exports scroll depth in visitors.csv", %{conn: conn, site: site} do
+ populate_stats(site, [
+ build(:pageview, user_id: 12, pathname: "/blog", timestamp: ~N[2020-01-05 00:00:00]),
+ build(:pageleave,
+ user_id: 12,
+ pathname: "/blog",
+ timestamp: ~N[2020-01-05 00:01:00],
+ scroll_depth: 40
+ ),
+ build(:pageview, user_id: 12, pathname: "/blog", timestamp: ~N[2020-01-05 10:00:00]),
+ build(:pageleave,
+ user_id: 12,
+ pathname: "/blog",
+ timestamp: ~N[2020-01-05 10:01:00],
+ scroll_depth: 17
+ ),
+ build(:pageview, user_id: 34, pathname: "/blog", timestamp: ~N[2020-01-07 00:00:00]),
+ build(:pageleave,
+ user_id: 34,
+ pathname: "/blog",
+ timestamp: ~N[2020-01-07 00:01:00],
+ scroll_depth: 90
+ )
+ ])
+
+ filters = Jason.encode!([[:is, "event:page", ["/blog"]]])
+
+ pages =
+ conn
+ |> get("/#{site.domain}/export?date=2020-01-07&period=7d&filters=#{filters}")
+ |> response(200)
+ |> unzip_and_parse_csv(~c"visitors.csv")
+
+ assert pages == [
+ [
+ "date",
+ "visitors",
+ "pageviews",
+ "visits",
+ "views_per_visit",
+ "bounce_rate",
+ "visit_duration",
+ "scroll_depth"
+ ],
+ ["2020-01-01", "0", "0", "0", "0.0", "0.0", "", ""],
+ ["2020-01-02", "0", "0", "0", "0.0", "0.0", "", ""],
+ ["2020-01-03", "0", "0", "0", "0.0", "0.0", "", ""],
+ ["2020-01-04", "0", "0", "0", "0.0", "0.0", "", ""],
+ ["2020-01-05", "1", "2", "2", "1.0", "100", "0", "40"],
+ ["2020-01-06", "0", "0", "0", "0.0", "0.0", "", ""],
+ ["2020-01-07", "1", "1", "1", "1.0", "100", "0", "90"],
+ [""]
+ ]
+ end
end
describe "GET /:domain/export - with a custom prop filter" do
@@ -650,14 +733,14 @@ defmodule PlausibleWeb.StatsControllerTest do
])
filters = Jason.encode!([[:is, "event:props:author", ["marko"]]])
- conn = get(conn, "/" <> site.domain <> "/export?period=day&filters=#{filters}")
- {:ok, zip} = :zip.unzip(response(conn, 200), [:memory])
-
- {_filename, result} =
- Enum.find(zip, fn {filename, _data} -> filename == ~c"custom_props.csv" end)
+ result =
+ conn
+ |> get("/" <> site.domain <> "/export?period=day&filters=#{filters}")
+ |> response(200)
+ |> unzip_and_parse_csv(~c"custom_props.csv")
- assert parse_csv(result) == [
+ assert result == [
["property", "value", "visitors", "events", "percentage"],
["author", "marko", "1", "1", "100.0"],
[""]
@@ -665,6 +748,12 @@ defmodule PlausibleWeb.StatsControllerTest do
end
end
+ defp unzip_and_parse_csv(archive, filename) do
+ {:ok, zip} = :zip.unzip(archive, [:memory])
+ {_filename, data} = Enum.find(zip, &(elem(&1, 0) == filename))
+ parse_csv(data)
+ end
+
defp assert_zip(conn, folder) do
assert conn.status == 200
@@ -675,10 +764,10 @@ defmodule PlausibleWeb.StatsControllerTest do
folder = Path.expand(folder, "test/plausible_web/controllers/CSVs")
- Enum.map(zip, &assert_csv(&1, folder))
+ Enum.map(zip, &assert_csv_by_fixture(&1, folder))
end
- defp assert_csv({file, downloaded}, folder) do
+ defp assert_csv_by_fixture({file, downloaded}, folder) do
file = Path.expand(file, folder)
{:ok, content} = File.read(file)
@@ -788,14 +877,14 @@ defmodule PlausibleWeb.StatsControllerTest do
insert(:goal, site: site, event_name: "Newsletter Signup")
filters = Jason.encode!([[:is, "event:goal", ["Newsletter Signup"]]])
- conn = get(conn, "/" <> site.domain <> "/export?period=day&filters=#{filters}")
- {:ok, zip} = :zip.unzip(response(conn, 200), [:memory])
-
- {_filename, result} =
- Enum.find(zip, fn {filename, _data} -> filename == ~c"custom_props.csv" end)
+ result =
+ conn
+ |> get("/" <> site.domain <> "/export?period=day&filters=#{filters}")
+ |> response(200)
+ |> unzip_and_parse_csv(~c"custom_props.csv")
- assert parse_csv(result) == [
+ assert result == [
["property", "value", "visitors", "events", "conversion_rate"],
["author", "marko", "2", "2", "50.0"],
["author", "uku", "1", "1", "25.0"],
@@ -844,15 +933,14 @@ defmodule PlausibleWeb.StatsControllerTest do
insert(:goal, site: site, event_name: "Signup")
filters = Jason.encode!([[:is, "event:goal", ["Signup"]]])
- conn = get(conn, "/#{site.domain}/export?filters=#{filters}")
-
- assert response = response(conn, 200)
- {:ok, zip} = :zip.unzip(response, [:memory])
- {_filename, os_versions} =
- Enum.find(zip, fn {filename, _data} -> filename == ~c"operating_system_versions.csv" end)
+ os_versions =
+ conn
+ |> get("/#{site.domain}/export?filters=#{filters}")
+ |> response(200)
+ |> unzip_and_parse_csv(~c"operating_system_versions.csv")
- assert parse_csv(os_versions) == [
+ assert os_versions == [
["name", "version", "conversions", "conversion_rate"],
["Mac", "14", "3", "75.0"],
["Ubuntu", "20.04", "2", "100.0"],