Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.
TypeScript analyticsanalytics-dashboardclickhouseelixirgoogle-analyticsmarketingopen-source-analyticsphoenixplausible-analyticspostgresqlprivacyprivacy-friendlyself-hostedsimple-analyticsstatisticstailwindcssweb-analyticswebsite-analyticswebsite-statswebsite-tracking
[FF] Average Scroll Depth Metric: dashboard CSV export (#4898)
RobertJoonas7f3e91e1cc13
diff --git a/lib/plausible/stats/timeseries.ex b/lib/plausible/stats/timeseries.exindex 1b45c95485..5de0d3c93d 100644--- a/lib/plausible/stats/timeseries.ex+++ b/lib/plausible/stats/timeseries.ex@@ -87,7 +87,7 @@ defmodule Plausible.Stats.Timeseries do:views_per_visit -> Map.merge(row, %{views_per_visit: 0.0}):conversion_rate -> Map.merge(row, %{conversion_rate: 0.0}):group_conversion_rate -> Map.merge(row, %{group_conversion_rate: 0.0})- :scroll_depth -> Map.merge(row, %{scroll_depth: 0})+ :scroll_depth -> Map.merge(row, %{scroll_depth: nil}):bounce_rate -> Map.merge(row, %{bounce_rate: 0.0}):visit_duration -> Map.merge(row, %{visit_duration: nil}):average_revenue -> Map.merge(row, %{average_revenue: nil})diff --git a/lib/plausible_web/controllers/api/stats_controller.ex b/lib/plausible_web/controllers/api/stats_controller.exindex 5fa2372adb..a4261e2adf 100644--- a/lib/plausible_web/controllers/api/stats_controller.ex+++ b/lib/plausible_web/controllers/api/stats_controller.ex@@ -831,9 +831,11 @@ defmodule PlausibleWeb.Api.StatsController doparams = Map.put(params, "property", "event:page")query = Query.from(site, params, debug_metadata(conn))+ include_scroll_depth? = !query.include_imported && scroll_depth_enabled?(site, current_user)+extra_metrics =cond do- params["detailed"] && !query.include_imported && scroll_depth_enabled?(site, current_user) ->+ params["detailed"] && include_scroll_depth? ->[:pageviews, :bounce_rate, :time_on_page, :scroll_depth]params["detailed"] ->@@ -856,7 +858,12 @@ defmodule PlausibleWeb.Api.StatsController do|> transform_keys(%{visitors: :conversions})|> to_csv([:name, :conversions, :conversion_rate])else- pages |> to_csv([:name, :visitors, :pageviews, :bounce_rate, :time_on_page])+ cols = [:name, :visitors, :pageviews, :bounce_rate, :time_on_page]++ # credo:disable-for-next-line Credo.Check.Refactor.Nesting+ cols = if include_scroll_depth?, do: cols ++ [:scroll_depth], else: cols++ pages |> to_csv(cols)endelsejson(conn, %{@@ -1614,7 +1621,7 @@ defmodule PlausibleWeb.Api.StatsController dodefp realtime_period_to_30m(params), do: params- defp scroll_depth_enabled?(site, user) do+ def scroll_depth_enabled?(site, user) doFunWithFlags.enabled?(:scroll_depth, for: user) ||FunWithFlags.enabled?(:scroll_depth, for: site)enddiff --git a/lib/plausible_web/controllers/stats_controller.ex b/lib/plausible_web/controllers/stats_controller.exindex d194107768..686567336f 100644--- a/lib/plausible_web/controllers/stats_controller.ex+++ b/lib/plausible_web/controllers/stats_controller.ex@@ -119,7 +119,7 @@ defmodule PlausibleWeb.StatsController dolimited_params = Map.merge(params, %{"limit" => "100"})csvs = %{- ~c"visitors.csv" => fn -> main_graph_csv(site, query) end,+ ~c"visitors.csv" => fn -> main_graph_csv(site, query, conn.assigns[:current_user]) end,~c"sources.csv" => fn -> Api.StatsController.sources(conn, params) end,~c"utm_mediums.csv" => fn -> Api.StatsController.utm_mediums(conn, params) end,~c"utm_sources.csv" => fn -> Api.StatsController.utm_sources(conn, params) end,@@ -177,8 +177,8 @@ defmodule PlausibleWeb.StatsController doendend- defp main_graph_csv(site, query) do- {metrics, column_headers} = csv_graph_metrics(query)+ defp main_graph_csv(site, query, current_user) do+ {metrics, column_headers} = csv_graph_metrics(query, site, current_user)map_bucket_to_row = fn bucket -> Enum.map([:date | metrics], &bucket[&1]) endprepend_column_headers = fn data -> [column_headers | data] end@@ -190,7 +190,12 @@ defmodule PlausibleWeb.StatsController do|> NimbleCSV.RFC4180.dump_to_iodata()end- defp csv_graph_metrics(query) do+ defp csv_graph_metrics(query, site, current_user) do+ include_scroll_depth? =+ !query.include_imported &&+ PlausibleWeb.Api.StatsController.scroll_depth_enabled?(site, current_user) &&+ Filters.filtering_on_dimension?(query, "event:page")+{metrics, column_headers} =if Filters.filtering_on_dimension?(query, "event:goal") do{@@ -207,6 +212,8 @@ defmodule PlausibleWeb.StatsController do:visit_duration]+ metrics = if include_scroll_depth?, do: metrics ++ [:scroll_depth], else: metrics+{metrics,[:date | metrics]diff --git a/test/plausible_web/controllers/CSVs/30d-filter-path/pages.csv b/test/plausible_web/controllers/CSVs/30d-filter-path/pages.csvindex 4741db70e8..ae0e05a64a 100644--- a/test/plausible_web/controllers/CSVs/30d-filter-path/pages.csv+++ b/test/plausible_web/controllers/CSVs/30d-filter-path/pages.csv@@ -1,2 +1,2 @@-name,visitors,pageviews,bounce_rate,time_on_page-/some-other-page,1,1,0,60.0+name,visitors,pageviews,bounce_rate,time_on_page,scroll_depth+/some-other-page,1,1,0,60.0,diff --git a/test/plausible_web/controllers/CSVs/30d-filter-path/visitors.csv b/test/plausible_web/controllers/CSVs/30d-filter-path/visitors.csvindex fa82414a60..2ac3de20cc 100644--- a/test/plausible_web/controllers/CSVs/30d-filter-path/visitors.csv+++ b/test/plausible_web/controllers/CSVs/30d-filter-path/visitors.csv@@ -1,32 +1,32 @@-date,visitors,pageviews,visits,views_per_visit,bounce_rate,visit_duration-2021-09-20,0,0,0,0.0,0.0,-2021-09-21,0,0,0,0.0,0.0,-2021-09-22,0,0,0,0.0,0.0,-2021-09-23,0,0,0,0.0,0.0,-2021-09-24,0,0,0,0.0,0.0,-2021-09-25,0,0,0,0.0,0.0,-2021-09-26,0,0,0,0.0,0.0,-2021-09-27,0,0,0,0.0,0.0,-2021-09-28,0,0,0,0.0,0.0,-2021-09-29,0,0,0,0.0,0.0,-2021-09-30,0,0,0,0.0,0.0,-2021-10-01,0,0,0,0.0,0.0,-2021-10-02,0,0,0,0.0,0.0,-2021-10-03,0,0,0,0.0,0.0,-2021-10-04,0,0,0,0.0,0.0,-2021-10-05,0,0,0,0.0,0.0,-2021-10-06,0,0,0,0.0,0.0,-2021-10-07,0,0,0,0.0,0.0,-2021-10-08,0,0,0,0.0,0.0,-2021-10-09,0,0,0,0.0,0.0,-2021-10-10,0,0,0,0.0,0.0,-2021-10-11,0,0,0,0.0,0.0,-2021-10-12,0,0,0,0.0,0.0,-2021-10-13,0,0,0,0.0,0.0,-2021-10-14,0,0,0,0.0,0.0,-2021-10-15,0,0,0,0.0,0.0,-2021-10-16,0,0,0,0.0,0.0,-2021-10-17,0,0,0,0.0,0.0,-2021-10-18,0,0,0,0.0,0.0,-2021-10-19,0,0,0,0.0,0.0,-2021-10-20,1,1,1,2.0,0,60+date,visitors,pageviews,visits,views_per_visit,bounce_rate,visit_duration,scroll_depth+2021-09-20,0,0,0,0.0,0.0,,+2021-09-21,0,0,0,0.0,0.0,,+2021-09-22,0,0,0,0.0,0.0,,+2021-09-23,0,0,0,0.0,0.0,,+2021-09-24,0,0,0,0.0,0.0,,+2021-09-25,0,0,0,0.0,0.0,,+2021-09-26,0,0,0,0.0,0.0,,+2021-09-27,0,0,0,0.0,0.0,,+2021-09-28,0,0,0,0.0,0.0,,+2021-09-29,0,0,0,0.0,0.0,,+2021-09-30,0,0,0,0.0,0.0,,+2021-10-01,0,0,0,0.0,0.0,,+2021-10-02,0,0,0,0.0,0.0,,+2021-10-03,0,0,0,0.0,0.0,,+2021-10-04,0,0,0,0.0,0.0,,+2021-10-05,0,0,0,0.0,0.0,,+2021-10-06,0,0,0,0.0,0.0,,+2021-10-07,0,0,0,0.0,0.0,,+2021-10-08,0,0,0,0.0,0.0,,+2021-10-09,0,0,0,0.0,0.0,,+2021-10-10,0,0,0,0.0,0.0,,+2021-10-11,0,0,0,0.0,0.0,,+2021-10-12,0,0,0,0.0,0.0,,+2021-10-13,0,0,0,0.0,0.0,,+2021-10-14,0,0,0,0.0,0.0,,+2021-10-15,0,0,0,0.0,0.0,,+2021-10-16,0,0,0,0.0,0.0,,+2021-10-17,0,0,0,0.0,0.0,,+2021-10-18,0,0,0,0.0,0.0,,+2021-10-19,0,0,0,0.0,0.0,,+2021-10-20,1,1,1,2.0,0,60,diff --git a/test/plausible_web/controllers/CSVs/30d/pages.csv b/test/plausible_web/controllers/CSVs/30d/pages.csvindex 62b246c420..c04e0e7d7c 100644--- a/test/plausible_web/controllers/CSVs/30d/pages.csv+++ b/test/plausible_web/controllers/CSVs/30d/pages.csv@@ -1,4 +1,4 @@-name,visitors,pageviews,bounce_rate,time_on_page-/,4,3,67,-/signup,1,1,0,60.0-/some-other-page,1,1,0,60.0+name,visitors,pageviews,bounce_rate,time_on_page,scroll_depth+/,4,3,67,,+/signup,1,1,0,60.0,+/some-other-page,1,1,0,60.0,diff --git a/test/plausible_web/controllers/CSVs/6m/pages.csv b/test/plausible_web/controllers/CSVs/6m/pages.csvindex ed895e036f..294a95100a 100644--- a/test/plausible_web/controllers/CSVs/6m/pages.csv+++ b/test/plausible_web/controllers/CSVs/6m/pages.csv@@ -1,4 +1,4 @@-name,visitors,pageviews,bounce_rate,time_on_page-/,5,4,75,-/signup,1,1,0,60.0-/some-other-page,1,1,0,60.0+name,visitors,pageviews,bounce_rate,time_on_page,scroll_depth+/,5,4,75,,+/signup,1,1,0,60.0,+/some-other-page,1,1,0,60.0,diff --git a/test/plausible_web/controllers/stats_controller_test.exs b/test/plausible_web/controllers/stats_controller_test.exsindex 0aa89f9bc6..586cb5b6b6 100644--- a/test/plausible_web/controllers/stats_controller_test.exs+++ b/test/plausible_web/controllers/stats_controller_test.exs@@ -217,6 +217,39 @@ defmodule PlausibleWeb.StatsControllerTest doassert ~c"utm_terms.csv" in zipend+ test "exports scroll depth metric in pages.csv", %{conn: conn, site: site} do+ t0 = ~N[2020-01-01 00:00:00]+ [t1, t2, t3] = for i <- 1..3, do: NaiveDateTime.add(t0, i, :minute)++ populate_stats(site, [+ build(:pageview, user_id: 12, pathname: "/blog", timestamp: t0),+ build(:pageleave, user_id: 12, pathname: "/blog", timestamp: t1, scroll_depth: 20),+ build(:pageview, user_id: 12, pathname: "/another", timestamp: t1),+ build(:pageleave, user_id: 12, pathname: "/another", timestamp: t2, scroll_depth: 24),+ build(:pageview, user_id: 34, pathname: "/blog", timestamp: t0),+ build(:pageleave, user_id: 34, pathname: "/blog", timestamp: t1, scroll_depth: 17),+ build(:pageview, user_id: 34, pathname: "/another", timestamp: t1),+ build(:pageleave, user_id: 34, pathname: "/another", timestamp: t2, scroll_depth: 26),+ build(:pageview, user_id: 34, pathname: "/blog", timestamp: t2),+ build(:pageleave, user_id: 34, pathname: "/blog", timestamp: t3, scroll_depth: 60),+ build(:pageview, user_id: 56, pathname: "/blog", timestamp: t0),+ build(:pageleave, user_id: 56, pathname: "/blog", timestamp: t1, scroll_depth: 100)+ ])++ pages =+ conn+ |> get("/#{site.domain}/export?date=2020-01-01")+ |> response(200)+ |> unzip_and_parse_csv(~c"pages.csv")++ assert pages == [+ ["name", "visitors", "pageviews", "bounce_rate", "time_on_page", "scroll_depth"],+ ["/blog", "3", "4", "33", "60.0", "60"],+ ["/another", "2", "2", "0", "60.0", "25"],+ [""]+ ]+ end+test "exports only internally used props in custom_props.csv for a growth plan", %{conn: conn,site: site@@ -231,14 +264,13 @@ defmodule PlausibleWeb.StatsControllerTest dobuild(:event, name: "File Download", "meta.key": ["url"], "meta.value": ["b"])])- conn = get(conn, "/" <> site.domain <> "/export?period=day")- assert response = response(conn, 200)- {:ok, zip} = :zip.unzip(response, [:memory])-- {_filename, result} =- Enum.find(zip, fn {filename, _data} -> filename == ~c"custom_props.csv" end)+ result =+ conn+ |> get("/" <> site.domain <> "/export?period=day")+ |> response(200)+ |> unzip_and_parse_csv(~c"custom_props.csv")- assert parse_csv(result) == [+ assert result == [["property", "value", "visitors", "events", "percentage"],["url", "(none)", "1", "1", "50.0"],["url", "b", "1", "1", "50.0"],@@ -298,14 +330,13 @@ defmodule PlausibleWeb.StatsControllerTest dobuild(:pageview)])- conn = get(conn, "/" <> site.domain <> "/export?period=day")- assert response = response(conn, 200)- {:ok, zip} = :zip.unzip(response, [:memory])-- {_filename, result} =- Enum.find(zip, fn {filename, _data} -> filename == ~c"custom_props.csv" end)+ result =+ conn+ |> get("/" <> site.domain <> "/export?period=day")+ |> response(200)+ |> unzip_and_parse_csv(~c"custom_props.csv")- assert parse_csv(result) == [+ assert result == [["property", "value", "visitors", "events", "percentage"],["author", "(none)", "3", "4", "50.0"],["author", "uku", "2", "2", "33.3"],@@ -318,15 +349,14 @@ defmodule PlausibleWeb.StatsControllerTest dotest "exports data grouped by interval", %{conn: conn, site: site} dopopulate_exported_stats(site)- conn = get(conn, "/" <> site.domain <> "/export?date=2021-10-20&period=30d&interval=week")- assert response = response(conn, 200)- {:ok, zip} = :zip.unzip(response, [:memory])-- {_filename, visitors} =- Enum.find(zip, fn {filename, _data} -> filename == ~c"visitors.csv" end)+ visitors =+ conn+ |> get("/" <> site.domain <> "/export?date=2021-10-20&period=30d&interval=week")+ |> response(200)+ |> unzip_and_parse_csv(~c"visitors.csv")- assert parse_csv(visitors) == [+ assert visitors == [["date","visitors",@@ -361,15 +391,13 @@ defmodule PlausibleWeb.StatsControllerTest dobuild(:pageview, operating_system: "Mac", operating_system_version: "13")])- conn = get(conn, "/#{site.domain}/export")-- assert response = response(conn, 200)- {:ok, zip} = :zip.unzip(response, [:memory])-- {_filename, os_versions} =- Enum.find(zip, fn {filename, _data} -> filename == ~c"operating_system_versions.csv" end)+ os_versions =+ conn+ |> get("/#{site.domain}/export")+ |> response(200)+ |> unzip_and_parse_csv(~c"operating_system_versions.csv")- assert parse_csv(os_versions) == [+ assert os_versions == [["name", "version", "visitors"],["Mac", "14", "3"],["Ubuntu", "20.04", "2"],@@ -632,6 +660,61 @@ defmodule PlausibleWeb.StatsControllerTest doconn = get(conn, "/#{site.domain}/export?date=2021-10-20&filters=#{filters}")assert_zip(conn, "30d-filter-path")end++ test "exports scroll depth in visitors.csv", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:pageview, user_id: 12, pathname: "/blog", timestamp: ~N[2020-01-05 00:00:00]),+ build(:pageleave,+ user_id: 12,+ pathname: "/blog",+ timestamp: ~N[2020-01-05 00:01:00],+ scroll_depth: 40+ ),+ build(:pageview, user_id: 12, pathname: "/blog", timestamp: ~N[2020-01-05 10:00:00]),+ build(:pageleave,+ user_id: 12,+ pathname: "/blog",+ timestamp: ~N[2020-01-05 10:01:00],+ scroll_depth: 17+ ),+ build(:pageview, user_id: 34, pathname: "/blog", timestamp: ~N[2020-01-07 00:00:00]),+ build(:pageleave,+ user_id: 34,+ pathname: "/blog",+ timestamp: ~N[2020-01-07 00:01:00],+ scroll_depth: 90+ )+ ])++ filters = Jason.encode!([[:is, "event:page", ["/blog"]]])++ pages =+ conn+ |> get("/#{site.domain}/export?date=2020-01-07&period=7d&filters=#{filters}")+ |> response(200)+ |> unzip_and_parse_csv(~c"visitors.csv")++ assert pages == [+ [+ "date",+ "visitors",+ "pageviews",+ "visits",+ "views_per_visit",+ "bounce_rate",+ "visit_duration",+ "scroll_depth"+ ],+ ["2020-01-01", "0", "0", "0", "0.0", "0.0", "", ""],+ ["2020-01-02", "0", "0", "0", "0.0", "0.0", "", ""],+ ["2020-01-03", "0", "0", "0", "0.0", "0.0", "", ""],+ ["2020-01-04", "0", "0", "0", "0.0", "0.0", "", ""],+ ["2020-01-05", "1", "2", "2", "1.0", "100", "0", "40"],+ ["2020-01-06", "0", "0", "0", "0.0", "0.0", "", ""],+ ["2020-01-07", "1", "1", "1", "1.0", "100", "0", "90"],+ [""]+ ]+ endenddescribe "GET /:domain/export - with a custom prop filter" do@@ -650,14 +733,14 @@ defmodule PlausibleWeb.StatsControllerTest do])filters = Jason.encode!([[:is, "event:props:author", ["marko"]]])- conn = get(conn, "/" <> site.domain <> "/export?period=day&filters=#{filters}")- {:ok, zip} = :zip.unzip(response(conn, 200), [:memory])-- {_filename, result} =- Enum.find(zip, fn {filename, _data} -> filename == ~c"custom_props.csv" end)+ result =+ conn+ |> get("/" <> site.domain <> "/export?period=day&filters=#{filters}")+ |> response(200)+ |> unzip_and_parse_csv(~c"custom_props.csv")- assert parse_csv(result) == [+ assert result == [["property", "value", "visitors", "events", "percentage"],["author", "marko", "1", "1", "100.0"],[""]@@ -665,6 +748,12 @@ defmodule PlausibleWeb.StatsControllerTest doendend+ defp unzip_and_parse_csv(archive, filename) do+ {:ok, zip} = :zip.unzip(archive, [:memory])+ {_filename, data} = Enum.find(zip, &(elem(&1, 0) == filename))+ parse_csv(data)+ end+defp assert_zip(conn, folder) doassert conn.status == 200@@ -675,10 +764,10 @@ defmodule PlausibleWeb.StatsControllerTest dofolder = Path.expand(folder, "test/plausible_web/controllers/CSVs")- Enum.map(zip, &assert_csv(&1, folder))+ Enum.map(zip, &assert_csv_by_fixture(&1, folder))end- defp assert_csv({file, downloaded}, folder) do+ defp assert_csv_by_fixture({file, downloaded}, folder) dofile = Path.expand(file, folder){:ok, content} = File.read(file)@@ -788,14 +877,14 @@ defmodule PlausibleWeb.StatsControllerTest doinsert(:goal, site: site, event_name: "Newsletter Signup")filters = Jason.encode!([[:is, "event:goal", ["Newsletter Signup"]]])- conn = get(conn, "/" <> site.domain <> "/export?period=day&filters=#{filters}")- {:ok, zip} = :zip.unzip(response(conn, 200), [:memory])-- {_filename, result} =- Enum.find(zip, fn {filename, _data} -> filename == ~c"custom_props.csv" end)+ result =+ conn+ |> get("/" <> site.domain <> "/export?period=day&filters=#{filters}")+ |> response(200)+ |> unzip_and_parse_csv(~c"custom_props.csv")- assert parse_csv(result) == [+ assert result == [["property", "value", "visitors", "events", "conversion_rate"],["author", "marko", "2", "2", "50.0"],["author", "uku", "1", "1", "25.0"],@@ -844,15 +933,14 @@ defmodule PlausibleWeb.StatsControllerTest doinsert(:goal, site: site, event_name: "Signup")filters = Jason.encode!([[:is, "event:goal", ["Signup"]]])- conn = get(conn, "/#{site.domain}/export?filters=#{filters}")-- assert response = response(conn, 200)- {:ok, zip} = :zip.unzip(response, [:memory])- {_filename, os_versions} =- Enum.find(zip, fn {filename, _data} -> filename == ~c"operating_system_versions.csv" end)+ os_versions =+ conn+ |> get("/#{site.domain}/export?filters=#{filters}")+ |> response(200)+ |> unzip_and_parse_csv(~c"operating_system_versions.csv")- assert parse_csv(os_versions) == [+ assert os_versions == [["name", "version", "conversions", "conversion_rate"],["Mac", "14", "3", "75.0"],["Ubuntu", "20.04", "2", "100.0"],