Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.
TypeScript analyticsanalytics-dashboardclickhouseelixirgoogle-analyticsmarketingopen-source-analyticsphoenixplausible-analyticspostgresqlprivacyprivacy-friendlyself-hostedsimple-analyticsstatisticstailwindcssweb-analyticswebsite-analyticswebsite-statswebsite-tracking
Handle missing scroll depth #2 (#5017)
Karl-Aksel Puulmann7c6ba04a3cdb
diff --git a/assets/js/dashboard/site-context.test.tsx b/assets/js/dashboard/site-context.test.tsxindex 012ca33714..9c4109bee1 100644--- a/assets/js/dashboard/site-context.test.tsx+++ b/assets/js/dashboard/site-context.test.tsx@@ -20,6 +20,7 @@ describe('parseSiteFromDataset', () => {data-revenue-goals='[{"currency":"USD","display_name":"Purchase"}]'data-funnels='[{"id":1,"name":"From homepage to login","steps_count":3}]'data-has-props="true"+ data-scroll-depth-visible="true"data-logged-in="true"data-stats-begin="2021-09-07"data-native-stats-begin="2022-09-02"@@ -43,6 +44,7 @@ describe('parseSiteFromDataset', () => {revenueGoals: [{ currency: 'USD', display_name: 'Purchase' }],funnels: [{ id: 1, name: 'From homepage to login', steps_count: 3 }],hasProps: true,+ scrollDepthVisible: true,statsBegin: '2021-09-07',nativeStatsBegin: '2022-09-02',embedded: false,diff --git a/assets/js/dashboard/site-context.tsx b/assets/js/dashboard/site-context.tsxindex 80e6f2ba52..215bcba48d 100644--- a/assets/js/dashboard/site-context.tsx+++ b/assets/js/dashboard/site-context.tsx@@ -7,6 +7,7 @@ export function parseSiteFromDataset(dataset: DOMStringMap): PlausibleSite {offset: parseInt(dataset.offset!, 10),hasGoals: dataset.hasGoals === 'true',hasProps: dataset.hasProps === 'true',+ scrollDepthVisible: dataset.scrollDepthVisible === 'true',funnelsAvailable: dataset.funnelsAvailable === 'true',propsAvailable: dataset.propsAvailable === 'true',conversionsOptedOut: dataset.conversionsOptedOut === 'true',@@ -36,6 +37,7 @@ const siteContextDefaultValue = {offset: 0,hasGoals: false,hasProps: false,+ scrollDepthVisible: false,funnelsAvailable: false,propsAvailable: false,conversionsOptedOut: false,diff --git a/assets/js/dashboard/stats/graph/graph-util.js b/assets/js/dashboard/stats/graph/graph-util.jsindex f80d105e78..56bc3047e7 100644--- a/assets/js/dashboard/stats/graph/graph-util.js+++ b/assets/js/dashboard/stats/graph/graph-util.js@@ -16,7 +16,7 @@ export function getGraphableMetrics(query, site) {return ["visitors", "events", "conversion_rate"]} else if (isPageFilter) {const pageFilterMetrics = ["visitors", "visits", "pageviews", "bounce_rate"]- return site.flags.scroll_depth ? [...pageFilterMetrics, "scroll_depth"] : pageFilterMetrics+ return site.scrollDepthVisible ? [...pageFilterMetrics, "scroll_depth"] : pageFilterMetrics} else {return ["visitors", "visits", "pageviews", "views_per_visit", "bounce_rate", "visit_duration"]}diff --git a/assets/js/dashboard/stats/modals/pages.js b/assets/js/dashboard/stats/modals/pages.jsindex 6129fe444c..79485a7946 100644--- a/assets/js/dashboard/stats/modals/pages.js+++ b/assets/js/dashboard/stats/modals/pages.js@@ -54,7 +54,7 @@ function PagesModal() {metrics.createTimeOnPage()]- return site.flags.scroll_depth ? [...defaultMetrics, metrics.createScrollDepth()] : defaultMetrics+ return site.scrollDepthVisible ? [...defaultMetrics, metrics.createScrollDepth()] : defaultMetrics}return (diff --git a/assets/test-utils/app-context-providers.tsx b/assets/test-utils/app-context-providers.tsxindex 5d26d3f2f5..96819be939 100644--- a/assets/test-utils/app-context-providers.tsx+++ b/assets/test-utils/app-context-providers.tsx@@ -26,6 +26,7 @@ export const TestContextProviders = ({offset: 0,hasGoals: false,hasProps: false,+ scrollDepthVisible: false,funnelsAvailable: false,propsAvailable: false,conversionsOptedOut: false,diff --git a/lib/plausible/exports.ex b/lib/plausible/exports.exindex 46255d628c..f082e5f5f2 100644--- a/lib/plausible/exports.ex+++ b/lib/plausible/exports.ex@@ -418,8 +418,7 @@ defmodule Plausible.Exports dosite = Plausible.Repo.get(Plausible.Site, site_id)current_user = current_user_id && Plausible.Repo.get(Plausible.Auth.User, current_user_id)- scroll_depth_enabled? =- PlausibleWeb.Api.StatsController.scroll_depth_enabled?(site, current_user)+ include_scroll_depth? = Plausible.Stats.ScrollDepth.check_feature_visible!(site, current_user)base_q =from(e in sampled("events_v2"),@@ -429,7 +428,7 @@ defmodule Plausible.Exports doorder_by: selected_as(:date))- if scroll_depth_enabled? do+ if include_scroll_depth? domax_scroll_depth_per_visitor_q =from(e in "events_v2",where: ^export_filter(site_id, date_range),diff --git a/lib/plausible/site.ex b/lib/plausible/site.exindex 37d0cb2187..11d89f4401 100644--- a/lib/plausible/site.ex+++ b/lib/plausible/site.ex@@ -21,6 +21,7 @@ defmodule Plausible.Site dofield :conversions_enabled, :boolean, default: truefield :props_enabled, :boolean, default: truefield :funnels_enabled, :boolean, default: true+ field :scroll_depth_visible_at, :naive_datetimefield :ingest_rate_limit_scale_seconds, :integer, default: 60# default is set via changeset/2diff --git a/lib/plausible/sites.ex b/lib/plausible/sites.exindex 04af80c24e..6dc806f7e8 100644--- a/lib/plausible/sites.ex+++ b/lib/plausible/sites.ex@@ -284,6 +284,14 @@ defmodule Plausible.Sites do|> Repo.update!()end+ def set_scroll_depth_visible_at(site) do+ utc_now = NaiveDateTime.utc_now(:second)++ site+ |> Ecto.Changeset.change(%{scroll_depth_visible_at: utc_now})+ |> Repo.update()+ end+def has_goals?(site) doRepo.exists?(from(g in Plausible.Goal,diff --git a/lib/plausible/stats/scroll_depth.ex b/lib/plausible/stats/scroll_depth.exnew file mode 100644index 0000000000..246c8a8527--- /dev/null+++ b/lib/plausible/stats/scroll_depth.ex@@ -0,0 +1,63 @@+defmodule Plausible.Stats.ScrollDepth do+ @moduledoc """+ Module to check whether the scroll depth metric is available and visible for a site.+ """++ import Ecto.Query+ require Logger++ alias Plausible.ClickhouseRepo++ def feature_available?(site, user) do+ FunWithFlags.enabled?(:scroll_depth, for: user) ||+ FunWithFlags.enabled?(:scroll_depth, for: site)+ end++ def feature_visible?(site, user) do+ feature_available?(site, user) && not is_nil(site.scroll_depth_visible_at)+ end++ @doc """+ Checks whether the scroll depth feature is visible for a site and updates the site record if it is.++ Note this function queries ClickHouse and may take a while to complete.+ """+ def check_feature_visible!(site, user) do+ cond do+ not feature_available?(site, user) ->+ false++ not is_nil(site.scroll_depth_visible_at) ->+ true++ is_nil(site.scroll_depth_visible_at) ->+ visible? = has_scroll_data_last_30d?(site)++ if visible? do+ Plausible.Sites.set_scroll_depth_visible_at(site)+ end++ visible?+ end+ end++ defp has_scroll_data_last_30d?(site) do+ try do+ ClickhouseRepo.exists?(+ from(e in "events_v2",+ where:+ e.site_id == ^site.id and+ e.name == "pageleave" and+ e.timestamp >= fragment("toStartOfDay(now() - toIntervalDay(30))") and+ e.scroll_depth > 0 and e.scroll_depth <= 100+ )+ )+ rescue+ # Avoid propagating error to the user, bringing down the site.+ error ->+ Logger.error("Error checking scroll data for site #{site.id}: #{inspect(error)}")++ false+ end+ end+enddiff --git a/lib/plausible_web/controllers/api/stats_controller.ex b/lib/plausible_web/controllers/api/stats_controller.exindex a4a279ed45..e9c2c3e6e2 100644--- a/lib/plausible_web/controllers/api/stats_controller.ex+++ b/lib/plausible_web/controllers/api/stats_controller.ex@@ -389,16 +389,16 @@ defmodule PlausibleWeb.Api.StatsController dopage_filter? =Filters.filtering_on_dimension?(query, "event:page", behavioral_filters: :ignore)- scroll_depth_enabled? = scroll_depth_enabled?(site, current_user)+ include_scroll_depth? = Plausible.Stats.ScrollDepth.feature_visible?(site, current_user)metrics = [:visitors, :visits, :pageviews, :sample_percent]metrics =cond do- page_filter? && scroll_depth_enabled? && query.include_imported ->+ page_filter? && include_scroll_depth? && query.include_imported ->metrics ++ [:scroll_depth]- page_filter? && scroll_depth_enabled? ->+ page_filter? && include_scroll_depth? ->metrics ++ [:bounce_rate, :scroll_depth, :time_on_page]page_filter? && query.include_imported ->@@ -851,7 +851,7 @@ defmodule PlausibleWeb.Api.StatsController doparams = Map.put(params, "property", "event:page")query = Query.from(site, params, debug_metadata(conn))- include_scroll_depth? = scroll_depth_enabled?(site, current_user)+ include_scroll_depth? = Plausible.Stats.ScrollDepth.feature_visible?(site, current_user)extra_metrics =cond do@@ -1663,9 +1663,4 @@ defmodule PlausibleWeb.Api.StatsController dodefp toplevel_goal_filter?(query) doFilters.filtering_on_dimension?(query, "event:goal", max_depth: 0)end-- def scroll_depth_enabled?(site, user) do- FunWithFlags.enabled?(:scroll_depth, for: user) ||- FunWithFlags.enabled?(:scroll_depth, for: site)- endenddiff --git a/lib/plausible_web/controllers/stats_controller.ex b/lib/plausible_web/controllers/stats_controller.exindex 336571fbd1..8d5dc4b8cb 100644--- a/lib/plausible_web/controllers/stats_controller.ex+++ b/lib/plausible_web/controllers/stats_controller.ex@@ -52,12 +52,16 @@ defmodule PlausibleWeb.StatsController dodef stats(%{assigns: %{site: site}} = conn, _params) dosite = Plausible.Repo.preload(site, :owner)+ current_user = conn.assigns[:current_user]stats_start_date = Plausible.Sites.stats_start_date(site)can_see_stats? = not Sites.locked?(site) or conn.assigns[:site_role] == :super_admindemo = site.domain == PlausibleWeb.Endpoint.host()dogfood_page_path = if demo, do: "/#{site.domain}", else: "/:dashboard"skip_to_dashboard? = conn.params["skip_to_dashboard"] == "true"+ scroll_depth_visible? =+ Plausible.Stats.ScrollDepth.check_feature_visible!(site, current_user)+cond do(stats_start_date && can_see_stats?) || (can_see_stats? && skip_to_dashboard?) ->conn@@ -68,11 +72,12 @@ defmodule PlausibleWeb.StatsController dorevenue_goals: list_revenue_goals(site),funnels: list_funnels(site),has_props: Plausible.Props.configured?(site),+ scroll_depth_visible: scroll_depth_visible?,stats_start_date: stats_start_date,native_stats_start_date: NaiveDateTime.to_date(site.native_stats_start_at),title: title(conn, site),demo: demo,- flags: get_flags(conn.assigns[:current_user], site),+ flags: get_flags(current_user, site),is_dbip: is_dbip(),dogfood_page_path: dogfood_page_path,load_dashboard_js: true@@ -193,8 +198,8 @@ defmodule PlausibleWeb.StatsController dodefp csv_graph_metrics(query, site, current_user) doinclude_scroll_depth? =!query.include_imported &&- PlausibleWeb.Api.StatsController.scroll_depth_enabled?(site, current_user) &&- Filters.filtering_on_dimension?(query, "event:page", behavioral_filters: :ignore)+ Filters.filtering_on_dimension?(query, "event:page", behavioral_filters: :ignore) &&+ Plausible.Stats.ScrollDepth.feature_visible?(site, current_user){metrics, column_headers} =if Filters.filtering_on_dimension?(query, "event:goal", max_depth: 0) do@@ -341,9 +346,13 @@ defmodule PlausibleWeb.StatsController dodefp render_shared_link(conn, shared_link) docond do!shared_link.site.locked ->+ current_user = conn.assigns[:current_user]shared_link = Plausible.Repo.preload(shared_link, site: :owner)stats_start_date = Plausible.Sites.stats_start_date(shared_link.site)+ scroll_depth_visible? =+ Plausible.Stats.ScrollDepth.check_feature_visible!(shared_link.site, current_user)+conn|> put_resp_header("x-robots-tag", "noindex, nofollow")|> delete_resp_header("x-frame-options")@@ -353,6 +362,7 @@ defmodule PlausibleWeb.StatsController dorevenue_goals: list_revenue_goals(shared_link.site),funnels: list_funnels(shared_link.site),has_props: Plausible.Props.configured?(shared_link.site),+ scroll_depth_visible: scroll_depth_visible?,stats_start_date: stats_start_date,native_stats_start_date: NaiveDateTime.to_date(shared_link.site.native_stats_start_at),title: title(conn, shared_link.site),@@ -362,7 +372,7 @@ defmodule PlausibleWeb.StatsController doembedded: conn.params["embed"] == "true",background: conn.params["background"],theme: conn.params["theme"],- flags: get_flags(conn.assigns[:current_user], shared_link.site),+ flags: get_flags(current_user, shared_link.site),is_dbip: is_dbip(),load_dashboard_js: true)diff --git a/lib/plausible_web/templates/stats/stats.html.heex b/lib/plausible_web/templates/stats/stats.html.heexindex 7575a60b8f..8bbecc573a 100644--- a/lib/plausible_web/templates/stats/stats.html.heex+++ b/lib/plausible_web/templates/stats/stats.html.heex@@ -30,6 +30,7 @@data-revenue-goals={Jason.encode!(@revenue_goals)}data-funnels={Jason.encode!(@funnels)}data-has-props={to_string(@has_props)}+ data-scroll-depth-visible={to_string(@scroll_depth_visible)}data-logged-in={to_string([email protected][:current_user])}data-stats-begin={@stats_start_date}data-native-stats-begin={@native_stats_start_date}diff --git a/test/plausible/exports_test.exs b/test/plausible/exports_test.exsindex 9c320235ff..adf362d132 100644--- a/test/plausible/exports_test.exs+++ b/test/plausible/exports_test.exs@@ -6,8 +6,10 @@ defmodule Plausible.ExportsTest do# for e2e export->import tests please see Plausible.Imported.CSVImporterTestdescribe "export_queries/2" do- test "returns named ecto queries" do- queries = Plausible.Exports.export_queries(_site_id = 1, nil)+ setup [:create_user, :create_site]++ test "returns named ecto queries", %{site: site} do+ queries = Plausible.Exports.export_queries(site.id, nil)assert queries |> Map.values() |> Enum.all?(&match?(%Ecto.Query{}, &1))assert Map.keys(queries) == [@@ -24,9 +26,9 @@ defmodule Plausible.ExportsTest do]end- test "with date range" do+ test "with date range", %{site: site} doqueries =- Plausible.Exports.export_queries(_site_id = 1, nil,+ Plausible.Exports.export_queries(site.id, nil,date_range: Date.range(~D[2023-01-01], ~D[2024-03-12]))@@ -44,8 +46,8 @@ defmodule Plausible.ExportsTest do]end- test "with custom extension" do- queries = Plausible.Exports.export_queries(_site_id = 1, nil, extname: ".ch")+ test "with custom extension", %{site: site} do+ queries = Plausible.Exports.export_queries(site.id, nil, extname: ".ch")assert Map.keys(queries) == ["imported_browsers.ch",diff --git a/test/plausible/imported/csv_importer_test.exs b/test/plausible/imported/csv_importer_test.exsindex efc4b05443..a205355be5 100644--- a/test/plausible/imported/csv_importer_test.exs+++ b/test/plausible/imported/csv_importer_test.exs@@ -587,80 +587,28 @@ defmodule Plausible.Imported.CSVImporterTest doprocess_csv.("fixture/markosaric_com_sessions_v2_2024_04_01_2024_04_30_dump.csv")])- # export archive to s3- on_ee do- assert {:ok, _job} =- Plausible.Exports.schedule_s3_export(exported_site.id, nil, user.email)- else- assert {:ok, %{args: %{"local_path" => local_path}}} =- Plausible.Exports.schedule_local_export(exported_site.id, user.email)- end-- assert %{success: 1} = Oban.drain_queue(queue: :analytics_exports, with_safety: false)-- assert %{success: 1} =- Oban.drain_queue(queue: :notify_exported_analytics, with_safety: false)-- # check mailbox- assert_receive {:delivered_email, email}, _within = :timer.seconds(5)- assert email.to == [{user.name, user.email}]-- assert email.html_body =~- ~s[Please click <a href="http://localhost:8000/#{URI.encode_www_form(exported_site.domain)}/download/export">here</a>]-- assert email.text_body =~- ~r[Please click here \(http://localhost:8000/#{URI.encode_www_form(exported_site.domain)}/download/export\) to start the download process.]-- # download archive- on_ee do- ExAws.request!(- ExAws.S3.download_file(- Plausible.S3.exports_bucket(),- to_string(exported_site.id),- Path.join(tmp_dir, "plausible-export.zip")- )- )- else- File.rename!(local_path, Path.join(tmp_dir, "plausible-export.zip"))- end-- # unzip archive- {:ok, files} =- :zip.unzip(to_charlist(Path.join(tmp_dir, "plausible-export.zip")), cwd: tmp_dir)-- # upload csvs- uploads =- Enum.map(files, fn file ->- on_ee do- %{s3_url: s3_url} = Plausible.S3.import_presign_upload(imported_site.id, file)- [bucket, key] = String.split(URI.parse(s3_url).path, "/", parts: 2)- ExAws.request!(ExAws.S3.put_object(bucket, key, File.read!(file)))- %{"filename" => Path.basename(file), "s3_url" => s3_url}- else- %{"filename" => Path.basename(file), "local_path" => file}- end- end)-- # run importer- date_range = CSVImporter.date_range(uploads)-- {:ok, _job} =- CSVImporter.new_import(imported_site, user,- start_date: date_range.first,- end_date: date_range.last,- uploads: uploads,- storage: on_ee(do: "s3", else: "local")- )-- assert %{success: 1} = Oban.drain_queue(queue: :analytics_imports, with_safety: false)+ initial_context = %{+ user: user,+ tmp_dir: tmp_dir,+ exported_site: exported_site,+ imported_site: imported_site+ }++ %{site_import: site_import} =+ initial_context+ |> export_archive()+ |> assert_email_notification()+ |> download_archive()+ |> unzip_archive()+ |> upload_csvs()+ |> run_import()- # validate importassert %SiteImport{start_date: ~D[2024-04-01],end_date: ~D[2024-04-30],source: :csv,status: :completed- } = Repo.get_by!(SiteImport, site_id: imported_site.id)+ } = site_importassert Plausible.Stats.Clickhouse.imported_pageview_count(exported_site) == 0assert Plausible.Stats.Clickhouse.imported_pageview_count(imported_site) == 1745@@ -1088,7 +1036,11 @@ defmodule Plausible.Imported.CSVImporterTest doexported_site = new_site(owner: user)imported_site = new_site(owner: user)- t0 = ~N[2020-01-01 00:00:00]+ t0 =+ NaiveDateTime.utc_now(:second)+ |> NaiveDateTime.add(-5, :day)+ |> NaiveDateTime.beginning_of_day()+[t1, t2, t3] = for i <- 1..3, do: NaiveDateTime.add(t0, i, :minute)stats =@@ -1114,72 +1066,36 @@ defmodule Plausible.Imported.CSVImporterTest dopopulate_stats(exported_site, stats)- # export archive to s3- on_ee do- assert {:ok, _job} =- Plausible.Exports.schedule_s3_export(exported_site.id, nil, user.email)- else- assert {:ok, %{args: %{"local_path" => local_path}}} =- Plausible.Exports.schedule_local_export(exported_site.id, user.email)- end+ initial_context = %{+ user: user,+ tmp_dir: tmp_dir,+ exported_site: exported_site,+ imported_site: imported_site+ }- assert %{success: 1} = Oban.drain_queue(queue: :analytics_exports, with_safety: false)+ %{site_import: site_import} =+ initial_context+ |> export_archive()+ |> download_archive()+ |> unzip_archive()+ |> upload_csvs()+ |> run_import()- # download archive- on_ee do- ExAws.request!(- ExAws.S3.download_file(- Plausible.S3.exports_bucket(),- to_string(exported_site.id),- Path.join(tmp_dir, "plausible-export.zip")- )- )- else- File.rename!(local_path, Path.join(tmp_dir, "plausible-export.zip"))- end+ assert %NaiveDateTime{} =+ Plausible.Repo.reload!(exported_site).scroll_depth_visible_at- # unzip archive- {:ok, files} =- :zip.unzip(to_charlist(Path.join(tmp_dir, "plausible-export.zip")), cwd: tmp_dir)-- # upload csvs- uploads =- Enum.map(files, fn file ->- on_ee do- %{s3_url: s3_url} = Plausible.S3.import_presign_upload(imported_site.id, file)- [bucket, key] = String.split(URI.parse(s3_url).path, "/", parts: 2)- content = File.read!(file)- ExAws.request!(ExAws.S3.put_object(bucket, key, content))- %{"filename" => Path.basename(file), "s3_url" => s3_url}- else- %{- "filename" => Path.basename(file),- "local_path" => file- }- end- end)-- # run importer- date_range = CSVImporter.date_range(uploads)-- {:ok, _job} =- CSVImporter.new_import(imported_site, user,- start_date: date_range.first,- end_date: date_range.last,- uploads: uploads,- storage: on_ee(do: "s3", else: "local")- )-- assert %{success: 1} = Oban.drain_queue(queue: :analytics_imports, with_safety: false)-- # validate importassert %SiteImport{- start_date: ~D[2020-01-01],- end_date: ~D[2020-01-02],+ start_date: start_date,+ end_date: end_date,source: :csv,status: :completed- } = Repo.get_by!(SiteImport, site_id: imported_site.id)+ } = site_import++ expected_start_date = t0 |> NaiveDateTime.to_date()+ expected_end_date = t0 |> NaiveDateTime.to_date() |> Date.add(1)+ assert start_date == expected_start_date+ assert end_date == expected_end_dateassert Plausible.Stats.Clickhouse.imported_pageview_count(exported_site) == 0assert Plausible.Stats.Clickhouse.imported_pageview_count(imported_site) == 9@@ -1196,13 +1112,18 @@ defmodule Plausible.Imported.CSVImporterTest do)|> Plausible.IngestRepo.all()- assert %{date: ~D[2020-01-01], page: "/", scroll_depth: 20, pageleave_visitors: 1} in imported_data+ assert %{date: expected_start_date, page: "/", scroll_depth: 20, pageleave_visitors: 1} in imported_data- assert %{date: ~D[2020-01-01], page: "/another", scroll_depth: 50, pageleave_visitors: 2} in imported_data+ assert %{+ date: expected_start_date,+ page: "/another",+ scroll_depth: 50,+ pageleave_visitors: 2+ } in imported_data- assert %{date: ~D[2020-01-01], page: "/blog", scroll_depth: 180, pageleave_visitors: 3} in imported_data+ assert %{date: expected_start_date, page: "/blog", scroll_depth: 180, pageleave_visitors: 3} in imported_data- assert %{date: ~D[2020-01-02], page: "/blog", scroll_depth: nil, pageleave_visitors: 0} in imported_data+ assert %{date: expected_end_date, page: "/blog", scroll_depth: nil, pageleave_visitors: 0} in imported_data# assert via stats queries that scroll_depth from imported# data matches the scroll_depth from native data@@ -1228,6 +1149,138 @@ defmodule Plausible.Imported.CSVImporterTest doassert query_scroll_depth_per_page.(conn, exported_site) == expected_resultsassert query_scroll_depth_per_page.(conn, imported_site) == expected_resultsend++ @tag :tmp_dir+ test "does not include scroll depth without existing pageleave data", %{+ user: user,+ tmp_dir: tmp_dir+ } do+ exported_site = new_site(owner: user)++ populate_stats(exported_site, [build(:pageview, timestamp: ~N[2021-01-01 00:00:00])])++ context = %{+ user: user,+ tmp_dir: tmp_dir,+ exported_site: exported_site,+ imported_site: new_site(owner: user)+ }++ %{exported_files: exported_files} =+ context+ |> export_archive()+ |> download_archive()+ |> unzip_archive()++ imported_pages_content =+ exported_files+ |> Enum.find(&String.contains?(&1, "imported_pages"))+ |> File.read!()++ assert is_nil(Plausible.Repo.reload!(exported_site).scroll_depth_visible_at)++ refute imported_pages_content =~ "scroll_depth"+ end+ end++ defp export_archive(%{user: user, exported_site: exported_site} = context) do+ context =+ on_ee do+ assert {:ok, _job} =+ Plausible.Exports.schedule_s3_export(exported_site.id, nil, user.email)++ context+ else+ assert {:ok, %{args: %{"local_path" => local_path}}} =+ Plausible.Exports.schedule_local_export(exported_site.id, user.email)++ Map.put(context, :local_path, local_path)+ end++ Oban.drain_queue(queue: :analytics_exports, with_safety: false)++ context+ end++ defp assert_email_notification(%{user: user, exported_site: exported_site} = context) do+ assert %{success: 1} =+ Oban.drain_queue(queue: :notify_exported_analytics, with_safety: false)++ assert_receive {:delivered_email, email}, _within = :timer.seconds(5)+ assert email.to == [{user.name, user.email}]++ assert email.html_body =~+ ~s[Please click <a href="http://localhost:8000/#{URI.encode_www_form(exported_site.domain)}/download/export">here</a>]++ assert email.text_body =~+ ~r[Please click here \(http://localhost:8000/#{URI.encode_www_form(exported_site.domain)}/download/export\) to start the download process.]++ context+ end++ defp download_archive(%{tmp_dir: tmp_dir} = context) do+ on_ee do+ ExAws.request!(+ ExAws.S3.download_file(+ Plausible.S3.exports_bucket(),+ to_string(context.exported_site.id),+ Path.join(tmp_dir, "plausible-export.zip")+ )+ )+ else+ File.rename!(context.local_path, Path.join(tmp_dir, "plausible-export.zip"))+ end++ context+ end++ defp unzip_archive(%{tmp_dir: tmp_dir} = context) do+ assert {:ok, files} =+ :zip.unzip(to_charlist(Path.join(tmp_dir, "plausible-export.zip")), cwd: tmp_dir)++ Map.put(context, :exported_files, files)+ end++ defp upload_csvs(%{exported_files: files} = context) do+ uploads =+ Enum.map(files, fn file ->+ on_ee do+ %{s3_url: s3_url} = Plausible.S3.import_presign_upload(context.imported_site.id, file)+ [bucket, key] = String.split(URI.parse(s3_url).path, "/", parts: 2)+ content = File.read!(file)+ ExAws.request!(ExAws.S3.put_object(bucket, key, content))+ %{"filename" => Path.basename(file), "s3_url" => s3_url}+ else+ %{+ "filename" => Path.basename(file),+ "local_path" => file+ }+ end+ end)++ Map.put(context, :uploaded_file_refs, uploads)+ end++ defp run_import(+ %{uploaded_file_refs: uploads, imported_site: imported_site, user: user} = context+ ) do+ date_range = CSVImporter.date_range(uploads)++ {:ok, _job} =+ CSVImporter.new_import(imported_site, user,+ start_date: date_range.first,+ end_date: date_range.last,+ uploads: uploads,+ storage: on_ee(do: "s3", else: "local")+ )++ assert %{success: 1} = Oban.drain_queue(queue: :analytics_imports, with_safety: false)++ site_import = Repo.get_by!(SiteImport, site_id: imported_site.id)++ assert %SiteImport{source: :csv, status: :completed} = site_import++ Map.put(context, :site_import, site_import)enddefp clean_buckets(_context) dodiff --git a/test/plausible_web/controllers/api/stats_controller/imported_test.exs b/test/plausible_web/controllers/api/stats_controller/imported_test.exsindex 048fc2ccbf..828a855985 100644--- a/test/plausible_web/controllers/api/stats_controller/imported_test.exs+++ b/test/plausible_web/controllers/api/stats_controller/imported_test.exs@@ -787,6 +787,8 @@ defmodule PlausibleWeb.Api.StatsController.ImportedTest dosite: site,import_id: import_id} do+ Plausible.Sites.set_scroll_depth_visible_at(site)+populate_stats(site, [build(:pageview,pathname: "/",diff --git a/test/plausible_web/controllers/api/stats_controller/pages_test.exs b/test/plausible_web/controllers/api/stats_controller/pages_test.exsindex 6a70985668..506f2a0698 100644--- a/test/plausible_web/controllers/api/stats_controller/pages_test.exs+++ b/test/plausible_web/controllers/api/stats_controller/pages_test.exs@@ -1,10 +1,17 @@defmodule PlausibleWeb.Api.StatsController.PagesTest douse PlausibleWeb.ConnCase+ use Plausible.Teams.Test@user_id Enum.random(1000..9999)describe "GET /api/stats/:domain/pages" do- setup [:create_user, :log_in, :create_site, :create_legacy_site_import]+ setup [+ :create_user,+ :log_in,+ :create_site,+ :create_legacy_site_import,+ :set_scroll_depth_visible_at+ ]test "returns top pages by visitors", %{conn: conn, site: site} dopopulate_stats(site, [@@ -648,6 +655,27 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do]end+ test "does not return scroll depth (in detailed mode) when site.scroll_depth_visible_at=nil",+ %{conn: conn, user: user} do+ site = new_site(owner: user)++ populate_stats(site, [build(:pageview)])++ pages =+ conn+ |> get("/api/stats/#{site.domain}/pages?detailed=true")+ |> json_response(200)+ |> Map.get("results")++ assert List.first(pages) == %{+ "bounce_rate" => 100,+ "name" => "/",+ "pageviews" => 1,+ "time_on_page" => nil,+ "visitors" => 1+ }+ end+test "calculates scroll_depth from native and imported data combined", %{conn: conn,site: sitediff --git a/test/plausible_web/controllers/api/stats_controller/top_stats_test.exs b/test/plausible_web/controllers/api/stats_controller/top_stats_test.exsindex f9336b5b6a..286a1073f6 100644--- a/test/plausible_web/controllers/api/stats_controller/top_stats_test.exs+++ b/test/plausible_web/controllers/api/stats_controller/top_stats_test.exs@@ -478,7 +478,13 @@ defmodule PlausibleWeb.Api.StatsController.TopStatsTest doenddescribe "GET /api/stats/top-stats - with imported data" do- setup [:create_user, :log_in, :create_site, :create_legacy_site_import]+ setup [+ :create_user,+ :log_in,+ :create_site,+ :create_legacy_site_import,+ :set_scroll_depth_visible_at+ ]test "returns divisible metrics as 0 when no stats exist", %{site: site,@@ -888,7 +894,7 @@ defmodule PlausibleWeb.Api.StatsController.TopStatsTest doenddescribe "GET /api/stats/top-stats - filters" do- setup [:create_user, :log_in, :create_site]+ setup [:create_user, :log_in, :create_site, :set_scroll_depth_visible_at]test "returns graph_metric key for top stats with a page filter", %{conn: conn,@@ -1301,6 +1307,29 @@ defmodule PlausibleWeb.Api.StatsController.TopStatsTest do"top_stats"]end++ test "does not return scroll depth when site.scroll_depth_visible_at=nil", %{+ conn: conn,+ user: user+ } do+ site = new_site(owner: user)++ filters = Jason.encode!([[:is, "event:page", ["/"]]])++ top_stats =+ conn+ |> get("/api/stats/#{site.domain}/top-stats?filters=#{filters}")+ |> json_response(200)+ |> Map.get("top_stats")++ assert [+ %{"name" => "Unique visitors"},+ %{"name" => "Total visits"},+ %{"name" => "Total pageviews"},+ %{"name" => "Bounce rate"},+ %{"name" => "Time on page"}+ ] = top_stats+ endenddescribe "GET /api/stats/top-stats - filtered for goal" dodiff --git a/test/plausible_web/controllers/stats_controller_test.exs b/test/plausible_web/controllers/stats_controller_test.exsindex 10edffb667..5fd9f620a6 100644--- a/test/plausible_web/controllers/stats_controller_test.exs+++ b/test/plausible_web/controllers/stats_controller_test.exs@@ -79,6 +79,37 @@ defmodule PlausibleWeb.StatsControllerTest doconn = get(conn, "/test-site.com")assert html_response(conn, 404) =~ "There's nothing here"end++ test "site.scroll_depth_visible_at gets updated correctly", %{conn: conn} do+ site = new_site(public: true)++ populate_stats(site, [build(:pageview)])++ # No pageleaves yet - `scroll_depth_visible_at` will remain `nil`+ html =+ conn+ |> get("/#{site.domain}")+ |> html_response(200)++ assert text_of_attr(html, @react_container, "data-scroll-depth-visible") == "false"++ site = Repo.reload!(site)+ assert is_nil(site.scroll_depth_visible_at)++ populate_stats(site, [+ build(:pageview, user_id: 123),+ build(:pageleave, user_id: 123, scroll_depth: 20)+ ])++ # Pageleaves exist now - `scroll_depth_visible_at` gets set to `utc_now`+ html =+ conn+ |> get("/#{site.domain}")+ |> html_response(200)++ assert text_of_attr(html, @react_container, "data-scroll-depth-visible") == "true"+ assert not is_nil(Repo.reload!(site).scroll_depth_visible_at)+ endenddescribe "GET /:domain - as a logged in user" do@@ -183,7 +214,7 @@ defmodule PlausibleWeb.StatsControllerTest doenddescribe "GET /:domain/export" do- setup [:create_user, :create_site, :log_in]+ setup [:create_user, :create_site, :set_scroll_depth_visible_at, :log_in]test "exports all the necessary CSV files", %{conn: conn, site: site} doconn = get(conn, "/" <> site.domain <> "/export")@@ -637,8 +668,9 @@ defmodule PlausibleWeb.StatsControllerTest doenddescribe "GET /:domain/export - via shared link" do- test "exports data in zipped csvs", %{conn: conn} do- site = new_site(domain: "new-site.com")+ setup [:create_user, :create_site, :set_scroll_depth_visible_at]++ test "exports data in zipped csvs", %{conn: conn, site: site} dolink = insert(:shared_link, site: site)populate_exported_stats(site)@@ -648,7 +680,7 @@ defmodule PlausibleWeb.StatsControllerTest doenddescribe "GET /:domain/export - for past 6 months" do- setup [:create_user, :create_site, :log_in]+ setup [:create_user, :create_site, :set_scroll_depth_visible_at, :log_in]test "exports 6 months of data in zipped csvs", %{conn: conn, site: site} dopopulate_exported_stats(site)@@ -658,7 +690,7 @@ defmodule PlausibleWeb.StatsControllerTest doenddescribe "GET /:domain/export - with path filter" do- setup [:create_user, :create_site, :log_in]+ setup [:create_user, :create_site, :set_scroll_depth_visible_at, :log_in]test "exports filtered data in zipped csvs", %{conn: conn, site: site} dopopulate_exported_stats(site)@@ -722,6 +754,34 @@ defmodule PlausibleWeb.StatsControllerTest do[""]]end++ test "does not export scroll depth in visitors.csv if site.scroll_depth_visible_at=nil", %{+ conn: conn,+ user: user+ } do+ site = new_site(owner: user)++ filters = Jason.encode!([[:is, "event:page", ["/"]]])++ column_headers =+ conn+ |> get(+ "/#{site.domain}/export?period=custom&from=2021-01-01&to=2021-01-02&filters=#{filters}"+ )+ |> response(200)+ |> unzip_and_parse_csv(~c"visitors.csv")+ |> List.first()++ assert column_headers == [+ "date",+ "visitors",+ "pageviews",+ "visits",+ "views_per_visit",+ "bounce_rate",+ "visit_duration"+ ]+ endenddescribe "GET /:domain/export - with a custom prop filter" do@@ -1033,6 +1093,36 @@ defmodule PlausibleWeb.StatsControllerTest doconn = get(conn, "/share/#{site2.domain}/?auth=#{site1_link.slug}")assert response(conn, 404) =~ "nothing here"end++ test "site.scroll_depth_visible_at gets updated correctly", %{conn: conn} do+ site = insert(:site)+ link = insert(:shared_link, site: site)++ # No pageleaves yet - `scroll_depth_visible_at` will remain `nil`+ html =+ conn+ |> get("/share/#{site.domain}/?auth=#{link.slug}")+ |> html_response(200)++ assert text_of_attr(html, @react_container, "data-scroll-depth-visible") == "false"++ site = Repo.reload!(site)+ assert is_nil(site.scroll_depth_visible_at)++ populate_stats(site, [+ build(:pageview, user_id: 123),+ build(:pageleave, user_id: 123, scroll_depth: 20)+ ])++ # Pageleaves exist now - `scroll_depth_visible_at` gets set to `utc_now`+ html =+ conn+ |> get("/share/#{site.domain}/?auth=#{link.slug}")+ |> html_response(200)++ assert text_of_attr(html, @react_container, "data-scroll-depth-visible") == "true"+ assert not is_nil(Repo.reload!(site).scroll_depth_visible_at)+ endenddescribe "GET /share/:slug - backwards compatibility" dodiff --git a/test/support/test_utils.ex b/test/support/test_utils.exindex 362363a032..eacc31f8cb 100644--- a/test/support/test_utils.ex+++ b/test/support/test_utils.ex@@ -64,6 +64,11 @@ defmodule Plausible.TestUtils do{:ok, site_import: site_import}end+ def set_scroll_depth_visible_at(%{site: site}) do+ Plausible.Sites.set_scroll_depth_visible_at(site)+ :ok+ end+def create_api_key(%{user: user}) doapi_key = Factory.insert(:api_key, user: user)