Skip to content

Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.

Handle missing scroll depth #2 (#5017)

Karl-Aksel Puulmann7c6ba04a3cdb
diff --git a/assets/js/dashboard/site-context.test.tsx b/assets/js/dashboard/site-context.test.tsx
index 012ca33714..9c4109bee1 100644
--- a/assets/js/dashboard/site-context.test.tsx
+++ b/assets/js/dashboard/site-context.test.tsx
@@ -20,6 +20,7 @@ describe('parseSiteFromDataset', () => {
data-revenue-goals='[{"currency":"USD","display_name":"Purchase"}]'
data-funnels='[{"id":1,"name":"From homepage to login","steps_count":3}]'
data-has-props="true"
+ data-scroll-depth-visible="true"
data-logged-in="true"
data-stats-begin="2021-09-07"
data-native-stats-begin="2022-09-02"
@@ -43,6 +44,7 @@ describe('parseSiteFromDataset', () => {
revenueGoals: [{ currency: 'USD', display_name: 'Purchase' }],
funnels: [{ id: 1, name: 'From homepage to login', steps_count: 3 }],
hasProps: true,
+ scrollDepthVisible: true,
statsBegin: '2021-09-07',
nativeStatsBegin: '2022-09-02',
embedded: false,
diff --git a/assets/js/dashboard/site-context.tsx b/assets/js/dashboard/site-context.tsx
index 80e6f2ba52..215bcba48d 100644
--- a/assets/js/dashboard/site-context.tsx
+++ b/assets/js/dashboard/site-context.tsx
@@ -7,6 +7,7 @@ export function parseSiteFromDataset(dataset: DOMStringMap): PlausibleSite {
offset: parseInt(dataset.offset!, 10),
hasGoals: dataset.hasGoals === 'true',
hasProps: dataset.hasProps === 'true',
+ scrollDepthVisible: dataset.scrollDepthVisible === 'true',
funnelsAvailable: dataset.funnelsAvailable === 'true',
propsAvailable: dataset.propsAvailable === 'true',
conversionsOptedOut: dataset.conversionsOptedOut === 'true',
@@ -36,6 +37,7 @@ const siteContextDefaultValue = {
offset: 0,
hasGoals: false,
hasProps: false,
+ scrollDepthVisible: false,
funnelsAvailable: false,
propsAvailable: false,
conversionsOptedOut: false,
diff --git a/assets/js/dashboard/stats/graph/graph-util.js b/assets/js/dashboard/stats/graph/graph-util.js
index f80d105e78..56bc3047e7 100644
--- a/assets/js/dashboard/stats/graph/graph-util.js
+++ b/assets/js/dashboard/stats/graph/graph-util.js
@@ -16,7 +16,7 @@ export function getGraphableMetrics(query, site) {
return ["visitors", "events", "conversion_rate"]
} else if (isPageFilter) {
const pageFilterMetrics = ["visitors", "visits", "pageviews", "bounce_rate"]
- return site.flags.scroll_depth ? [...pageFilterMetrics, "scroll_depth"] : pageFilterMetrics
+ return site.scrollDepthVisible ? [...pageFilterMetrics, "scroll_depth"] : pageFilterMetrics
} else {
return ["visitors", "visits", "pageviews", "views_per_visit", "bounce_rate", "visit_duration"]
}
diff --git a/assets/js/dashboard/stats/modals/pages.js b/assets/js/dashboard/stats/modals/pages.js
index 6129fe444c..79485a7946 100644
--- a/assets/js/dashboard/stats/modals/pages.js
+++ b/assets/js/dashboard/stats/modals/pages.js
@@ -54,7 +54,7 @@ function PagesModal() {
metrics.createTimeOnPage()
]
- return site.flags.scroll_depth ? [...defaultMetrics, metrics.createScrollDepth()] : defaultMetrics
+ return site.scrollDepthVisible ? [...defaultMetrics, metrics.createScrollDepth()] : defaultMetrics
}
return (
diff --git a/assets/test-utils/app-context-providers.tsx b/assets/test-utils/app-context-providers.tsx
index 5d26d3f2f5..96819be939 100644
--- a/assets/test-utils/app-context-providers.tsx
+++ b/assets/test-utils/app-context-providers.tsx
@@ -26,6 +26,7 @@ export const TestContextProviders = ({
offset: 0,
hasGoals: false,
hasProps: false,
+ scrollDepthVisible: false,
funnelsAvailable: false,
propsAvailable: false,
conversionsOptedOut: false,
diff --git a/lib/plausible/exports.ex b/lib/plausible/exports.ex
index 46255d628c..f082e5f5f2 100644
--- a/lib/plausible/exports.ex
+++ b/lib/plausible/exports.ex
@@ -418,8 +418,7 @@ defmodule Plausible.Exports do
site = Plausible.Repo.get(Plausible.Site, site_id)
current_user = current_user_id && Plausible.Repo.get(Plausible.Auth.User, current_user_id)
- scroll_depth_enabled? =
- PlausibleWeb.Api.StatsController.scroll_depth_enabled?(site, current_user)
+ include_scroll_depth? = Plausible.Stats.ScrollDepth.check_feature_visible!(site, current_user)
base_q =
from(e in sampled("events_v2"),
@@ -429,7 +428,7 @@ defmodule Plausible.Exports do
order_by: selected_as(:date)
)
- if scroll_depth_enabled? do
+ if include_scroll_depth? do
max_scroll_depth_per_visitor_q =
from(e in "events_v2",
where: ^export_filter(site_id, date_range),
diff --git a/lib/plausible/site.ex b/lib/plausible/site.ex
index 37d0cb2187..11d89f4401 100644
--- a/lib/plausible/site.ex
+++ b/lib/plausible/site.ex
@@ -21,6 +21,7 @@ defmodule Plausible.Site do
field :conversions_enabled, :boolean, default: true
field :props_enabled, :boolean, default: true
field :funnels_enabled, :boolean, default: true
+ field :scroll_depth_visible_at, :naive_datetime
field :ingest_rate_limit_scale_seconds, :integer, default: 60
# default is set via changeset/2
diff --git a/lib/plausible/sites.ex b/lib/plausible/sites.ex
index 04af80c24e..6dc806f7e8 100644
--- a/lib/plausible/sites.ex
+++ b/lib/plausible/sites.ex
@@ -284,6 +284,14 @@ defmodule Plausible.Sites do
|> Repo.update!()
end
+ def set_scroll_depth_visible_at(site) do
+ utc_now = NaiveDateTime.utc_now(:second)
+
+ site
+ |> Ecto.Changeset.change(%{scroll_depth_visible_at: utc_now})
+ |> Repo.update()
+ end
+
def has_goals?(site) do
Repo.exists?(
from(g in Plausible.Goal,
diff --git a/lib/plausible/stats/scroll_depth.ex b/lib/plausible/stats/scroll_depth.ex
new file mode 100644
index 0000000000..246c8a8527
--- /dev/null
+++ b/lib/plausible/stats/scroll_depth.ex
@@ -0,0 +1,63 @@
+defmodule Plausible.Stats.ScrollDepth do
+ @moduledoc """
+ Module to check whether the scroll depth metric is available and visible for a site.
+ """
+
+ import Ecto.Query
+ require Logger
+
+ alias Plausible.ClickhouseRepo
+
+ def feature_available?(site, user) do
+ FunWithFlags.enabled?(:scroll_depth, for: user) ||
+ FunWithFlags.enabled?(:scroll_depth, for: site)
+ end
+
+ def feature_visible?(site, user) do
+ feature_available?(site, user) && not is_nil(site.scroll_depth_visible_at)
+ end
+
+ @doc """
+ Checks whether the scroll depth feature is visible for a site and updates the site record if it is.
+
+ Note this function queries ClickHouse and may take a while to complete.
+ """
+ def check_feature_visible!(site, user) do
+ cond do
+ not feature_available?(site, user) ->
+ false
+
+ not is_nil(site.scroll_depth_visible_at) ->
+ true
+
+ is_nil(site.scroll_depth_visible_at) ->
+ visible? = has_scroll_data_last_30d?(site)
+
+ if visible? do
+ Plausible.Sites.set_scroll_depth_visible_at(site)
+ end
+
+ visible?
+ end
+ end
+
+ defp has_scroll_data_last_30d?(site) do
+ try do
+ ClickhouseRepo.exists?(
+ from(e in "events_v2",
+ where:
+ e.site_id == ^site.id and
+ e.name == "pageleave" and
+ e.timestamp >= fragment("toStartOfDay(now() - toIntervalDay(30))") and
+ e.scroll_depth > 0 and e.scroll_depth <= 100
+ )
+ )
+ rescue
+ # Avoid propagating error to the user, bringing down the site.
+ error ->
+ Logger.error("Error checking scroll data for site #{site.id}: #{inspect(error)}")
+
+ false
+ end
+ end
+end
diff --git a/lib/plausible_web/controllers/api/stats_controller.ex b/lib/plausible_web/controllers/api/stats_controller.ex
index a4a279ed45..e9c2c3e6e2 100644
--- a/lib/plausible_web/controllers/api/stats_controller.ex
+++ b/lib/plausible_web/controllers/api/stats_controller.ex
@@ -389,16 +389,16 @@ defmodule PlausibleWeb.Api.StatsController do
page_filter? =
Filters.filtering_on_dimension?(query, "event:page", behavioral_filters: :ignore)
- scroll_depth_enabled? = scroll_depth_enabled?(site, current_user)
+ include_scroll_depth? = Plausible.Stats.ScrollDepth.feature_visible?(site, current_user)
metrics = [:visitors, :visits, :pageviews, :sample_percent]
metrics =
cond do
- page_filter? && scroll_depth_enabled? && query.include_imported ->
+ page_filter? && include_scroll_depth? && query.include_imported ->
metrics ++ [:scroll_depth]
- page_filter? && scroll_depth_enabled? ->
+ page_filter? && include_scroll_depth? ->
metrics ++ [:bounce_rate, :scroll_depth, :time_on_page]
page_filter? && query.include_imported ->
@@ -851,7 +851,7 @@ defmodule PlausibleWeb.Api.StatsController do
params = Map.put(params, "property", "event:page")
query = Query.from(site, params, debug_metadata(conn))
- include_scroll_depth? = scroll_depth_enabled?(site, current_user)
+ include_scroll_depth? = Plausible.Stats.ScrollDepth.feature_visible?(site, current_user)
extra_metrics =
cond do
@@ -1663,9 +1663,4 @@ defmodule PlausibleWeb.Api.StatsController do
defp toplevel_goal_filter?(query) do
Filters.filtering_on_dimension?(query, "event:goal", max_depth: 0)
end
-
- def scroll_depth_enabled?(site, user) do
- FunWithFlags.enabled?(:scroll_depth, for: user) ||
- FunWithFlags.enabled?(:scroll_depth, for: site)
- end
end
diff --git a/lib/plausible_web/controllers/stats_controller.ex b/lib/plausible_web/controllers/stats_controller.ex
index 336571fbd1..8d5dc4b8cb 100644
--- a/lib/plausible_web/controllers/stats_controller.ex
+++ b/lib/plausible_web/controllers/stats_controller.ex
@@ -52,12 +52,16 @@ defmodule PlausibleWeb.StatsController do
def stats(%{assigns: %{site: site}} = conn, _params) do
site = Plausible.Repo.preload(site, :owner)
+ current_user = conn.assigns[:current_user]
stats_start_date = Plausible.Sites.stats_start_date(site)
can_see_stats? = not Sites.locked?(site) or conn.assigns[:site_role] == :super_admin
demo = site.domain == PlausibleWeb.Endpoint.host()
dogfood_page_path = if demo, do: "/#{site.domain}", else: "/:dashboard"
skip_to_dashboard? = conn.params["skip_to_dashboard"] == "true"
+ scroll_depth_visible? =
+ Plausible.Stats.ScrollDepth.check_feature_visible!(site, current_user)
+
cond do
(stats_start_date && can_see_stats?) || (can_see_stats? && skip_to_dashboard?) ->
conn
@@ -68,11 +72,12 @@ defmodule PlausibleWeb.StatsController do
revenue_goals: list_revenue_goals(site),
funnels: list_funnels(site),
has_props: Plausible.Props.configured?(site),
+ scroll_depth_visible: scroll_depth_visible?,
stats_start_date: stats_start_date,
native_stats_start_date: NaiveDateTime.to_date(site.native_stats_start_at),
title: title(conn, site),
demo: demo,
- flags: get_flags(conn.assigns[:current_user], site),
+ flags: get_flags(current_user, site),
is_dbip: is_dbip(),
dogfood_page_path: dogfood_page_path,
load_dashboard_js: true
@@ -193,8 +198,8 @@ defmodule PlausibleWeb.StatsController do
defp csv_graph_metrics(query, site, current_user) do
include_scroll_depth? =
!query.include_imported &&
- PlausibleWeb.Api.StatsController.scroll_depth_enabled?(site, current_user) &&
- Filters.filtering_on_dimension?(query, "event:page", behavioral_filters: :ignore)
+ Filters.filtering_on_dimension?(query, "event:page", behavioral_filters: :ignore) &&
+ Plausible.Stats.ScrollDepth.feature_visible?(site, current_user)
{metrics, column_headers} =
if Filters.filtering_on_dimension?(query, "event:goal", max_depth: 0) do
@@ -341,9 +346,13 @@ defmodule PlausibleWeb.StatsController do
defp render_shared_link(conn, shared_link) do
cond do
!shared_link.site.locked ->
+ current_user = conn.assigns[:current_user]
shared_link = Plausible.Repo.preload(shared_link, site: :owner)
stats_start_date = Plausible.Sites.stats_start_date(shared_link.site)
+ scroll_depth_visible? =
+ Plausible.Stats.ScrollDepth.check_feature_visible!(shared_link.site, current_user)
+
conn
|> put_resp_header("x-robots-tag", "noindex, nofollow")
|> delete_resp_header("x-frame-options")
@@ -353,6 +362,7 @@ defmodule PlausibleWeb.StatsController do
revenue_goals: list_revenue_goals(shared_link.site),
funnels: list_funnels(shared_link.site),
has_props: Plausible.Props.configured?(shared_link.site),
+ scroll_depth_visible: scroll_depth_visible?,
stats_start_date: stats_start_date,
native_stats_start_date: NaiveDateTime.to_date(shared_link.site.native_stats_start_at),
title: title(conn, shared_link.site),
@@ -362,7 +372,7 @@ defmodule PlausibleWeb.StatsController do
embedded: conn.params["embed"] == "true",
background: conn.params["background"],
theme: conn.params["theme"],
- flags: get_flags(conn.assigns[:current_user], shared_link.site),
+ flags: get_flags(current_user, shared_link.site),
is_dbip: is_dbip(),
load_dashboard_js: true
)
diff --git a/lib/plausible_web/templates/stats/stats.html.heex b/lib/plausible_web/templates/stats/stats.html.heex
index 7575a60b8f..8bbecc573a 100644
--- a/lib/plausible_web/templates/stats/stats.html.heex
+++ b/lib/plausible_web/templates/stats/stats.html.heex
@@ -30,6 +30,7 @@
data-revenue-goals={Jason.encode!(@revenue_goals)}
data-funnels={Jason.encode!(@funnels)}
data-has-props={to_string(@has_props)}
+ data-scroll-depth-visible={to_string(@scroll_depth_visible)}
data-logged-in={to_string([email protected][:current_user])}
data-stats-begin={@stats_start_date}
data-native-stats-begin={@native_stats_start_date}
diff --git a/test/plausible/exports_test.exs b/test/plausible/exports_test.exs
index 9c320235ff..adf362d132 100644
--- a/test/plausible/exports_test.exs
+++ b/test/plausible/exports_test.exs
@@ -6,8 +6,10 @@ defmodule Plausible.ExportsTest do
# for e2e export->import tests please see Plausible.Imported.CSVImporterTest
describe "export_queries/2" do
- test "returns named ecto queries" do
- queries = Plausible.Exports.export_queries(_site_id = 1, nil)
+ setup [:create_user, :create_site]
+
+ test "returns named ecto queries", %{site: site} do
+ queries = Plausible.Exports.export_queries(site.id, nil)
assert queries |> Map.values() |> Enum.all?(&match?(%Ecto.Query{}, &1))
assert Map.keys(queries) == [
@@ -24,9 +26,9 @@ defmodule Plausible.ExportsTest do
]
end
- test "with date range" do
+ test "with date range", %{site: site} do
queries =
- Plausible.Exports.export_queries(_site_id = 1, nil,
+ Plausible.Exports.export_queries(site.id, nil,
date_range: Date.range(~D[2023-01-01], ~D[2024-03-12])
)
@@ -44,8 +46,8 @@ defmodule Plausible.ExportsTest do
]
end
- test "with custom extension" do
- queries = Plausible.Exports.export_queries(_site_id = 1, nil, extname: ".ch")
+ test "with custom extension", %{site: site} do
+ queries = Plausible.Exports.export_queries(site.id, nil, extname: ".ch")
assert Map.keys(queries) == [
"imported_browsers.ch",
diff --git a/test/plausible/imported/csv_importer_test.exs b/test/plausible/imported/csv_importer_test.exs
index efc4b05443..a205355be5 100644
--- a/test/plausible/imported/csv_importer_test.exs
+++ b/test/plausible/imported/csv_importer_test.exs
@@ -587,80 +587,28 @@ defmodule Plausible.Imported.CSVImporterTest do
process_csv.("fixture/markosaric_com_sessions_v2_2024_04_01_2024_04_30_dump.csv")
])
- # export archive to s3
- on_ee do
- assert {:ok, _job} =
- Plausible.Exports.schedule_s3_export(exported_site.id, nil, user.email)
- else
- assert {:ok, %{args: %{"local_path" => local_path}}} =
- Plausible.Exports.schedule_local_export(exported_site.id, user.email)
- end
-
- assert %{success: 1} = Oban.drain_queue(queue: :analytics_exports, with_safety: false)
-
- assert %{success: 1} =
- Oban.drain_queue(queue: :notify_exported_analytics, with_safety: false)
-
- # check mailbox
- assert_receive {:delivered_email, email}, _within = :timer.seconds(5)
- assert email.to == [{user.name, user.email}]
-
- assert email.html_body =~
- ~s[Please click <a href="http://localhost:8000/#{URI.encode_www_form(exported_site.domain)}/download/export">here</a>]
-
- assert email.text_body =~
- ~r[Please click here \(http://localhost:8000/#{URI.encode_www_form(exported_site.domain)}/download/export\) to start the download process.]
-
- # download archive
- on_ee do
- ExAws.request!(
- ExAws.S3.download_file(
- Plausible.S3.exports_bucket(),
- to_string(exported_site.id),
- Path.join(tmp_dir, "plausible-export.zip")
- )
- )
- else
- File.rename!(local_path, Path.join(tmp_dir, "plausible-export.zip"))
- end
-
- # unzip archive
- {:ok, files} =
- :zip.unzip(to_charlist(Path.join(tmp_dir, "plausible-export.zip")), cwd: tmp_dir)
-
- # upload csvs
- uploads =
- Enum.map(files, fn file ->
- on_ee do
- %{s3_url: s3_url} = Plausible.S3.import_presign_upload(imported_site.id, file)
- [bucket, key] = String.split(URI.parse(s3_url).path, "/", parts: 2)
- ExAws.request!(ExAws.S3.put_object(bucket, key, File.read!(file)))
- %{"filename" => Path.basename(file), "s3_url" => s3_url}
- else
- %{"filename" => Path.basename(file), "local_path" => file}
- end
- end)
-
- # run importer
- date_range = CSVImporter.date_range(uploads)
-
- {:ok, _job} =
- CSVImporter.new_import(imported_site, user,
- start_date: date_range.first,
- end_date: date_range.last,
- uploads: uploads,
- storage: on_ee(do: "s3", else: "local")
- )
-
- assert %{success: 1} = Oban.drain_queue(queue: :analytics_imports, with_safety: false)
+ initial_context = %{
+ user: user,
+ tmp_dir: tmp_dir,
+ exported_site: exported_site,
+ imported_site: imported_site
+ }
+
+ %{site_import: site_import} =
+ initial_context
+ |> export_archive()
+ |> assert_email_notification()
+ |> download_archive()
+ |> unzip_archive()
+ |> upload_csvs()
+ |> run_import()
- # validate import
assert %SiteImport{
start_date: ~D[2024-04-01],
end_date: ~D[2024-04-30],
source: :csv,
status: :completed
- } = Repo.get_by!(SiteImport, site_id: imported_site.id)
+ } = site_import
assert Plausible.Stats.Clickhouse.imported_pageview_count(exported_site) == 0
assert Plausible.Stats.Clickhouse.imported_pageview_count(imported_site) == 1745
@@ -1088,7 +1036,11 @@ defmodule Plausible.Imported.CSVImporterTest do
exported_site = new_site(owner: user)
imported_site = new_site(owner: user)
- t0 = ~N[2020-01-01 00:00:00]
+ t0 =
+ NaiveDateTime.utc_now(:second)
+ |> NaiveDateTime.add(-5, :day)
+ |> NaiveDateTime.beginning_of_day()
+
[t1, t2, t3] = for i <- 1..3, do: NaiveDateTime.add(t0, i, :minute)
stats =
@@ -1114,72 +1066,36 @@ defmodule Plausible.Imported.CSVImporterTest do
populate_stats(exported_site, stats)
- # export archive to s3
- on_ee do
- assert {:ok, _job} =
- Plausible.Exports.schedule_s3_export(exported_site.id, nil, user.email)
- else
- assert {:ok, %{args: %{"local_path" => local_path}}} =
- Plausible.Exports.schedule_local_export(exported_site.id, user.email)
- end
+ initial_context = %{
+ user: user,
+ tmp_dir: tmp_dir,
+ exported_site: exported_site,
+ imported_site: imported_site
+ }
- assert %{success: 1} = Oban.drain_queue(queue: :analytics_exports, with_safety: false)
+ %{site_import: site_import} =
+ initial_context
+ |> export_archive()
+ |> download_archive()
+ |> unzip_archive()
+ |> upload_csvs()
+ |> run_import()
- # download archive
- on_ee do
- ExAws.request!(
- ExAws.S3.download_file(
- Plausible.S3.exports_bucket(),
- to_string(exported_site.id),
- Path.join(tmp_dir, "plausible-export.zip")
- )
- )
- else
- File.rename!(local_path, Path.join(tmp_dir, "plausible-export.zip"))
- end
+ assert %NaiveDateTime{} =
+ Plausible.Repo.reload!(exported_site).scroll_depth_visible_at
- # unzip archive
- {:ok, files} =
- :zip.unzip(to_charlist(Path.join(tmp_dir, "plausible-export.zip")), cwd: tmp_dir)
-
- # upload csvs
- uploads =
- Enum.map(files, fn file ->
- on_ee do
- %{s3_url: s3_url} = Plausible.S3.import_presign_upload(imported_site.id, file)
- [bucket, key] = String.split(URI.parse(s3_url).path, "/", parts: 2)
- content = File.read!(file)
- ExAws.request!(ExAws.S3.put_object(bucket, key, content))
- %{"filename" => Path.basename(file), "s3_url" => s3_url}
- else
- %{
- "filename" => Path.basename(file),
- "local_path" => file
- }
- end
- end)
-
- # run importer
- date_range = CSVImporter.date_range(uploads)
-
- {:ok, _job} =
- CSVImporter.new_import(imported_site, user,
- start_date: date_range.first,
- end_date: date_range.last,
- uploads: uploads,
- storage: on_ee(do: "s3", else: "local")
- )
-
- assert %{success: 1} = Oban.drain_queue(queue: :analytics_imports, with_safety: false)
-
- # validate import
assert %SiteImport{
- start_date: ~D[2020-01-01],
- end_date: ~D[2020-01-02],
+ start_date: start_date,
+ end_date: end_date,
source: :csv,
status: :completed
- } = Repo.get_by!(SiteImport, site_id: imported_site.id)
+ } = site_import
+
+ expected_start_date = t0 |> NaiveDateTime.to_date()
+ expected_end_date = t0 |> NaiveDateTime.to_date() |> Date.add(1)
+ assert start_date == expected_start_date
+ assert end_date == expected_end_date
assert Plausible.Stats.Clickhouse.imported_pageview_count(exported_site) == 0
assert Plausible.Stats.Clickhouse.imported_pageview_count(imported_site) == 9
@@ -1196,13 +1112,18 @@ defmodule Plausible.Imported.CSVImporterTest do
)
|> Plausible.IngestRepo.all()
- assert %{date: ~D[2020-01-01], page: "/", scroll_depth: 20, pageleave_visitors: 1} in imported_data
+ assert %{date: expected_start_date, page: "/", scroll_depth: 20, pageleave_visitors: 1} in imported_data
- assert %{date: ~D[2020-01-01], page: "/another", scroll_depth: 50, pageleave_visitors: 2} in imported_data
+ assert %{
+ date: expected_start_date,
+ page: "/another",
+ scroll_depth: 50,
+ pageleave_visitors: 2
+ } in imported_data
- assert %{date: ~D[2020-01-01], page: "/blog", scroll_depth: 180, pageleave_visitors: 3} in imported_data
+ assert %{date: expected_start_date, page: "/blog", scroll_depth: 180, pageleave_visitors: 3} in imported_data
- assert %{date: ~D[2020-01-02], page: "/blog", scroll_depth: nil, pageleave_visitors: 0} in imported_data
+ assert %{date: expected_end_date, page: "/blog", scroll_depth: nil, pageleave_visitors: 0} in imported_data
# assert via stats queries that scroll_depth from imported
# data matches the scroll_depth from native data
@@ -1228,6 +1149,138 @@ defmodule Plausible.Imported.CSVImporterTest do
assert query_scroll_depth_per_page.(conn, exported_site) == expected_results
assert query_scroll_depth_per_page.(conn, imported_site) == expected_results
end
+
+ @tag :tmp_dir
+ test "does not include scroll depth without existing pageleave data", %{
+ user: user,
+ tmp_dir: tmp_dir
+ } do
+ exported_site = new_site(owner: user)
+
+ populate_stats(exported_site, [build(:pageview, timestamp: ~N[2021-01-01 00:00:00])])
+
+ context = %{
+ user: user,
+ tmp_dir: tmp_dir,
+ exported_site: exported_site,
+ imported_site: new_site(owner: user)
+ }
+
+ %{exported_files: exported_files} =
+ context
+ |> export_archive()
+ |> download_archive()
+ |> unzip_archive()
+
+ imported_pages_content =
+ exported_files
+ |> Enum.find(&String.contains?(&1, "imported_pages"))
+ |> File.read!()
+
+ assert is_nil(Plausible.Repo.reload!(exported_site).scroll_depth_visible_at)
+
+ refute imported_pages_content =~ "scroll_depth"
+ end
+ end
+
+ defp export_archive(%{user: user, exported_site: exported_site} = context) do
+ context =
+ on_ee do
+ assert {:ok, _job} =
+ Plausible.Exports.schedule_s3_export(exported_site.id, nil, user.email)
+
+ context
+ else
+ assert {:ok, %{args: %{"local_path" => local_path}}} =
+ Plausible.Exports.schedule_local_export(exported_site.id, user.email)
+
+ Map.put(context, :local_path, local_path)
+ end
+
+ Oban.drain_queue(queue: :analytics_exports, with_safety: false)
+
+ context
+ end
+
+ defp assert_email_notification(%{user: user, exported_site: exported_site} = context) do
+ assert %{success: 1} =
+ Oban.drain_queue(queue: :notify_exported_analytics, with_safety: false)
+
+ assert_receive {:delivered_email, email}, _within = :timer.seconds(5)
+ assert email.to == [{user.name, user.email}]
+
+ assert email.html_body =~
+ ~s[Please click <a href="http://localhost:8000/#{URI.encode_www_form(exported_site.domain)}/download/export">here</a>]
+
+ assert email.text_body =~
+ ~r[Please click here \(http://localhost:8000/#{URI.encode_www_form(exported_site.domain)}/download/export\) to start the download process.]
+
+ context
+ end
+
+ defp download_archive(%{tmp_dir: tmp_dir} = context) do
+ on_ee do
+ ExAws.request!(
+ ExAws.S3.download_file(
+ Plausible.S3.exports_bucket(),
+ to_string(context.exported_site.id),
+ Path.join(tmp_dir, "plausible-export.zip")
+ )
+ )
+ else
+ File.rename!(context.local_path, Path.join(tmp_dir, "plausible-export.zip"))
+ end
+
+ context
+ end
+
+ defp unzip_archive(%{tmp_dir: tmp_dir} = context) do
+ assert {:ok, files} =
+ :zip.unzip(to_charlist(Path.join(tmp_dir, "plausible-export.zip")), cwd: tmp_dir)
+
+ Map.put(context, :exported_files, files)
+ end
+
+ defp upload_csvs(%{exported_files: files} = context) do
+ uploads =
+ Enum.map(files, fn file ->
+ on_ee do
+ %{s3_url: s3_url} = Plausible.S3.import_presign_upload(context.imported_site.id, file)
+ [bucket, key] = String.split(URI.parse(s3_url).path, "/", parts: 2)
+ content = File.read!(file)
+ ExAws.request!(ExAws.S3.put_object(bucket, key, content))
+ %{"filename" => Path.basename(file), "s3_url" => s3_url}
+ else
+ %{
+ "filename" => Path.basename(file),
+ "local_path" => file
+ }
+ end
+ end)
+
+ Map.put(context, :uploaded_file_refs, uploads)
+ end
+
+ defp run_import(
+ %{uploaded_file_refs: uploads, imported_site: imported_site, user: user} = context
+ ) do
+ date_range = CSVImporter.date_range(uploads)
+
+ {:ok, _job} =
+ CSVImporter.new_import(imported_site, user,
+ start_date: date_range.first,
+ end_date: date_range.last,
+ uploads: uploads,
+ storage: on_ee(do: "s3", else: "local")
+ )
+
+ assert %{success: 1} = Oban.drain_queue(queue: :analytics_imports, with_safety: false)
+
+ site_import = Repo.get_by!(SiteImport, site_id: imported_site.id)
+
+ assert %SiteImport{source: :csv, status: :completed} = site_import
+
+ Map.put(context, :site_import, site_import)
end
defp clean_buckets(_context) do
diff --git a/test/plausible_web/controllers/api/stats_controller/imported_test.exs b/test/plausible_web/controllers/api/stats_controller/imported_test.exs
index 048fc2ccbf..828a855985 100644
--- a/test/plausible_web/controllers/api/stats_controller/imported_test.exs
+++ b/test/plausible_web/controllers/api/stats_controller/imported_test.exs
@@ -787,6 +787,8 @@ defmodule PlausibleWeb.Api.StatsController.ImportedTest do
site: site,
import_id: import_id
} do
+ Plausible.Sites.set_scroll_depth_visible_at(site)
+
populate_stats(site, [
build(:pageview,
pathname: "/",
diff --git a/test/plausible_web/controllers/api/stats_controller/pages_test.exs b/test/plausible_web/controllers/api/stats_controller/pages_test.exs
index 6a70985668..506f2a0698 100644
--- a/test/plausible_web/controllers/api/stats_controller/pages_test.exs
+++ b/test/plausible_web/controllers/api/stats_controller/pages_test.exs
@@ -1,10 +1,17 @@
defmodule PlausibleWeb.Api.StatsController.PagesTest do
use PlausibleWeb.ConnCase
+ use Plausible.Teams.Test
@user_id Enum.random(1000..9999)
describe "GET /api/stats/:domain/pages" do
- setup [:create_user, :log_in, :create_site, :create_legacy_site_import]
+ setup [
+ :create_user,
+ :log_in,
+ :create_site,
+ :create_legacy_site_import,
+ :set_scroll_depth_visible_at
+ ]
test "returns top pages by visitors", %{conn: conn, site: site} do
populate_stats(site, [
@@ -648,6 +655,27 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
]
end
+ test "does not return scroll depth (in detailed mode) when site.scroll_depth_visible_at=nil",
+ %{conn: conn, user: user} do
+ site = new_site(owner: user)
+
+ populate_stats(site, [build(:pageview)])
+
+ pages =
+ conn
+ |> get("/api/stats/#{site.domain}/pages?detailed=true")
+ |> json_response(200)
+ |> Map.get("results")
+
+ assert List.first(pages) == %{
+ "bounce_rate" => 100,
+ "name" => "/",
+ "pageviews" => 1,
+ "time_on_page" => nil,
+ "visitors" => 1
+ }
+ end
+
test "calculates scroll_depth from native and imported data combined", %{
conn: conn,
site: site
diff --git a/test/plausible_web/controllers/api/stats_controller/top_stats_test.exs b/test/plausible_web/controllers/api/stats_controller/top_stats_test.exs
index f9336b5b6a..286a1073f6 100644
--- a/test/plausible_web/controllers/api/stats_controller/top_stats_test.exs
+++ b/test/plausible_web/controllers/api/stats_controller/top_stats_test.exs
@@ -478,7 +478,13 @@ defmodule PlausibleWeb.Api.StatsController.TopStatsTest do
end
describe "GET /api/stats/top-stats - with imported data" do
- setup [:create_user, :log_in, :create_site, :create_legacy_site_import]
+ setup [
+ :create_user,
+ :log_in,
+ :create_site,
+ :create_legacy_site_import,
+ :set_scroll_depth_visible_at
+ ]
test "returns divisible metrics as 0 when no stats exist", %{
site: site,
@@ -888,7 +894,7 @@ defmodule PlausibleWeb.Api.StatsController.TopStatsTest do
end
describe "GET /api/stats/top-stats - filters" do
- setup [:create_user, :log_in, :create_site]
+ setup [:create_user, :log_in, :create_site, :set_scroll_depth_visible_at]
test "returns graph_metric key for top stats with a page filter", %{
conn: conn,
@@ -1301,6 +1307,29 @@ defmodule PlausibleWeb.Api.StatsController.TopStatsTest do
"top_stats"
]
end
+
+ test "does not return scroll depth when site.scroll_depth_visible_at=nil", %{
+ conn: conn,
+ user: user
+ } do
+ site = new_site(owner: user)
+
+ filters = Jason.encode!([[:is, "event:page", ["/"]]])
+
+ top_stats =
+ conn
+ |> get("/api/stats/#{site.domain}/top-stats?filters=#{filters}")
+ |> json_response(200)
+ |> Map.get("top_stats")
+
+ assert [
+ %{"name" => "Unique visitors"},
+ %{"name" => "Total visits"},
+ %{"name" => "Total pageviews"},
+ %{"name" => "Bounce rate"},
+ %{"name" => "Time on page"}
+ ] = top_stats
+ end
end
describe "GET /api/stats/top-stats - filtered for goal" do
diff --git a/test/plausible_web/controllers/stats_controller_test.exs b/test/plausible_web/controllers/stats_controller_test.exs
index 10edffb667..5fd9f620a6 100644
--- a/test/plausible_web/controllers/stats_controller_test.exs
+++ b/test/plausible_web/controllers/stats_controller_test.exs
@@ -79,6 +79,37 @@ defmodule PlausibleWeb.StatsControllerTest do
conn = get(conn, "/test-site.com")
assert html_response(conn, 404) =~ "There's nothing here"
end
+
+ test "site.scroll_depth_visible_at gets updated correctly", %{conn: conn} do
+ site = new_site(public: true)
+
+ populate_stats(site, [build(:pageview)])
+
+ # No pageleaves yet - `scroll_depth_visible_at` will remain `nil`
+ html =
+ conn
+ |> get("/#{site.domain}")
+ |> html_response(200)
+
+ assert text_of_attr(html, @react_container, "data-scroll-depth-visible") == "false"
+
+ site = Repo.reload!(site)
+ assert is_nil(site.scroll_depth_visible_at)
+
+ populate_stats(site, [
+ build(:pageview, user_id: 123),
+ build(:pageleave, user_id: 123, scroll_depth: 20)
+ ])
+
+ # Pageleaves exist now - `scroll_depth_visible_at` gets set to `utc_now`
+ html =
+ conn
+ |> get("/#{site.domain}")
+ |> html_response(200)
+
+ assert text_of_attr(html, @react_container, "data-scroll-depth-visible") == "true"
+ assert not is_nil(Repo.reload!(site).scroll_depth_visible_at)
+ end
end
describe "GET /:domain - as a logged in user" do
@@ -183,7 +214,7 @@ defmodule PlausibleWeb.StatsControllerTest do
end
describe "GET /:domain/export" do
- setup [:create_user, :create_site, :log_in]
+ setup [:create_user, :create_site, :set_scroll_depth_visible_at, :log_in]
test "exports all the necessary CSV files", %{conn: conn, site: site} do
conn = get(conn, "/" <> site.domain <> "/export")
@@ -637,8 +668,9 @@ defmodule PlausibleWeb.StatsControllerTest do
end
describe "GET /:domain/export - via shared link" do
- test "exports data in zipped csvs", %{conn: conn} do
- site = new_site(domain: "new-site.com")
+ setup [:create_user, :create_site, :set_scroll_depth_visible_at]
+
+ test "exports data in zipped csvs", %{conn: conn, site: site} do
link = insert(:shared_link, site: site)
populate_exported_stats(site)
@@ -648,7 +680,7 @@ defmodule PlausibleWeb.StatsControllerTest do
end
describe "GET /:domain/export - for past 6 months" do
- setup [:create_user, :create_site, :log_in]
+ setup [:create_user, :create_site, :set_scroll_depth_visible_at, :log_in]
test "exports 6 months of data in zipped csvs", %{conn: conn, site: site} do
populate_exported_stats(site)
@@ -658,7 +690,7 @@ defmodule PlausibleWeb.StatsControllerTest do
end
describe "GET /:domain/export - with path filter" do
- setup [:create_user, :create_site, :log_in]
+ setup [:create_user, :create_site, :set_scroll_depth_visible_at, :log_in]
test "exports filtered data in zipped csvs", %{conn: conn, site: site} do
populate_exported_stats(site)
@@ -722,6 +754,34 @@ defmodule PlausibleWeb.StatsControllerTest do
[""]
]
end
+
+ test "does not export scroll depth in visitors.csv if site.scroll_depth_visible_at=nil", %{
+ conn: conn,
+ user: user
+ } do
+ site = new_site(owner: user)
+
+ filters = Jason.encode!([[:is, "event:page", ["/"]]])
+
+ column_headers =
+ conn
+ |> get(
+ "/#{site.domain}/export?period=custom&from=2021-01-01&to=2021-01-02&filters=#{filters}"
+ )
+ |> response(200)
+ |> unzip_and_parse_csv(~c"visitors.csv")
+ |> List.first()
+
+ assert column_headers == [
+ "date",
+ "visitors",
+ "pageviews",
+ "visits",
+ "views_per_visit",
+ "bounce_rate",
+ "visit_duration"
+ ]
+ end
end
describe "GET /:domain/export - with a custom prop filter" do
@@ -1033,6 +1093,36 @@ defmodule PlausibleWeb.StatsControllerTest do
conn = get(conn, "/share/#{site2.domain}/?auth=#{site1_link.slug}")
assert response(conn, 404) =~ "nothing here"
end
+
+ test "site.scroll_depth_visible_at gets updated correctly", %{conn: conn} do
+ site = insert(:site)
+ link = insert(:shared_link, site: site)
+
+ # No pageleaves yet - `scroll_depth_visible_at` will remain `nil`
+ html =
+ conn
+ |> get("/share/#{site.domain}/?auth=#{link.slug}")
+ |> html_response(200)
+
+ assert text_of_attr(html, @react_container, "data-scroll-depth-visible") == "false"
+
+ site = Repo.reload!(site)
+ assert is_nil(site.scroll_depth_visible_at)
+
+ populate_stats(site, [
+ build(:pageview, user_id: 123),
+ build(:pageleave, user_id: 123, scroll_depth: 20)
+ ])
+
+ # Pageleaves exist now - `scroll_depth_visible_at` gets set to `utc_now`
+ html =
+ conn
+ |> get("/share/#{site.domain}/?auth=#{link.slug}")
+ |> html_response(200)
+
+ assert text_of_attr(html, @react_container, "data-scroll-depth-visible") == "true"
+ assert not is_nil(Repo.reload!(site).scroll_depth_visible_at)
+ end
end
describe "GET /share/:slug - backwards compatibility" do
diff --git a/test/support/test_utils.ex b/test/support/test_utils.ex
index 362363a032..eacc31f8cb 100644
--- a/test/support/test_utils.ex
+++ b/test/support/test_utils.ex
@@ -64,6 +64,11 @@ defmodule Plausible.TestUtils do
{:ok, site_import: site_import}
end
+ def set_scroll_depth_visible_at(%{site: site}) do
+ Plausible.Sites.set_scroll_depth_visible_at(site)
+ :ok
+ end
+
def create_api_key(%{user: user}) do
api_key = Factory.insert(:api_key, user: user)