Skip to content

Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.

Scroll depth: ingestion, query and import/export changes (#5072)

Karl-Aksel Puulmann53581973d05e
diff --git a/lib/plausible/exports.ex b/lib/plausible/exports.ex
index 6cb93f3739..f9355a1574 100644
--- a/lib/plausible/exports.ex
+++ b/lib/plausible/exports.ex
@@ -284,11 +284,7 @@ defmodule Plausible.Exports do
defmacrop visit_duration(t) do
quote do
selected_as(
- fragment(
- "toUInt64(round(greatest(sum(?*?),0)*any(_sample_factor)))",
- unquote(t).sign,
- unquote(t).duration
- ),
+ scale_sample(fragment("greatest(sum(?*?),0)", unquote(t).sign, unquote(t).duration)),
:visit_duration
)
end
@@ -297,7 +293,7 @@ defmodule Plausible.Exports do
defmacrop visitors(t) do
quote do
selected_as(
- fragment("toUInt64(round(uniq(?)*any(_sample_factor)))", unquote(t).user_id),
+ scale_sample(fragment("uniq(?)", unquote(t).user_id)),
:visitors
)
end
@@ -306,7 +302,7 @@ defmodule Plausible.Exports do
defmacrop visits(t) do
quote do
selected_as(
- fragment("toUInt64(round(greatest(sum(?),0)*any(_sample_factor)))", unquote(t).sign),
+ scale_sample(fragment("greatest(sum(?),0)", unquote(t).sign)),
:visits
)
end
@@ -315,10 +311,12 @@ defmodule Plausible.Exports do
defmacrop bounces(t) do
quote do
selected_as(
- fragment(
- "toUInt32(round(greatest(sum(?*?),0)*any(_sample_factor)))",
- unquote(t).sign,
- unquote(t).is_bounce
+ scale_sample(
+ fragment(
+ "greatest(sum(?*?),0)",
+ unquote(t).sign,
+ unquote(t).is_bounce
+ )
),
:bounces
)
@@ -328,10 +326,12 @@ defmodule Plausible.Exports do
defmacrop pageviews(t) do
quote do
selected_as(
- fragment(
- "toUInt64(round(greatest(sum(?*?),0)*any(_sample_factor)))",
- unquote(t).sign,
- unquote(t).pageviews
+ scale_sample(
+ fragment(
+ "greatest(sum(?*?),0)",
+ unquote(t).sign,
+ unquote(t).pageviews
+ )
),
:pageviews
)
@@ -359,7 +359,7 @@ defmodule Plausible.Exports do
date: date(e.timestamp, ^timezone),
pageviews:
selected_as(
- fragment("toUInt64(round(countIf(?='pageview')*any(_sample_factor)))", e.name),
+ scale_sample(fragment("countIf(?='pageview')", e.name)),
:pageviews
)
}
@@ -428,33 +428,27 @@ defmodule Plausible.Exports do
order_by: selected_as(:date)
)
- # :TODO: To be removed in the next PR
- if include_scroll_depth? and false do
- max_scroll_depth_per_visitor_q =
+ if include_scroll_depth? do
+ max_scroll_depth_per_session_q =
from(e in "events_v2",
where: ^export_filter(site_id, date_range),
where: e.name == "engagement" and e.scroll_depth <= 100,
select: %{
date: date(e.timestamp, ^timezone),
page: selected_as(e.pathname, :page),
- user_id: e.user_id,
+ session_id: e.session_id,
max_scroll_depth: max(e.scroll_depth)
},
- group_by: [e.user_id, selected_as(:date), selected_as(:page)]
+ group_by: [e.session_id, selected_as(:date), selected_as(:page)]
)
scroll_depth_q =
- from(p in subquery(max_scroll_depth_per_visitor_q),
+ from(p in subquery(max_scroll_depth_per_session_q),
select: %{
date: p.date,
page: p.page,
- scroll_depth:
- fragment(
- "if(isNull(sum(?)), NULL, toUInt64(sum(?)))",
- p.max_scroll_depth,
- p.max_scroll_depth
- ),
- pageleave_visitors: fragment("uniq(?)", p.user_id)
+ total_scroll_depth: fragment("sum(?)", p.max_scroll_depth),
+ total_scroll_depth_visits: fragment("uniq(?)", p.session_id)
},
group_by: [:date, :page]
)
@@ -467,13 +461,13 @@ defmodule Plausible.Exports do
selected_as(fragment("any(?)", e.hostname), :hostname),
selected_as(e.pathname, :page),
selected_as(
- fragment("toUInt64(round(uniq(?)*any(_sample_factor)))", e.session_id),
+ scale_sample(fragment("uniq(?)", e.session_id)),
:visits
),
visitors(e),
- selected_as(fragment("toUInt64(round(count()*any(_sample_factor)))"), :pageviews),
- selected_as(fragment("any(?)", s.scroll_depth), :scroll_depth),
- selected_as(fragment("any(?)", s.pageleave_visitors), :pageleave_visitors)
+ selected_as(scale_sample(fragment("count()")), :pageviews),
+ selected_as(fragment("any(?)", s.total_scroll_depth), :total_scroll_depth),
+ selected_as(fragment("any(?)", s.total_scroll_depth_visits), :total_scroll_depth_visits)
]
)
else
@@ -483,11 +477,11 @@ defmodule Plausible.Exports do
selected_as(fragment("any(?)", e.hostname), :hostname),
selected_as(e.pathname, :page),
selected_as(
- fragment("toUInt64(round(uniq(?)*any(_sample_factor)))", e.session_id),
+ scale_sample(fragment("uniq(?)", e.session_id)),
:visits
),
visitors(e),
- selected_as(fragment("toUInt64(round(count()*any(_sample_factor)))"), :pageviews)
+ selected_as(scale_sample(fragment("count()")), :pageviews)
])
end
end
@@ -502,7 +496,7 @@ defmodule Plausible.Exports do
s.entry_page,
visitors(s),
selected_as(
- fragment("toUInt64(round(greatest(sum(?),0)*any(_sample_factor)))", s.sign),
+ scale_sample(fragment("greatest(sum(?),0)", s.sign)),
:entrances
),
visit_duration(s),
@@ -522,7 +516,7 @@ defmodule Plausible.Exports do
visitors(s),
visit_duration(s),
selected_as(
- fragment("toUInt64(round(greatest(sum(?),0)*any(_sample_factor)))", s.sign),
+ scale_sample(fragment("greatest(sum(?),0)", s.sign)),
:exits
),
bounces(s),
@@ -563,7 +557,7 @@ defmodule Plausible.Exports do
:path
),
visitors(e),
- selected_as(fragment("toUInt64(round(count()*any(_sample_factor)))"), :events)
+ selected_as(scale_sample(fragment("count()")), :events)
]
end
diff --git a/lib/plausible/imported/csv_importer.ex b/lib/plausible/imported/csv_importer.ex
index ff763e304b..d3ad7bffa0 100644
--- a/lib/plausible/imported/csv_importer.ex
+++ b/lib/plausible/imported/csv_importer.ex
@@ -5,6 +5,7 @@ defmodule Plausible.Imported.CSVImporter do
"""
use Plausible.Imported.Importer
+ import Ecto.Query, only: [from: 2]
@impl true
def name(), do: :csv
@@ -54,6 +55,26 @@ defmodule Plausible.Imported.CSVImporter do
{:error, Exception.message(e)}
end
+ def on_success(site_import, _extra_data) do
+ has_scroll_depth? =
+ Plausible.ClickhouseRepo.exists?(
+ from(i in "imported_pages",
+ where: i.site_id == ^site_import.site_id,
+ where: i.import_id == ^site_import.id,
+ where: i.total_scroll_depth_visits > 0,
+ select: 1
+ )
+ )
+
+ if has_scroll_depth? do
+ site_import
+ |> Ecto.Changeset.change(%{has_scroll_depth: true})
+ |> Plausible.Repo.update!()
+ end
+
+ :ok
+ end
+
defp import_s3(ch, site_import, uploads) do
%{
id: import_id,
@@ -75,7 +96,7 @@ defmodule Plausible.Imported.CSVImporter do
statement =
"""
INSERT INTO {table:Identifier}(site_id,import_id,#{s3_columns}) \
- SELECT {site_id:UInt64}, {import_id:UInt64}, * \
+ SELECT {site_id:UInt64}, {import_id:UInt64}, #{s3_columns} \
FROM s3({s3_url:String},{s3_access_key_id:String},{s3_secret_access_key:String},{s3_format:String},{s3_structure:String}) \
WHERE date >= {start_date:Date} AND date <= {end_date:Date}\
"""
@@ -119,7 +140,7 @@ defmodule Plausible.Imported.CSVImporter do
statement =
"""
INSERT INTO {table:Identifier}(site_id,import_id,#{input_columns}) \
- SELECT {site_id:UInt64}, {import_id:UInt64}, * \
+ SELECT {site_id:UInt64}, {import_id:UInt64}, #{input_columns} \
FROM input({input_structure:String}) \
WHERE date >= {start_date:Date} AND date <= {end_date:Date} \
FORMAT CSVWithNames
@@ -161,7 +182,7 @@ defmodule Plausible.Imported.CSVImporter do
"imported_operating_systems" =>
"date Date, operating_system String, operating_system_version String, visitors UInt64, visits UInt64, visit_duration UInt64, bounces UInt32, pageviews UInt64",
"imported_pages" =>
- "date Date, hostname String, page String, visits UInt64, visitors UInt64, pageviews UInt64",
+ "date Date, hostname String, page String, visits UInt64, visitors UInt64, pageviews UInt64, total_scroll_depth UInt64, total_scroll_depth_visits UInt64",
"imported_sources" =>
"date Date, source String, referrer String, utm_source String, utm_medium String, utm_campaign String, utm_content String, utm_term String, pageviews UInt64, visitors UInt64, visits UInt64, visit_duration UInt64, bounces UInt32",
"imported_visitors" =>
diff --git a/lib/plausible/imported/page.ex b/lib/plausible/imported/page.ex
index 747c5e0306..6e27c1bd89 100644
--- a/lib/plausible/imported/page.ex
+++ b/lib/plausible/imported/page.ex
@@ -15,7 +15,7 @@ defmodule Plausible.Imported.Page do
field :pageviews, Ch, type: "UInt64"
field :exits, Ch, type: "UInt64"
field :time_on_page, Ch, type: "UInt64"
- # field :scroll_depth, Ch, type: "Nullable(UInt64)"
- # field :pageleave_visitors, Ch, type: "UInt64"
+ field :total_scroll_depth, Ch, type: "UInt64"
+ field :total_scroll_depth_visits, Ch, type: "UInt64"
end
end
diff --git a/lib/plausible/ingestion/event.ex b/lib/plausible/ingestion/event.ex
index c9a52edd1f..adbd488f82 100644
--- a/lib/plausible/ingestion/event.ex
+++ b/lib/plausible/ingestion/event.ex
@@ -35,7 +35,7 @@ defmodule Plausible.Ingestion.Event do
| :site_hostname_allowlist
| :verification_agent
| :lock_timeout
- | :no_session_for_pageleave
+ | :no_session_for_engagement
@type t() :: %__MODULE__{
domain: String.t() | nil,
@@ -385,8 +385,8 @@ defmodule Plausible.Ingestion.Event do
)
case session_result do
- {:ok, :no_session_for_pageleave} ->
- drop(event, :no_session_for_pageleave)
+ {:ok, :no_session_for_engagement} ->
+ drop(event, :no_session_for_engagement)
{:error, :timeout} ->
drop(event, :lock_timeout)
diff --git a/lib/plausible/ingestion/request.ex b/lib/plausible/ingestion/request.ex
index b0ec08147b..7ae04ac0a4 100644
--- a/lib/plausible/ingestion/request.ex
+++ b/lib/plausible/ingestion/request.ex
@@ -250,7 +250,7 @@ defmodule Plausible.Ingestion.Request do
end
defp put_scroll_depth(changeset, %{} = request_body) do
- if Changeset.get_field(changeset, :event_name) in ["pageleave", "engagement"] do
+ if Changeset.get_field(changeset, :event_name) == "engagement" do
scroll_depth =
case request_body["sd"] do
sd when is_integer(sd) and sd >= 0 and sd <= 100 -> sd
diff --git a/lib/plausible/session/cache_store.ex b/lib/plausible/session/cache_store.ex
index 5e0a1eaf6f..54a5bb488f 100644
--- a/lib/plausible/session/cache_store.ex
+++ b/lib/plausible/session/cache_store.ex
@@ -25,15 +25,14 @@ defmodule Plausible.Session.CacheStore do
)
end
- defp handle_event(%{name: name} = event, found_session, _, _)
- when name in ["pageleave", "engagement"] do
+ defp handle_event(%{name: "engagement"} = event, found_session, _, _) do
if found_session do
# Make sure the session is kept active in the in-memory session cache
refresh_session_cache(found_session, event.timestamp)
found_session
else
- :no_session_for_pageleave
+ :no_session_for_engagement
end
end
diff --git a/lib/plausible/stats/imported/sql/expression.ex b/lib/plausible/stats/imported/sql/expression.ex
index 69b8dbf34b..95d005e676 100644
--- a/lib/plausible/stats/imported/sql/expression.ex
+++ b/lib/plausible/stats/imported/sql/expression.ex
@@ -115,12 +115,12 @@ defmodule Plausible.Stats.Imported.SQL.Expression do
wrap_alias([i], %{pageviews: sum(i.pageviews), __internal_visits: sum(i.visits)})
end
- # defp select_metric(:scroll_depth, "imported_pages") do
- # wrap_alias([i], %{
- # scroll_depth_sum: sum(i.scroll_depth),
- # pageleave_visitors: sum(i.pageleave_visitors)
- # })
- # end
+ defp select_metric(:scroll_depth, "imported_pages") do
+ wrap_alias([i], %{
+ total_scroll_depth: sum(i.total_scroll_depth),
+ total_scroll_depth_visits: sum(i.total_scroll_depth_visits)
+ })
+ end
defp select_metric(_metric, _table), do: %{}
@@ -364,6 +364,10 @@ defmodule Plausible.Stats.Imported.SQL.Expression do
# information from imported data here already.
def select_joined_metrics(q, [:scroll_depth | rest]) do
q
+ |> select_merge_as([s, i], %{
+ __imported_total_scroll_depth: i.total_scroll_depth,
+ __imported_total_scroll_depth_visits: i.total_scroll_depth_visits
+ })
|> select_joined_metrics(rest)
end
diff --git a/lib/plausible/stats/sql/expression.ex b/lib/plausible/stats/sql/expression.ex
index b69b2454e0..4708d8027f 100644
--- a/lib/plausible/stats/sql/expression.ex
+++ b/lib/plausible/stats/sql/expression.ex
@@ -191,27 +191,25 @@ defmodule Plausible.Stats.SQL.Expression do
def event_metric(:pageviews) do
wrap_alias([e], %{
- pageviews:
- fragment("toUInt64(round(countIf(? = 'pageview') * any(_sample_factor)))", e.name)
+ pageviews: scale_sample(fragment("countIf(? = 'pageview')", e.name))
})
end
def event_metric(:events) do
wrap_alias([e], %{
- events:
- fragment("toUInt64(round(countIf(? != 'engagement') * any(_sample_factor)))", e.name)
+ events: scale_sample(fragment("countIf(? != 'engagement')", e.name))
})
end
def event_metric(:visitors) do
wrap_alias([e], %{
- visitors: fragment("toUInt64(round(uniq(?) * any(_sample_factor)))", e.user_id)
+ visitors: scale_sample(fragment("uniq(?)", e.user_id))
})
end
def event_metric(:visits) do
wrap_alias([e], %{
- visits: fragment("toUInt64(round(uniq(?) * any(_sample_factor)))", e.session_id)
+ visits: scale_sample(fragment("uniq(?)", e.session_id))
})
end
@@ -273,26 +271,25 @@ defmodule Plausible.Stats.SQL.Expression do
def session_metric(:visits, _query) do
wrap_alias([s], %{
- visits: fragment("toUInt64(round(sum(?) * any(_sample_factor)))", s.sign)
+ visits: scale_sample(fragment("sum(?)", s.sign))
})
end
def session_metric(:pageviews, _query) do
wrap_alias([s], %{
- pageviews:
- fragment("toUInt64(round(sum(? * ?) * any(_sample_factor)))", s.sign, s.pageviews)
+ pageviews: scale_sample(fragment("sum(? * ?)", s.sign, s.pageviews))
})
end
def session_metric(:events, _query) do
wrap_alias([s], %{
- events: fragment("toUInt64(round(sum(? * ?) * any(_sample_factor)))", s.sign, s.events)
+ events: scale_sample(fragment("sum(? * ?)", s.sign, s.events))
})
end
def session_metric(:visitors, _query) do
wrap_alias([s], %{
- visitors: fragment("toUInt64(round(uniq(?) * any(_sample_factor)))", s.user_id)
+ visitors: scale_sample(fragment("uniq(?)", s.user_id))
})
end
diff --git a/lib/plausible/stats/sql/fragments.ex b/lib/plausible/stats/sql/fragments.ex
index 7092db19b1..8e39fb823f 100644
--- a/lib/plausible/stats/sql/fragments.ex
+++ b/lib/plausible/stats/sql/fragments.ex
@@ -10,15 +10,21 @@ defmodule Plausible.Stats.SQL.Fragments do
end
end
+ defmacro scale_sample(sampled_fragment) do
+ quote do
+ fragment("toUInt64(round(? * any(_sample_factor)))", unquote(sampled_fragment))
+ end
+ end
+
defmacro uniq(user_id) do
quote do
- fragment("toUInt64(round(uniq(?) * any(_sample_factor)))", unquote(user_id))
+ scale_sample(fragment("uniq(?)", unquote(user_id)))
end
end
defmacro total() do
quote do
- fragment("toUInt64(round(count(*) * any(_sample_factor)))")
+ scale_sample(fragment("count()"))
end
end
diff --git a/lib/plausible/stats/sql/special_metrics.ex b/lib/plausible/stats/sql/special_metrics.ex
index 8fd32065f2..8ade1c18a5 100644
--- a/lib/plausible/stats/sql/special_metrics.ex
+++ b/lib/plausible/stats/sql/special_metrics.ex
@@ -129,15 +129,15 @@ defmodule Plausible.Stats.SQL.SpecialMetrics do
def maybe_add_scroll_depth(q, site, query) do
if :scroll_depth in query.metrics do
- max_per_visitor_q =
+ max_per_session_q =
Base.base_event_query(site, query)
|> where([e], e.name == "engagement" and e.scroll_depth <= 100)
|> select([e], %{
- user_id: e.user_id,
+ session_id: e.session_id,
max_scroll_depth: max(e.scroll_depth)
})
|> SQL.QueryBuilder.build_group_by(:events, query)
- |> group_by([e], e.user_id)
+ |> group_by([e], e.session_id)
dim_shortnames = Enum.map(query.dimensions, fn dim -> shortname(query, dim) end)
@@ -150,12 +150,13 @@ defmodule Plausible.Stats.SQL.SpecialMetrics do
dim_shortnames
|> Enum.map(fn dim -> dynamic([p], field(p, ^dim)) end)
- scroll_depth_sum_q =
- subquery(max_per_visitor_q)
- |> select([p], %{
- scroll_depth_sum:
- fragment("if(count(?) = 0, NULL, sum(?))", p.user_id, p.max_scroll_depth),
- pageleave_visitors: fragment("count(?)", p.user_id)
+ total_scroll_depth_q =
+ subquery(max_per_session_q)
+ |> select([], %{})
+ |> select_merge_as([p], %{
+ # Note: No need to upscale sample size here since it would end up cancelling out due to the result being an average
+ total_scroll_depth: fragment("sum(?)", p.max_scroll_depth),
+ total_scroll_depth_visits: fragment("uniq(?)", p.session_id)
})
|> select_merge(^dim_select)
|> group_by(^dim_group_by)
@@ -171,7 +172,7 @@ defmodule Plausible.Stats.SQL.SpecialMetrics do
end
joined_q =
- join(q, :left, [e], s in subquery(scroll_depth_sum_q), on: ^join_on_dim_condition)
+ join(q, :left, [e], s in subquery(total_scroll_depth_q), on: ^join_on_dim_condition)
if query.include_imported do
joined_q
@@ -179,17 +180,14 @@ defmodule Plausible.Stats.SQL.SpecialMetrics do
scroll_depth:
fragment(
"""
- case
- when isNotNull(?) then
- toUInt8(round(? / ?))
- else
- NULL
- end
+ if(? + ? > 0, toInt8(round((? + ?) / (? + ?))), NULL)
""",
- # Case 3: Only native scroll depth sum is present
- s.scroll_depth_sum,
- s.scroll_depth_sum,
- s.pageleave_visitors
+ s.total_scroll_depth_visits,
+ selected_as(:__imported_total_scroll_depth_visits),
+ s.total_scroll_depth,
+ selected_as(:__imported_total_scroll_depth),
+ s.total_scroll_depth_visits,
+ selected_as(:__imported_total_scroll_depth_visits)
)
})
else
@@ -198,9 +196,9 @@ defmodule Plausible.Stats.SQL.SpecialMetrics do
scroll_depth:
fragment(
"if(any(?) > 0, toUInt8(round(any(?) / any(?))), NULL)",
- s.pageleave_visitors,
- s.scroll_depth_sum,
- s.pageleave_visitors
+ s.total_scroll_depth_visits,
+ s.total_scroll_depth,
+ s.total_scroll_depth_visits
)
})
end
@@ -248,7 +246,7 @@ defmodule Plausible.Stats.SQL.SpecialMetrics do
defp total_visitors(site, query) do
Base.base_event_query(site, query)
|> select([e],
- total_visitors: fragment("toUInt64(round(uniq(?) * any(_sample_factor)))", e.user_id)
+ total_visitors: scale_sample(fragment("uniq(?)", e.user_id))
)
end
end
diff --git a/test/plausible/imported/csv_importer_test.exs b/test/plausible/imported/csv_importer_test.exs
index d508e3c40b..2c2f457347 100644
--- a/test/plausible/imported/csv_importer_test.exs
+++ b/test/plausible/imported/csv_importer_test.exs
@@ -363,8 +363,7 @@ defmodule Plausible.Imported.CSVImporterTest do
assert Plausible.Stats.Clickhouse.imported_pageview_count(site) == 99
end
- @tag skip: "To be re-enabled in the next PR"
- test "imports scroll_depth as null when the column does not exist in pages CSV",
+ test "imports scroll_depth as 0 when the column does not exist in pages CSV",
%{site: site, user: user} = ctx do
_ = ctx
@@ -424,9 +423,23 @@ defmodule Plausible.Imported.CSVImporterTest do
status: :completed
} = Repo.get_by!(SiteImport, site_id: site.id)
- # q = from(i in "imported_pages", where: i.site_id == ^site.id, select: i.scroll_depth)
+ q =
+ from(
+ i in "imported_pages",
+ where: i.site_id == ^site.id,
+ select: %{
+ total_scroll_depth: i.total_scroll_depth,
+ total_scroll_depth_visits: i.total_scroll_depth_visits
+ }
+ )
- # assert List.duplicate(nil, 16) == Plausible.IngestRepo.all(q)
+ assert List.duplicate(
+ %{
+ total_scroll_depth: 0,
+ total_scroll_depth_visits: 0
+ },
+ 16
+ ) == Plausible.IngestRepo.all(q)
end
test "accepts cells without quotes", %{site: site, user: user} = ctx do
@@ -1033,7 +1046,6 @@ defmodule Plausible.Imported.CSVImporterTest do
end
@tag :tmp_dir
- @tag skip: "To be re-enabled in the next PR"
test "scroll_depth", %{conn: conn, user: user, tmp_dir: tmp_dir} do
exported_site = new_site(owner: user)
imported_site = new_site(owner: user)
@@ -1109,24 +1121,39 @@ defmodule Plausible.Imported.CSVImporterTest do
select: %{
date: i.date,
page: i.page,
- scroll_depth: i.scroll_depth,
- pageleave_visitors: i.pageleave_visitors
+ total_scroll_depth: i.total_scroll_depth,
+ total_scroll_depth_visits: i.total_scroll_depth_visits
}
)
|> Plausible.IngestRepo.all()
- assert %{date: expected_start_date, page: "/", scroll_depth: 20, pageleave_visitors: 1} in imported_data
+ assert %{
+ date: expected_start_date,
+ page: "/",
+ total_scroll_depth: 20,
+ total_scroll_depth_visits: 1
+ } in imported_data
assert %{
date: expected_start_date,
page: "/another",
- scroll_depth: 50,
- pageleave_visitors: 2
+ total_scroll_depth: 50,
+ total_scroll_depth_visits: 2
} in imported_data
- assert %{date: expected_start_date, page: "/blog", scroll_depth: 180, pageleave_visitors: 3} in imported_data
+ assert %{
+ date: expected_start_date,
+ page: "/blog",
+ total_scroll_depth: 180,
+ total_scroll_depth_visits: 3
+ } in imported_data
- assert %{date: expected_end_date, page: "/blog", scroll_depth: nil, pageleave_visitors: 0} in imported_data
+ assert %{
+ date: expected_end_date,
+ page: "/blog",
+ total_scroll_depth: 0,
+ total_scroll_depth_visits: 0
+ } in imported_data
# assert via stats queries that scroll_depth from imported
# data matches the scroll_depth from native data
@@ -1154,7 +1181,6 @@ defmodule Plausible.Imported.CSVImporterTest do
end
@tag :tmp_dir
- @tag skip: "To be re-enabled in the next PR"
test "does not include scroll depth without existing engagement data", %{
user: user,
tmp_dir: tmp_dir
diff --git a/test/plausible/ingestion/event_test.exs b/test/plausible/ingestion/event_test.exs
index e19d1da117..2af62536da 100644
--- a/test/plausible/ingestion/event_test.exs
+++ b/test/plausible/ingestion/event_test.exs
@@ -320,11 +320,11 @@ defmodule Plausible.Ingestion.EventTest do
end
end
- test "drops pageleave event when no session found from cache" do
+ test "drops engagement event when no session found from cache" do
site = new_site()
payload = %{
- name: "pageleave",
+ name: "engagement",
url: "https://#{site.domain}/123",
d: "#{site.domain}"
}
@@ -333,7 +333,7 @@ defmodule Plausible.Ingestion.EventTest do
assert {:ok, request} = Request.build(conn)
assert {:ok, %{buffered: [], dropped: [dropped]}} = Event.build_and_buffer(request)
- assert dropped.drop_reason == :no_session_for_pageleave
+ assert dropped.drop_reason == :no_session_for_engagement
end
@tag :ee_only
diff --git a/test/plausible_web/controllers/api/external_controller_test.exs b/test/plausible_web/controllers/api/external_controller_test.exs
index 1efaa98240..28c8939dc8 100644
--- a/test/plausible_web/controllers/api/external_controller_test.exs
+++ b/test/plausible_web/controllers/api/external_controller_test.exs
@@ -649,7 +649,7 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
assert Map.get(event, :"meta.value") == ["true", "12"]
end
- test "records custom props for a pageleave event", %{conn: conn, site: site} do
+ test "records custom props for a engagement event", %{conn: conn, site: site} do
post(conn, "/api/event", %{
n: "pageview",
u: "https://ab.cd",
@@ -657,7 +657,7 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
})
post(conn, "/api/event", %{
- name: "pageleave",
+ name: "engagement",
url: "http://ab.cd/",
domain: site.domain,
props: %{
@@ -666,10 +666,10 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
}
})
- pageleave = get_events(site) |> Enum.find(&(&1.name == "pageleave"))
+ engagement = get_events(site) |> Enum.find(&(&1.name == "engagement"))
- assert Map.get(pageleave, :"meta.key") == ["bool_test", "number_test"]
- assert Map.get(pageleave, :"meta.value") == ["true", "12"]
+ assert Map.get(engagement, :"meta.key") == ["bool_test", "number_test"]
+ assert Map.get(engagement, :"meta.value") == ["true", "12"]
end
test "filters out bad props", %{conn: conn, site: site} do
@@ -1302,54 +1302,54 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do
test "ingests scroll_depth as 255 when sd not in params", %{conn: conn, site: site} do
post(conn, "/api/event", %{n: "pageview", u: "https://test.com", d: site.domain})
- post(conn, "/api/event", %{n: "pageleave", u: "https://test.com", d: site.domain})
+ post(conn, "/api/event", %{n: "engagement", u: "https://test.com", d: site.domain})
- pageleave = get_events(site) |> Enum.find(&(&1.name == "pageleave"))
+ engagement = get_events(site) |> Enum.find(&(&1.name == "engagement"))
- assert pageleave.scroll_depth == 255
+ assert engagement.scroll_depth == 255
end
- test "sd field is ignored if name is not pageleave", %{conn: conn, site: site} do
+ test "sd field is ignored if name is not engagement", %{conn: conn, site: site} do
post(conn, "/api/event", %{n: "pageview", u: "https://test.com", d: site.domain, sd: 10})
post(conn, "/api/event", %{n: "custom_e", u: "https://test.com", d: site.domain, sd: 10})
assert [%{scroll_depth: 0}, %{scroll_depth: 0}] = get_events(site)
end
- test "ingests valid scroll_depth for a pageleave", %{conn: conn, site: site} do
+ test "ingests valid scroll_depth for a engagement", %{conn: conn, site: site} do
post(conn, "/api/event", %{n: "pageview", u: "https://test.com", d: site.domain})
- post(conn, "/api/event", %{n: "pageleave", u: "https://test.com", d: site.domain, sd: 25})
+ post(conn, "/api/event", %{n: "engagement", u: "https://test.com", d: site.domain, sd: 25})
- pageleave = get_events(site) |> Enum.find(&(&1.name == "pageleave"))
+ engagement = get_events(site) |> Enum.find(&(&1.name == "engagement"))
- assert pageleave.scroll_depth == 25
+ assert engagement.scroll_depth == 25
end
test "ingests scroll_depth as 100 when sd > 100", %{conn: conn, site: site} do
post(conn, "/api/event", %{n: "pageview", u: "https://test.com", d: site.domain})
- post(conn, "/api/event", %{n: "pageleave", u: "https://test.com", d: site.domain, sd: 101})
+ post(conn, "/api/event", %{n: "engagement", u: "https://test.com", d: site.domain, sd: 101})
- pageleave = get_events(site) |> Enum.find(&(&1.name == "pageleave"))
+ engagement = get_events(site) |> Enum.find(&(&1.name == "engagement"))
- assert pageleave.scroll_depth == 100
+ assert engagement.scroll_depth == 100
end
test "ingests scroll_depth as 255 when sd is a string", %{conn: conn, site: site} do
post(conn, "/api/event", %{n: "pageview", u: "https://test.com", d: site.domain})
- post(conn, "/api/event", %{n: "pageleave", u: "https://test.com", d: site.domain, sd: "1"})
+ post(conn, "/api/event", %{n: "engagement", u: "https://test.com", d: site.domain, sd: "1"})
- pageleave = get_events(site) |> Enum.find(&(&1.name == "pageleave"))
+ engagement = get_events(site) |> Enum.find(&(&1.name == "engagement"))
- assert pageleave.scroll_depth == 255
+ assert engagement.scroll_depth == 255
end
test "ingests scroll_depth as 255 when sd is a negative integer", %{conn: conn, site: site} do
post(conn, "/api/event", %{n: "pageview", u: "https://test.com", d: site.domain})
- post(conn, "/api/event", %{n: "pageleave", u: "https://test.com", d: site.domain, sd: -1})
+ post(conn, "/api/event", %{n: "engagement", u: "https://test.com", d: site.domain, sd: -1})
- pageleave = get_events(site) |> Enum.find(&(&1.name == "pageleave"))
+ engagement = get_events(site) |> Enum.find(&(&1.name == "engagement"))
- assert pageleave.scroll_depth == 255
+ assert engagement.scroll_depth == 255
end
test "ingests valid scroll_depth for a engagement event", %{conn: conn, site: site} do
diff --git a/test/plausible_web/controllers/api/external_stats_controller/query_imported_test.exs b/test/plausible_web/controllers/api/external_stats_controller/query_imported_test.exs
index 9557b9f35e..c3c92b30ae 100644
--- a/test/plausible_web/controllers/api/external_stats_controller/query_imported_test.exs
+++ b/test/plausible_web/controllers/api/external_stats_controller/query_imported_test.exs
@@ -1524,7 +1524,6 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryImportedTest do
refute meta["metric_warnings"]["scroll_depth"]
end
- @tag skip: "To be re-enabled in the next PR"
test "does not return warning when imported scroll depth exists", %{
conn: conn,
site: site
@@ -1548,8 +1547,8 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryImportedTest do
build(:imported_pages,
page: "/",
date: ~D[2021-02-01],
- scroll_depth: 80,
- pageleave_visitors: 1
+ total_scroll_depth: 80,
+ total_scroll_depth_visits: 1
),
build(:imported_pages, page: "/", date: ~D[2021-02-28])
])
diff --git a/test/plausible_web/controllers/api/external_stats_controller/query_test.exs b/test/plausible_web/controllers/api/external_stats_controller/query_test.exs
index c1882d6c37..a5acd797b7 100644
--- a/test/plausible_web/controllers/api/external_stats_controller/query_test.exs
+++ b/test/plausible_web/controllers/api/external_stats_controller/query_test.exs
@@ -3892,6 +3892,35 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest do
]
end
+ test "scroll_depth metric is session-based", %{conn: conn, site: site} do
+ t0 = ~N[2020-01-01 00:00:00]
+ t1 = NaiveDateTime.add(t0, 10, :minute)
+ t2 = NaiveDateTime.add(t0, 3, :hour)
+
+ populate_stats(site, [
+ build(:pageview, user_id: 12, pathname: "/blog", timestamp: t0),
+ build(:engagement, user_id: 12, pathname: "/blog", timestamp: t0, scroll_depth: 20),
+ build(:pageview, user_id: 12, pathname: "/blog", timestamp: t1),
+ build(:engagement, user_id: 12, pathname: "/blog", timestamp: t1, scroll_depth: 40),
+ # Different session
+ build(:pageview, user_id: 12, pathname: "/blog", timestamp: t2),
+ build(:engagement, user_id: 12, pathname: "/blog", timestamp: t2, scroll_depth: 80)
+ ])
+
+ conn =
+ post(conn, "/api/v2/query-internal-test", %{
+ "site_id" => site.domain,
+ "metrics" => ["scroll_depth"],
+ "date_range" => "all",
+ "dimensions" => ["event:page"]
+ })
+
+ assert json_response(conn, 200)["results"] == [
+ # avg(max(20, 40), 80) = 60
+ %{"dimensions" => ["/blog"], "metrics" => [60]}
+ ]
+ end
+
test "can sort by scroll_depth in event:page breakdown", %{conn: conn, site: site} do
t0 = ~N[2020-01-01 00:00:00]
[t1, t2, t3] = for i <- 1..3, do: NaiveDateTime.add(t0, i, :minute)
diff --git a/test/plausible_web/controllers/api/stats_controller/main_graph_test.exs b/test/plausible_web/controllers/api/stats_controller/main_graph_test.exs
index fbdfaccdd5..3b36f463a6 100644
--- a/test/plausible_web/controllers/api/stats_controller/main_graph_test.exs
+++ b/test/plausible_web/controllers/api/stats_controller/main_graph_test.exs
@@ -634,7 +634,6 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do
assert plot == [40, 20, 0, 0, 0, 0, 0]
end
- @tag skip: "To be re-enabled in the next PR"
test "returns scroll depth per day with imported data", %{conn: conn, site: site} do
site_import = insert(:site_import, site: site)
@@ -653,18 +652,18 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do
date: ~D[2020-01-02],
page: "/",
visitors: 1,
- scroll_depth: 40,
- pageleave_visitors: 1
+ total_scroll_depth: 40,
+ total_scroll_depth_visits: 1
),
# 2020-01-03 - only imported data
build(:imported_pages,
date: ~D[2020-01-03],
page: "/",
visitors: 1,
- scroll_depth: 90,
- pageleave_visitors: 1
+ total_scroll_depth: 90,
+ total_scroll_depth_visits: 1
),
- build(:imported_pages, date: ~D[2020-01-03], page: "/", visitors: 100, scroll_depth: nil)
+ build(:imported_pages, date: ~D[2020-01-03], page: "/", visitors: 100)
])
filters = Jason.encode!([[:is, "event:page", ["/"]]])
diff --git a/test/plausible_web/controllers/api/stats_controller/pages_test.exs b/test/plausible_web/controllers/api/stats_controller/pages_test.exs
index d4d363b0d3..673b091655 100644
--- a/test/plausible_web/controllers/api/stats_controller/pages_test.exs
+++ b/test/plausible_web/controllers/api/stats_controller/pages_test.exs
@@ -676,7 +676,6 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
}
end
- @tag skip: "To be re-enabled in the next PR"
test "calculates scroll_depth from native and imported data combined", %{
conn: conn,
site: site
@@ -695,8 +694,8 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
pageviews: 3,
time_on_page: 90,
page: "/blog",
- scroll_depth: 120,
- pageleave_visitors: 3
+ total_scroll_depth: 120,
+ total_scroll_depth_visits: 3
)
])
@@ -718,7 +717,6 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
]
end
- @tag skip: "To be re-enabled in the next PR"
test "handles missing scroll_depth data from native and imported sources", %{
conn: conn,
site: site
@@ -752,8 +750,8 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
pageviews: 4,
time_on_page: 180,
page: "/native-and-imported",
- scroll_depth: 120,
- pageleave_visitors: 3
+ total_scroll_depth: 120,
+ total_scroll_depth_visits: 3
),
build(:imported_pages,
date: ~D[2020-01-01],
@@ -761,8 +759,8 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
pageviews: 30,
time_on_page: 300,
page: "/imported-only",
- scroll_depth: 100,
- pageleave_visitors: 10
+ total_scroll_depth: 100,
+ total_scroll_depth_visits: 10
)
])
@@ -800,7 +798,6 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
]
end
- @tag skip: "To be re-enabled in the next PR"
test "can query scroll depth only from imported data, ignoring rows where scroll depth doesn't exist",
%{
conn: conn,
@@ -812,15 +809,15 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
visitors: 10,
pageviews: 10,
page: "/blog",
- scroll_depth: 100,
- pageleave_visitors: 10
+ total_scroll_depth: 100,
+ total_scroll_depth_visits: 10
),
build(:imported_pages,
date: ~D[2020-01-01],
visitors: 100,
pageviews: 150,
page: "/blog",
- scroll_depth: nil
+ total_scroll_depth: 0
)
])
@@ -1140,7 +1137,6 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
]
end
- @tag skip: "To be re-enabled in the next PR"
test "returns scroll depth warning code", %{conn: conn, site: site} do
Plausible.Sites.set_scroll_depth_visible_at(site)
@@ -1153,7 +1149,6 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
"no_imported_scroll_depth"
end
- @tag skip: "To be re-enabled in the next PR"
test "returns imported pages with a pageview goal filter", %{conn: conn, site: site} do
insert(:goal, site: site, page_path: "/blog**")
diff --git a/test/plausible_web/controllers/api/stats_controller/top_stats_test.exs b/test/plausible_web/controllers/api/stats_controller/top_stats_test.exs
index d0fcfd9134..7075108e2c 100644
--- a/test/plausible_web/controllers/api/stats_controller/top_stats_test.exs
+++ b/test/plausible_web/controllers/api/stats_controller/top_stats_test.exs
@@ -1018,7 +1018,6 @@ defmodule PlausibleWeb.Api.StatsController.TopStatsTest do
]
end
- @tag skip: "To be re-enabled in the next PR"
test "returns scroll_depth with a page filter with imported data", %{conn: conn, site: site} do
site_import =
insert(:site_import, site: site, start_date: ~D[2021-01-01], has_scroll_depth: true)
@@ -1034,10 +1033,10 @@ defmodule PlausibleWeb.Api.StatsController.TopStatsTest do
page: "/",
date: ~D[2021-01-01],
visitors: 8,
- scroll_depth: 410,
- pageleave_visitors: 8
+ total_scroll_depth: 410,
+ total_scroll_depth_visits: 8
),
- build(:imported_pages, page: "/", date: ~D[2021-01-02], visitors: 100, scroll_depth: nil)
+ build(:imported_pages, page: "/", date: ~D[2021-01-02], visitors: 100)
])
filters = Jason.encode!([[:is, "event:page", ["/"]]])
diff --git a/test/plausible_web/controllers/stats_controller_test.exs b/test/plausible_web/controllers/stats_controller_test.exs
index 96be532488..f9b30d67f7 100644
--- a/test/plausible_web/controllers/stats_controller_test.exs
+++ b/test/plausible_web/controllers/stats_controller_test.exs
@@ -748,7 +748,7 @@ defmodule PlausibleWeb.StatsControllerTest do
["2020-01-02", "0", "0", "0", "0.0", "0.0", "", ""],
["2020-01-03", "0", "0", "0", "0.0", "0.0", "", ""],
["2020-01-04", "0", "0", "0", "0.0", "0.0", "", ""],
- ["2020-01-05", "1", "2", "2", "1.0", "100", "0", "40"],
+ ["2020-01-05", "1", "2", "2", "1.0", "100", "0", "28"],
["2020-01-06", "0", "0", "0", "0.0", "0.0", "", ""],
["2020-01-07", "1", "1", "1", "1.0", "100", "0", "90"],
[""]