Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.
TypeScript analyticsanalytics-dashboardclickhouseelixirgoogle-analyticsmarketingopen-source-analyticsphoenixplausible-analyticspostgresqlprivacyprivacy-friendlyself-hostedsimple-analyticsstatisticstailwindcssweb-analyticswebsite-analyticswebsite-statswebsite-tracking
Scroll depth: ingestion, query and import/export changes (#5072)
Karl-Aksel Puulmann53581973d05e
diff --git a/lib/plausible/exports.ex b/lib/plausible/exports.exindex 6cb93f3739..f9355a1574 100644--- a/lib/plausible/exports.ex+++ b/lib/plausible/exports.ex@@ -284,11 +284,7 @@ defmodule Plausible.Exports dodefmacrop visit_duration(t) doquote doselected_as(- fragment(- "toUInt64(round(greatest(sum(?*?),0)*any(_sample_factor)))",- unquote(t).sign,- unquote(t).duration- ),+ scale_sample(fragment("greatest(sum(?*?),0)", unquote(t).sign, unquote(t).duration)),:visit_duration)end@@ -297,7 +293,7 @@ defmodule Plausible.Exports dodefmacrop visitors(t) doquote doselected_as(- fragment("toUInt64(round(uniq(?)*any(_sample_factor)))", unquote(t).user_id),+ scale_sample(fragment("uniq(?)", unquote(t).user_id)),:visitors)end@@ -306,7 +302,7 @@ defmodule Plausible.Exports dodefmacrop visits(t) doquote doselected_as(- fragment("toUInt64(round(greatest(sum(?),0)*any(_sample_factor)))", unquote(t).sign),+ scale_sample(fragment("greatest(sum(?),0)", unquote(t).sign)),:visits)end@@ -315,10 +311,12 @@ defmodule Plausible.Exports dodefmacrop bounces(t) doquote doselected_as(- fragment(- "toUInt32(round(greatest(sum(?*?),0)*any(_sample_factor)))",- unquote(t).sign,- unquote(t).is_bounce+ scale_sample(+ fragment(+ "greatest(sum(?*?),0)",+ unquote(t).sign,+ unquote(t).is_bounce+ )),:bounces)@@ -328,10 +326,12 @@ defmodule Plausible.Exports dodefmacrop pageviews(t) doquote doselected_as(- fragment(- "toUInt64(round(greatest(sum(?*?),0)*any(_sample_factor)))",- unquote(t).sign,- unquote(t).pageviews+ scale_sample(+ fragment(+ "greatest(sum(?*?),0)",+ unquote(t).sign,+ unquote(t).pageviews+ )),:pageviews)@@ -359,7 +359,7 @@ defmodule Plausible.Exports dodate: date(e.timestamp, ^timezone),pageviews:selected_as(- fragment("toUInt64(round(countIf(?='pageview')*any(_sample_factor)))", e.name),+ scale_sample(fragment("countIf(?='pageview')", e.name)),:pageviews)}@@ -428,33 +428,27 @@ defmodule Plausible.Exports doorder_by: selected_as(:date))- # :TODO: To be removed in the next PR- if include_scroll_depth? and false do- max_scroll_depth_per_visitor_q =+ if include_scroll_depth? do+ max_scroll_depth_per_session_q =from(e in "events_v2",where: ^export_filter(site_id, date_range),where: e.name == "engagement" and e.scroll_depth <= 100,select: %{date: date(e.timestamp, ^timezone),page: selected_as(e.pathname, :page),- user_id: e.user_id,+ session_id: e.session_id,max_scroll_depth: max(e.scroll_depth)},- group_by: [e.user_id, selected_as(:date), selected_as(:page)]+ group_by: [e.session_id, selected_as(:date), selected_as(:page)])scroll_depth_q =- from(p in subquery(max_scroll_depth_per_visitor_q),+ from(p in subquery(max_scroll_depth_per_session_q),select: %{date: p.date,page: p.page,- scroll_depth:- fragment(- "if(isNull(sum(?)), NULL, toUInt64(sum(?)))",- p.max_scroll_depth,- p.max_scroll_depth- ),- pageleave_visitors: fragment("uniq(?)", p.user_id)+ total_scroll_depth: fragment("sum(?)", p.max_scroll_depth),+ total_scroll_depth_visits: fragment("uniq(?)", p.session_id)},group_by: [:date, :page])@@ -467,13 +461,13 @@ defmodule Plausible.Exports doselected_as(fragment("any(?)", e.hostname), :hostname),selected_as(e.pathname, :page),selected_as(- fragment("toUInt64(round(uniq(?)*any(_sample_factor)))", e.session_id),+ scale_sample(fragment("uniq(?)", e.session_id)),:visits),visitors(e),- selected_as(fragment("toUInt64(round(count()*any(_sample_factor)))"), :pageviews),- selected_as(fragment("any(?)", s.scroll_depth), :scroll_depth),- selected_as(fragment("any(?)", s.pageleave_visitors), :pageleave_visitors)+ selected_as(scale_sample(fragment("count()")), :pageviews),+ selected_as(fragment("any(?)", s.total_scroll_depth), :total_scroll_depth),+ selected_as(fragment("any(?)", s.total_scroll_depth_visits), :total_scroll_depth_visits)])else@@ -483,11 +477,11 @@ defmodule Plausible.Exports doselected_as(fragment("any(?)", e.hostname), :hostname),selected_as(e.pathname, :page),selected_as(- fragment("toUInt64(round(uniq(?)*any(_sample_factor)))", e.session_id),+ scale_sample(fragment("uniq(?)", e.session_id)),:visits),visitors(e),- selected_as(fragment("toUInt64(round(count()*any(_sample_factor)))"), :pageviews)+ selected_as(scale_sample(fragment("count()")), :pageviews)])endend@@ -502,7 +496,7 @@ defmodule Plausible.Exports dos.entry_page,visitors(s),selected_as(- fragment("toUInt64(round(greatest(sum(?),0)*any(_sample_factor)))", s.sign),+ scale_sample(fragment("greatest(sum(?),0)", s.sign)),:entrances),visit_duration(s),@@ -522,7 +516,7 @@ defmodule Plausible.Exports dovisitors(s),visit_duration(s),selected_as(- fragment("toUInt64(round(greatest(sum(?),0)*any(_sample_factor)))", s.sign),+ scale_sample(fragment("greatest(sum(?),0)", s.sign)),:exits),bounces(s),@@ -563,7 +557,7 @@ defmodule Plausible.Exports do:path),visitors(e),- selected_as(fragment("toUInt64(round(count()*any(_sample_factor)))"), :events)+ selected_as(scale_sample(fragment("count()")), :events)]enddiff --git a/lib/plausible/imported/csv_importer.ex b/lib/plausible/imported/csv_importer.exindex ff763e304b..d3ad7bffa0 100644--- a/lib/plausible/imported/csv_importer.ex+++ b/lib/plausible/imported/csv_importer.ex@@ -5,6 +5,7 @@ defmodule Plausible.Imported.CSVImporter do"""use Plausible.Imported.Importer+ import Ecto.Query, only: [from: 2]@impl truedef name(), do: :csv@@ -54,6 +55,26 @@ defmodule Plausible.Imported.CSVImporter do{:error, Exception.message(e)}end+ def on_success(site_import, _extra_data) do+ has_scroll_depth? =+ Plausible.ClickhouseRepo.exists?(+ from(i in "imported_pages",+ where: i.site_id == ^site_import.site_id,+ where: i.import_id == ^site_import.id,+ where: i.total_scroll_depth_visits > 0,+ select: 1+ )+ )++ if has_scroll_depth? do+ site_import+ |> Ecto.Changeset.change(%{has_scroll_depth: true})+ |> Plausible.Repo.update!()+ end++ :ok+ end+defp import_s3(ch, site_import, uploads) do%{id: import_id,@@ -75,7 +96,7 @@ defmodule Plausible.Imported.CSVImporter dostatement ="""INSERT INTO {table:Identifier}(site_id,import_id,#{s3_columns}) \- SELECT {site_id:UInt64}, {import_id:UInt64}, * \+ SELECT {site_id:UInt64}, {import_id:UInt64}, #{s3_columns} \FROM s3({s3_url:String},{s3_access_key_id:String},{s3_secret_access_key:String},{s3_format:String},{s3_structure:String}) \WHERE date >= {start_date:Date} AND date <= {end_date:Date}\"""@@ -119,7 +140,7 @@ defmodule Plausible.Imported.CSVImporter dostatement ="""INSERT INTO {table:Identifier}(site_id,import_id,#{input_columns}) \- SELECT {site_id:UInt64}, {import_id:UInt64}, * \+ SELECT {site_id:UInt64}, {import_id:UInt64}, #{input_columns} \FROM input({input_structure:String}) \WHERE date >= {start_date:Date} AND date <= {end_date:Date} \FORMAT CSVWithNames@@ -161,7 +182,7 @@ defmodule Plausible.Imported.CSVImporter do"imported_operating_systems" =>"date Date, operating_system String, operating_system_version String, visitors UInt64, visits UInt64, visit_duration UInt64, bounces UInt32, pageviews UInt64","imported_pages" =>- "date Date, hostname String, page String, visits UInt64, visitors UInt64, pageviews UInt64",+ "date Date, hostname String, page String, visits UInt64, visitors UInt64, pageviews UInt64, total_scroll_depth UInt64, total_scroll_depth_visits UInt64","imported_sources" =>"date Date, source String, referrer String, utm_source String, utm_medium String, utm_campaign String, utm_content String, utm_term String, pageviews UInt64, visitors UInt64, visits UInt64, visit_duration UInt64, bounces UInt32","imported_visitors" =>diff --git a/lib/plausible/imported/page.ex b/lib/plausible/imported/page.exindex 747c5e0306..6e27c1bd89 100644--- a/lib/plausible/imported/page.ex+++ b/lib/plausible/imported/page.ex@@ -15,7 +15,7 @@ defmodule Plausible.Imported.Page dofield :pageviews, Ch, type: "UInt64"field :exits, Ch, type: "UInt64"field :time_on_page, Ch, type: "UInt64"- # field :scroll_depth, Ch, type: "Nullable(UInt64)"- # field :pageleave_visitors, Ch, type: "UInt64"+ field :total_scroll_depth, Ch, type: "UInt64"+ field :total_scroll_depth_visits, Ch, type: "UInt64"endenddiff --git a/lib/plausible/ingestion/event.ex b/lib/plausible/ingestion/event.exindex c9a52edd1f..adbd488f82 100644--- a/lib/plausible/ingestion/event.ex+++ b/lib/plausible/ingestion/event.ex@@ -35,7 +35,7 @@ defmodule Plausible.Ingestion.Event do| :site_hostname_allowlist| :verification_agent| :lock_timeout- | :no_session_for_pageleave+ | :no_session_for_engagement@type t() :: %__MODULE__{domain: String.t() | nil,@@ -385,8 +385,8 @@ defmodule Plausible.Ingestion.Event do)case session_result do- {:ok, :no_session_for_pageleave} ->- drop(event, :no_session_for_pageleave)+ {:ok, :no_session_for_engagement} ->+ drop(event, :no_session_for_engagement){:error, :timeout} ->drop(event, :lock_timeout)diff --git a/lib/plausible/ingestion/request.ex b/lib/plausible/ingestion/request.exindex b0ec08147b..7ae04ac0a4 100644--- a/lib/plausible/ingestion/request.ex+++ b/lib/plausible/ingestion/request.ex@@ -250,7 +250,7 @@ defmodule Plausible.Ingestion.Request doenddefp put_scroll_depth(changeset, %{} = request_body) do- if Changeset.get_field(changeset, :event_name) in ["pageleave", "engagement"] do+ if Changeset.get_field(changeset, :event_name) == "engagement" doscroll_depth =case request_body["sd"] dosd when is_integer(sd) and sd >= 0 and sd <= 100 -> sddiff --git a/lib/plausible/session/cache_store.ex b/lib/plausible/session/cache_store.exindex 5e0a1eaf6f..54a5bb488f 100644--- a/lib/plausible/session/cache_store.ex+++ b/lib/plausible/session/cache_store.ex@@ -25,15 +25,14 @@ defmodule Plausible.Session.CacheStore do)end- defp handle_event(%{name: name} = event, found_session, _, _)- when name in ["pageleave", "engagement"] do+ defp handle_event(%{name: "engagement"} = event, found_session, _, _) doif found_session do# Make sure the session is kept active in the in-memory session cacherefresh_session_cache(found_session, event.timestamp)found_sessionelse- :no_session_for_pageleave+ :no_session_for_engagementendenddiff --git a/lib/plausible/stats/imported/sql/expression.ex b/lib/plausible/stats/imported/sql/expression.exindex 69b8dbf34b..95d005e676 100644--- a/lib/plausible/stats/imported/sql/expression.ex+++ b/lib/plausible/stats/imported/sql/expression.ex@@ -115,12 +115,12 @@ defmodule Plausible.Stats.Imported.SQL.Expression dowrap_alias([i], %{pageviews: sum(i.pageviews), __internal_visits: sum(i.visits)})end- # defp select_metric(:scroll_depth, "imported_pages") do- # wrap_alias([i], %{- # scroll_depth_sum: sum(i.scroll_depth),- # pageleave_visitors: sum(i.pageleave_visitors)- # })- # end+ defp select_metric(:scroll_depth, "imported_pages") do+ wrap_alias([i], %{+ total_scroll_depth: sum(i.total_scroll_depth),+ total_scroll_depth_visits: sum(i.total_scroll_depth_visits)+ })+ enddefp select_metric(_metric, _table), do: %{}@@ -364,6 +364,10 @@ defmodule Plausible.Stats.Imported.SQL.Expression do# information from imported data here already.def select_joined_metrics(q, [:scroll_depth | rest]) doq+ |> select_merge_as([s, i], %{+ __imported_total_scroll_depth: i.total_scroll_depth,+ __imported_total_scroll_depth_visits: i.total_scroll_depth_visits+ })|> select_joined_metrics(rest)enddiff --git a/lib/plausible/stats/sql/expression.ex b/lib/plausible/stats/sql/expression.exindex b69b2454e0..4708d8027f 100644--- a/lib/plausible/stats/sql/expression.ex+++ b/lib/plausible/stats/sql/expression.ex@@ -191,27 +191,25 @@ defmodule Plausible.Stats.SQL.Expression dodef event_metric(:pageviews) dowrap_alias([e], %{- pageviews:- fragment("toUInt64(round(countIf(? = 'pageview') * any(_sample_factor)))", e.name)+ pageviews: scale_sample(fragment("countIf(? = 'pageview')", e.name))})enddef event_metric(:events) dowrap_alias([e], %{- events:- fragment("toUInt64(round(countIf(? != 'engagement') * any(_sample_factor)))", e.name)+ events: scale_sample(fragment("countIf(? != 'engagement')", e.name))})enddef event_metric(:visitors) dowrap_alias([e], %{- visitors: fragment("toUInt64(round(uniq(?) * any(_sample_factor)))", e.user_id)+ visitors: scale_sample(fragment("uniq(?)", e.user_id))})enddef event_metric(:visits) dowrap_alias([e], %{- visits: fragment("toUInt64(round(uniq(?) * any(_sample_factor)))", e.session_id)+ visits: scale_sample(fragment("uniq(?)", e.session_id))})end@@ -273,26 +271,25 @@ defmodule Plausible.Stats.SQL.Expression dodef session_metric(:visits, _query) dowrap_alias([s], %{- visits: fragment("toUInt64(round(sum(?) * any(_sample_factor)))", s.sign)+ visits: scale_sample(fragment("sum(?)", s.sign))})enddef session_metric(:pageviews, _query) dowrap_alias([s], %{- pageviews:- fragment("toUInt64(round(sum(? * ?) * any(_sample_factor)))", s.sign, s.pageviews)+ pageviews: scale_sample(fragment("sum(? * ?)", s.sign, s.pageviews))})enddef session_metric(:events, _query) dowrap_alias([s], %{- events: fragment("toUInt64(round(sum(? * ?) * any(_sample_factor)))", s.sign, s.events)+ events: scale_sample(fragment("sum(? * ?)", s.sign, s.events))})enddef session_metric(:visitors, _query) dowrap_alias([s], %{- visitors: fragment("toUInt64(round(uniq(?) * any(_sample_factor)))", s.user_id)+ visitors: scale_sample(fragment("uniq(?)", s.user_id))})enddiff --git a/lib/plausible/stats/sql/fragments.ex b/lib/plausible/stats/sql/fragments.exindex 7092db19b1..8e39fb823f 100644--- a/lib/plausible/stats/sql/fragments.ex+++ b/lib/plausible/stats/sql/fragments.ex@@ -10,15 +10,21 @@ defmodule Plausible.Stats.SQL.Fragments doendend+ defmacro scale_sample(sampled_fragment) do+ quote do+ fragment("toUInt64(round(? * any(_sample_factor)))", unquote(sampled_fragment))+ end+ end+defmacro uniq(user_id) doquote do- fragment("toUInt64(round(uniq(?) * any(_sample_factor)))", unquote(user_id))+ scale_sample(fragment("uniq(?)", unquote(user_id)))endenddefmacro total() doquote do- fragment("toUInt64(round(count(*) * any(_sample_factor)))")+ scale_sample(fragment("count()"))endenddiff --git a/lib/plausible/stats/sql/special_metrics.ex b/lib/plausible/stats/sql/special_metrics.exindex 8fd32065f2..8ade1c18a5 100644--- a/lib/plausible/stats/sql/special_metrics.ex+++ b/lib/plausible/stats/sql/special_metrics.ex@@ -129,15 +129,15 @@ defmodule Plausible.Stats.SQL.SpecialMetrics dodef maybe_add_scroll_depth(q, site, query) doif :scroll_depth in query.metrics do- max_per_visitor_q =+ max_per_session_q =Base.base_event_query(site, query)|> where([e], e.name == "engagement" and e.scroll_depth <= 100)|> select([e], %{- user_id: e.user_id,+ session_id: e.session_id,max_scroll_depth: max(e.scroll_depth)})|> SQL.QueryBuilder.build_group_by(:events, query)- |> group_by([e], e.user_id)+ |> group_by([e], e.session_id)dim_shortnames = Enum.map(query.dimensions, fn dim -> shortname(query, dim) end)@@ -150,12 +150,13 @@ defmodule Plausible.Stats.SQL.SpecialMetrics dodim_shortnames|> Enum.map(fn dim -> dynamic([p], field(p, ^dim)) end)- scroll_depth_sum_q =- subquery(max_per_visitor_q)- |> select([p], %{- scroll_depth_sum:- fragment("if(count(?) = 0, NULL, sum(?))", p.user_id, p.max_scroll_depth),- pageleave_visitors: fragment("count(?)", p.user_id)+ total_scroll_depth_q =+ subquery(max_per_session_q)+ |> select([], %{})+ |> select_merge_as([p], %{+ # Note: No need to upscale sample size here since it would end up cancelling out due to the result being an average+ total_scroll_depth: fragment("sum(?)", p.max_scroll_depth),+ total_scroll_depth_visits: fragment("uniq(?)", p.session_id)})|> select_merge(^dim_select)|> group_by(^dim_group_by)@@ -171,7 +172,7 @@ defmodule Plausible.Stats.SQL.SpecialMetrics doendjoined_q =- join(q, :left, [e], s in subquery(scroll_depth_sum_q), on: ^join_on_dim_condition)+ join(q, :left, [e], s in subquery(total_scroll_depth_q), on: ^join_on_dim_condition)if query.include_imported dojoined_q@@ -179,17 +180,14 @@ defmodule Plausible.Stats.SQL.SpecialMetrics doscroll_depth:fragment("""- case- when isNotNull(?) then- toUInt8(round(? / ?))- else- NULL- end+ if(? + ? > 0, toInt8(round((? + ?) / (? + ?))), NULL)""",- # Case 3: Only native scroll depth sum is present- s.scroll_depth_sum,- s.scroll_depth_sum,- s.pageleave_visitors+ s.total_scroll_depth_visits,+ selected_as(:__imported_total_scroll_depth_visits),+ s.total_scroll_depth,+ selected_as(:__imported_total_scroll_depth),+ s.total_scroll_depth_visits,+ selected_as(:__imported_total_scroll_depth_visits))})else@@ -198,9 +196,9 @@ defmodule Plausible.Stats.SQL.SpecialMetrics doscroll_depth:fragment("if(any(?) > 0, toUInt8(round(any(?) / any(?))), NULL)",- s.pageleave_visitors,- s.scroll_depth_sum,- s.pageleave_visitors+ s.total_scroll_depth_visits,+ s.total_scroll_depth,+ s.total_scroll_depth_visits)})end@@ -248,7 +246,7 @@ defmodule Plausible.Stats.SQL.SpecialMetrics dodefp total_visitors(site, query) doBase.base_event_query(site, query)|> select([e],- total_visitors: fragment("toUInt64(round(uniq(?) * any(_sample_factor)))", e.user_id)+ total_visitors: scale_sample(fragment("uniq(?)", e.user_id)))endenddiff --git a/test/plausible/imported/csv_importer_test.exs b/test/plausible/imported/csv_importer_test.exsindex d508e3c40b..2c2f457347 100644--- a/test/plausible/imported/csv_importer_test.exs+++ b/test/plausible/imported/csv_importer_test.exs@@ -363,8 +363,7 @@ defmodule Plausible.Imported.CSVImporterTest doassert Plausible.Stats.Clickhouse.imported_pageview_count(site) == 99end- @tag skip: "To be re-enabled in the next PR"- test "imports scroll_depth as null when the column does not exist in pages CSV",+ test "imports scroll_depth as 0 when the column does not exist in pages CSV",%{site: site, user: user} = ctx do_ = ctx@@ -424,9 +423,23 @@ defmodule Plausible.Imported.CSVImporterTest dostatus: :completed} = Repo.get_by!(SiteImport, site_id: site.id)- # q = from(i in "imported_pages", where: i.site_id == ^site.id, select: i.scroll_depth)+ q =+ from(+ i in "imported_pages",+ where: i.site_id == ^site.id,+ select: %{+ total_scroll_depth: i.total_scroll_depth,+ total_scroll_depth_visits: i.total_scroll_depth_visits+ }+ )- # assert List.duplicate(nil, 16) == Plausible.IngestRepo.all(q)+ assert List.duplicate(+ %{+ total_scroll_depth: 0,+ total_scroll_depth_visits: 0+ },+ 16+ ) == Plausible.IngestRepo.all(q)endtest "accepts cells without quotes", %{site: site, user: user} = ctx do@@ -1033,7 +1046,6 @@ defmodule Plausible.Imported.CSVImporterTest doend@tag :tmp_dir- @tag skip: "To be re-enabled in the next PR"test "scroll_depth", %{conn: conn, user: user, tmp_dir: tmp_dir} doexported_site = new_site(owner: user)imported_site = new_site(owner: user)@@ -1109,24 +1121,39 @@ defmodule Plausible.Imported.CSVImporterTest doselect: %{date: i.date,page: i.page,- scroll_depth: i.scroll_depth,- pageleave_visitors: i.pageleave_visitors+ total_scroll_depth: i.total_scroll_depth,+ total_scroll_depth_visits: i.total_scroll_depth_visits})|> Plausible.IngestRepo.all()- assert %{date: expected_start_date, page: "/", scroll_depth: 20, pageleave_visitors: 1} in imported_data+ assert %{+ date: expected_start_date,+ page: "/",+ total_scroll_depth: 20,+ total_scroll_depth_visits: 1+ } in imported_dataassert %{date: expected_start_date,page: "/another",- scroll_depth: 50,- pageleave_visitors: 2+ total_scroll_depth: 50,+ total_scroll_depth_visits: 2} in imported_data- assert %{date: expected_start_date, page: "/blog", scroll_depth: 180, pageleave_visitors: 3} in imported_data+ assert %{+ date: expected_start_date,+ page: "/blog",+ total_scroll_depth: 180,+ total_scroll_depth_visits: 3+ } in imported_data- assert %{date: expected_end_date, page: "/blog", scroll_depth: nil, pageleave_visitors: 0} in imported_data+ assert %{+ date: expected_end_date,+ page: "/blog",+ total_scroll_depth: 0,+ total_scroll_depth_visits: 0+ } in imported_data# assert via stats queries that scroll_depth from imported# data matches the scroll_depth from native data@@ -1154,7 +1181,6 @@ defmodule Plausible.Imported.CSVImporterTest doend@tag :tmp_dir- @tag skip: "To be re-enabled in the next PR"test "does not include scroll depth without existing engagement data", %{user: user,tmp_dir: tmp_dirdiff --git a/test/plausible/ingestion/event_test.exs b/test/plausible/ingestion/event_test.exsindex e19d1da117..2af62536da 100644--- a/test/plausible/ingestion/event_test.exs+++ b/test/plausible/ingestion/event_test.exs@@ -320,11 +320,11 @@ defmodule Plausible.Ingestion.EventTest doendend- test "drops pageleave event when no session found from cache" do+ test "drops engagement event when no session found from cache" dosite = new_site()payload = %{- name: "pageleave",+ name: "engagement",url: "https://#{site.domain}/123",d: "#{site.domain}"}@@ -333,7 +333,7 @@ defmodule Plausible.Ingestion.EventTest doassert {:ok, request} = Request.build(conn)assert {:ok, %{buffered: [], dropped: [dropped]}} = Event.build_and_buffer(request)- assert dropped.drop_reason == :no_session_for_pageleave+ assert dropped.drop_reason == :no_session_for_engagementend@tag :ee_onlydiff --git a/test/plausible_web/controllers/api/external_controller_test.exs b/test/plausible_web/controllers/api/external_controller_test.exsindex 1efaa98240..28c8939dc8 100644--- a/test/plausible_web/controllers/api/external_controller_test.exs+++ b/test/plausible_web/controllers/api/external_controller_test.exs@@ -649,7 +649,7 @@ defmodule PlausibleWeb.Api.ExternalControllerTest doassert Map.get(event, :"meta.value") == ["true", "12"]end- test "records custom props for a pageleave event", %{conn: conn, site: site} do+ test "records custom props for a engagement event", %{conn: conn, site: site} dopost(conn, "/api/event", %{n: "pageview",u: "https://ab.cd",@@ -657,7 +657,7 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do})post(conn, "/api/event", %{- name: "pageleave",+ name: "engagement",url: "http://ab.cd/",domain: site.domain,props: %{@@ -666,10 +666,10 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do}})- pageleave = get_events(site) |> Enum.find(&(&1.name == "pageleave"))+ engagement = get_events(site) |> Enum.find(&(&1.name == "engagement"))- assert Map.get(pageleave, :"meta.key") == ["bool_test", "number_test"]- assert Map.get(pageleave, :"meta.value") == ["true", "12"]+ assert Map.get(engagement, :"meta.key") == ["bool_test", "number_test"]+ assert Map.get(engagement, :"meta.value") == ["true", "12"]endtest "filters out bad props", %{conn: conn, site: site} do@@ -1302,54 +1302,54 @@ defmodule PlausibleWeb.Api.ExternalControllerTest dotest "ingests scroll_depth as 255 when sd not in params", %{conn: conn, site: site} dopost(conn, "/api/event", %{n: "pageview", u: "https://test.com", d: site.domain})- post(conn, "/api/event", %{n: "pageleave", u: "https://test.com", d: site.domain})+ post(conn, "/api/event", %{n: "engagement", u: "https://test.com", d: site.domain})- pageleave = get_events(site) |> Enum.find(&(&1.name == "pageleave"))+ engagement = get_events(site) |> Enum.find(&(&1.name == "engagement"))- assert pageleave.scroll_depth == 255+ assert engagement.scroll_depth == 255end- test "sd field is ignored if name is not pageleave", %{conn: conn, site: site} do+ test "sd field is ignored if name is not engagement", %{conn: conn, site: site} dopost(conn, "/api/event", %{n: "pageview", u: "https://test.com", d: site.domain, sd: 10})post(conn, "/api/event", %{n: "custom_e", u: "https://test.com", d: site.domain, sd: 10})assert [%{scroll_depth: 0}, %{scroll_depth: 0}] = get_events(site)end- test "ingests valid scroll_depth for a pageleave", %{conn: conn, site: site} do+ test "ingests valid scroll_depth for a engagement", %{conn: conn, site: site} dopost(conn, "/api/event", %{n: "pageview", u: "https://test.com", d: site.domain})- post(conn, "/api/event", %{n: "pageleave", u: "https://test.com", d: site.domain, sd: 25})+ post(conn, "/api/event", %{n: "engagement", u: "https://test.com", d: site.domain, sd: 25})- pageleave = get_events(site) |> Enum.find(&(&1.name == "pageleave"))+ engagement = get_events(site) |> Enum.find(&(&1.name == "engagement"))- assert pageleave.scroll_depth == 25+ assert engagement.scroll_depth == 25endtest "ingests scroll_depth as 100 when sd > 100", %{conn: conn, site: site} dopost(conn, "/api/event", %{n: "pageview", u: "https://test.com", d: site.domain})- post(conn, "/api/event", %{n: "pageleave", u: "https://test.com", d: site.domain, sd: 101})+ post(conn, "/api/event", %{n: "engagement", u: "https://test.com", d: site.domain, sd: 101})- pageleave = get_events(site) |> Enum.find(&(&1.name == "pageleave"))+ engagement = get_events(site) |> Enum.find(&(&1.name == "engagement"))- assert pageleave.scroll_depth == 100+ assert engagement.scroll_depth == 100endtest "ingests scroll_depth as 255 when sd is a string", %{conn: conn, site: site} dopost(conn, "/api/event", %{n: "pageview", u: "https://test.com", d: site.domain})- post(conn, "/api/event", %{n: "pageleave", u: "https://test.com", d: site.domain, sd: "1"})+ post(conn, "/api/event", %{n: "engagement", u: "https://test.com", d: site.domain, sd: "1"})- pageleave = get_events(site) |> Enum.find(&(&1.name == "pageleave"))+ engagement = get_events(site) |> Enum.find(&(&1.name == "engagement"))- assert pageleave.scroll_depth == 255+ assert engagement.scroll_depth == 255endtest "ingests scroll_depth as 255 when sd is a negative integer", %{conn: conn, site: site} dopost(conn, "/api/event", %{n: "pageview", u: "https://test.com", d: site.domain})- post(conn, "/api/event", %{n: "pageleave", u: "https://test.com", d: site.domain, sd: -1})+ post(conn, "/api/event", %{n: "engagement", u: "https://test.com", d: site.domain, sd: -1})- pageleave = get_events(site) |> Enum.find(&(&1.name == "pageleave"))+ engagement = get_events(site) |> Enum.find(&(&1.name == "engagement"))- assert pageleave.scroll_depth == 255+ assert engagement.scroll_depth == 255endtest "ingests valid scroll_depth for a engagement event", %{conn: conn, site: site} dodiff --git a/test/plausible_web/controllers/api/external_stats_controller/query_imported_test.exs b/test/plausible_web/controllers/api/external_stats_controller/query_imported_test.exsindex 9557b9f35e..c3c92b30ae 100644--- a/test/plausible_web/controllers/api/external_stats_controller/query_imported_test.exs+++ b/test/plausible_web/controllers/api/external_stats_controller/query_imported_test.exs@@ -1524,7 +1524,6 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryImportedTest dorefute meta["metric_warnings"]["scroll_depth"]end- @tag skip: "To be re-enabled in the next PR"test "does not return warning when imported scroll depth exists", %{conn: conn,site: site@@ -1548,8 +1547,8 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryImportedTest dobuild(:imported_pages,page: "/",date: ~D[2021-02-01],- scroll_depth: 80,- pageleave_visitors: 1+ total_scroll_depth: 80,+ total_scroll_depth_visits: 1),build(:imported_pages, page: "/", date: ~D[2021-02-28])])diff --git a/test/plausible_web/controllers/api/external_stats_controller/query_test.exs b/test/plausible_web/controllers/api/external_stats_controller/query_test.exsindex c1882d6c37..a5acd797b7 100644--- a/test/plausible_web/controllers/api/external_stats_controller/query_test.exs+++ b/test/plausible_web/controllers/api/external_stats_controller/query_test.exs@@ -3892,6 +3892,35 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest do]end+ test "scroll_depth metric is session-based", %{conn: conn, site: site} do+ t0 = ~N[2020-01-01 00:00:00]+ t1 = NaiveDateTime.add(t0, 10, :minute)+ t2 = NaiveDateTime.add(t0, 3, :hour)++ populate_stats(site, [+ build(:pageview, user_id: 12, pathname: "/blog", timestamp: t0),+ build(:engagement, user_id: 12, pathname: "/blog", timestamp: t0, scroll_depth: 20),+ build(:pageview, user_id: 12, pathname: "/blog", timestamp: t1),+ build(:engagement, user_id: 12, pathname: "/blog", timestamp: t1, scroll_depth: 40),+ # Different session+ build(:pageview, user_id: 12, pathname: "/blog", timestamp: t2),+ build(:engagement, user_id: 12, pathname: "/blog", timestamp: t2, scroll_depth: 80)+ ])++ conn =+ post(conn, "/api/v2/query-internal-test", %{+ "site_id" => site.domain,+ "metrics" => ["scroll_depth"],+ "date_range" => "all",+ "dimensions" => ["event:page"]+ })++ assert json_response(conn, 200)["results"] == [+ # avg(max(20, 40), 80) = 60+ %{"dimensions" => ["/blog"], "metrics" => [60]}+ ]+ end+test "can sort by scroll_depth in event:page breakdown", %{conn: conn, site: site} dot0 = ~N[2020-01-01 00:00:00][t1, t2, t3] = for i <- 1..3, do: NaiveDateTime.add(t0, i, :minute)diff --git a/test/plausible_web/controllers/api/stats_controller/main_graph_test.exs b/test/plausible_web/controllers/api/stats_controller/main_graph_test.exsindex fbdfaccdd5..3b36f463a6 100644--- a/test/plausible_web/controllers/api/stats_controller/main_graph_test.exs+++ b/test/plausible_web/controllers/api/stats_controller/main_graph_test.exs@@ -634,7 +634,6 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest doassert plot == [40, 20, 0, 0, 0, 0, 0]end- @tag skip: "To be re-enabled in the next PR"test "returns scroll depth per day with imported data", %{conn: conn, site: site} dosite_import = insert(:site_import, site: site)@@ -653,18 +652,18 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest dodate: ~D[2020-01-02],page: "/",visitors: 1,- scroll_depth: 40,- pageleave_visitors: 1+ total_scroll_depth: 40,+ total_scroll_depth_visits: 1),# 2020-01-03 - only imported databuild(:imported_pages,date: ~D[2020-01-03],page: "/",visitors: 1,- scroll_depth: 90,- pageleave_visitors: 1+ total_scroll_depth: 90,+ total_scroll_depth_visits: 1),- build(:imported_pages, date: ~D[2020-01-03], page: "/", visitors: 100, scroll_depth: nil)+ build(:imported_pages, date: ~D[2020-01-03], page: "/", visitors: 100)])filters = Jason.encode!([[:is, "event:page", ["/"]]])diff --git a/test/plausible_web/controllers/api/stats_controller/pages_test.exs b/test/plausible_web/controllers/api/stats_controller/pages_test.exsindex d4d363b0d3..673b091655 100644--- a/test/plausible_web/controllers/api/stats_controller/pages_test.exs+++ b/test/plausible_web/controllers/api/stats_controller/pages_test.exs@@ -676,7 +676,6 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do}end- @tag skip: "To be re-enabled in the next PR"test "calculates scroll_depth from native and imported data combined", %{conn: conn,site: site@@ -695,8 +694,8 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest dopageviews: 3,time_on_page: 90,page: "/blog",- scroll_depth: 120,- pageleave_visitors: 3+ total_scroll_depth: 120,+ total_scroll_depth_visits: 3)])@@ -718,7 +717,6 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do]end- @tag skip: "To be re-enabled in the next PR"test "handles missing scroll_depth data from native and imported sources", %{conn: conn,site: site@@ -752,8 +750,8 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest dopageviews: 4,time_on_page: 180,page: "/native-and-imported",- scroll_depth: 120,- pageleave_visitors: 3+ total_scroll_depth: 120,+ total_scroll_depth_visits: 3),build(:imported_pages,date: ~D[2020-01-01],@@ -761,8 +759,8 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest dopageviews: 30,time_on_page: 300,page: "/imported-only",- scroll_depth: 100,- pageleave_visitors: 10+ total_scroll_depth: 100,+ total_scroll_depth_visits: 10)])@@ -800,7 +798,6 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do]end- @tag skip: "To be re-enabled in the next PR"test "can query scroll depth only from imported data, ignoring rows where scroll depth doesn't exist",%{conn: conn,@@ -812,15 +809,15 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest dovisitors: 10,pageviews: 10,page: "/blog",- scroll_depth: 100,- pageleave_visitors: 10+ total_scroll_depth: 100,+ total_scroll_depth_visits: 10),build(:imported_pages,date: ~D[2020-01-01],visitors: 100,pageviews: 150,page: "/blog",- scroll_depth: nil+ total_scroll_depth: 0)])@@ -1140,7 +1137,6 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do]end- @tag skip: "To be re-enabled in the next PR"test "returns scroll depth warning code", %{conn: conn, site: site} doPlausible.Sites.set_scroll_depth_visible_at(site)@@ -1153,7 +1149,6 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do"no_imported_scroll_depth"end- @tag skip: "To be re-enabled in the next PR"test "returns imported pages with a pageview goal filter", %{conn: conn, site: site} doinsert(:goal, site: site, page_path: "/blog**")diff --git a/test/plausible_web/controllers/api/stats_controller/top_stats_test.exs b/test/plausible_web/controllers/api/stats_controller/top_stats_test.exsindex d0fcfd9134..7075108e2c 100644--- a/test/plausible_web/controllers/api/stats_controller/top_stats_test.exs+++ b/test/plausible_web/controllers/api/stats_controller/top_stats_test.exs@@ -1018,7 +1018,6 @@ defmodule PlausibleWeb.Api.StatsController.TopStatsTest do]end- @tag skip: "To be re-enabled in the next PR"test "returns scroll_depth with a page filter with imported data", %{conn: conn, site: site} dosite_import =insert(:site_import, site: site, start_date: ~D[2021-01-01], has_scroll_depth: true)@@ -1034,10 +1033,10 @@ defmodule PlausibleWeb.Api.StatsController.TopStatsTest dopage: "/",date: ~D[2021-01-01],visitors: 8,- scroll_depth: 410,- pageleave_visitors: 8+ total_scroll_depth: 410,+ total_scroll_depth_visits: 8),- build(:imported_pages, page: "/", date: ~D[2021-01-02], visitors: 100, scroll_depth: nil)+ build(:imported_pages, page: "/", date: ~D[2021-01-02], visitors: 100)])filters = Jason.encode!([[:is, "event:page", ["/"]]])diff --git a/test/plausible_web/controllers/stats_controller_test.exs b/test/plausible_web/controllers/stats_controller_test.exsindex 96be532488..f9b30d67f7 100644--- a/test/plausible_web/controllers/stats_controller_test.exs+++ b/test/plausible_web/controllers/stats_controller_test.exs@@ -748,7 +748,7 @@ defmodule PlausibleWeb.StatsControllerTest do["2020-01-02", "0", "0", "0", "0.0", "0.0", "", ""],["2020-01-03", "0", "0", "0", "0.0", "0.0", "", ""],["2020-01-04", "0", "0", "0", "0.0", "0.0", "", ""],- ["2020-01-05", "1", "2", "2", "1.0", "100", "0", "40"],+ ["2020-01-05", "1", "2", "2", "1.0", "100", "0", "28"],["2020-01-06", "0", "0", "0", "0.0", "0.0", "", ""],["2020-01-07", "1", "1", "1", "1.0", "100", "0", "90"],[""]