Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.
TypeScript analyticsanalytics-dashboardclickhouseelixirgoogle-analyticsmarketingopen-source-analyticsphoenixplausible-analyticspostgresqlprivacyprivacy-friendlyself-hostedsimple-analyticsstatisticstailwindcssweb-analyticswebsite-analyticswebsite-statswebsite-tracking
Average Scroll Depth Metric: imported data (#4915)
RobertJoonas3473910c97a3
diff --git a/lib/plausible/exports.ex b/lib/plausible/exports.exindex 1501944375..46255d628c 100644--- a/lib/plausible/exports.ex+++ b/lib/plausible/exports.ex@@ -8,13 +8,15 @@ defmodule Plausible.Exports doimport Ecto.Query@doc "Schedules CSV export job to S3 storage"- @spec schedule_s3_export(pos_integer, String.t()) :: {:ok, Oban.Job.t()} | {:error, :no_data}- def schedule_s3_export(site_id, email_to) do+ @spec schedule_s3_export(pos_integer, pos_integer | nil, String.t()) ::+ {:ok, Oban.Job.t()} | {:error, :no_data}+ def schedule_s3_export(site_id, current_user_id, email_to) dowith :ok <- ensure_has_data(site_id) doargs = %{"storage" => "s3","site_id" => site_id,"email_to" => email_to,+ "current_user_id" => current_user_id,"s3_bucket" => Plausible.S3.exports_bucket(),"s3_path" => s3_export_key(site_id)}@@ -207,13 +209,13 @@ defmodule Plausible.Exports doBuilds Ecto queries to export data from `events_v2` and `sessions_v2`tables into the format of `imported_*` tables for a website."""- @spec export_queries(pos_integer,+ @spec export_queries(pos_integer, pos_integer | nil,extname: String.t(),date_range: Date.Range.t(),timezone: String.t()) ::%{String.t() => Ecto.Query.t()}- def export_queries(site_id, opts \\ []) do+ def export_queries(site_id, current_user_id, opts \\ []) doextname = opts[:extname] || ".csv"date_range = opts[:date_range]timezone = opts[:timezone] || "UTC"@@ -232,7 +234,8 @@ defmodule Plausible.Exports do%{filename.("imported_visitors") => export_visitors_q(site_id, timezone, date_range),filename.("imported_sources") => export_sources_q(site_id, timezone, date_range),- filename.("imported_pages") => export_pages_q(site_id, timezone, date_range),+ filename.("imported_pages") =>+ export_pages_q(site_id, current_user_id, timezone, date_range),filename.("imported_entry_pages") => export_entry_pages_q(site_id, timezone, date_range),filename.("imported_exit_pages") => export_exit_pages_q(site_id, timezone, date_range),filename.("imported_custom_events") =>@@ -411,13 +414,71 @@ defmodule Plausible.Exports do]end- defp export_pages_q(site_id, timezone, date_range) do- from e in sampled("events_v2"),- where: ^export_filter(site_id, date_range),- where: [name: "pageview"],- group_by: [selected_as(:date), e.pathname],- order_by: selected_as(:date),- select: [+ defp export_pages_q(site_id, current_user_id, timezone, date_range) do+ site = Plausible.Repo.get(Plausible.Site, site_id)+ current_user = current_user_id && Plausible.Repo.get(Plausible.Auth.User, current_user_id)++ scroll_depth_enabled? =+ PlausibleWeb.Api.StatsController.scroll_depth_enabled?(site, current_user)++ base_q =+ from(e in sampled("events_v2"),+ where: ^export_filter(site_id, date_range),+ where: [name: "pageview"],+ group_by: [selected_as(:date), selected_as(:page)],+ order_by: selected_as(:date)+ )++ if scroll_depth_enabled? do+ max_scroll_depth_per_visitor_q =+ from(e in "events_v2",+ where: ^export_filter(site_id, date_range),+ where: e.name == "pageleave" and e.scroll_depth <= 100,+ select: %{+ date: date(e.timestamp, ^timezone),+ page: selected_as(e.pathname, :page),+ user_id: e.user_id,+ max_scroll_depth: max(e.scroll_depth)+ },+ group_by: [e.user_id, selected_as(:date), selected_as(:page)]+ )++ scroll_depth_q =+ from(p in subquery(max_scroll_depth_per_visitor_q),+ select: %{+ date: p.date,+ page: p.page,+ scroll_depth:+ fragment(+ "if(isNull(sum(?)), NULL, toUInt64(sum(?)))",+ p.max_scroll_depth,+ p.max_scroll_depth+ ),+ pageleave_visitors: count(p.user_id)+ },+ group_by: [:date, :page]+ )++ from(e in base_q,+ left_join: s in subquery(scroll_depth_q),+ on: s.date == selected_as(:date) and s.page == selected_as(:page),+ select: [+ date(e.timestamp, ^timezone),+ selected_as(fragment("any(?)", e.hostname), :hostname),+ selected_as(e.pathname, :page),+ selected_as(+ fragment("toUInt64(round(uniq(?)*any(_sample_factor)))", e.session_id),+ :visits+ ),+ visitors(e),+ selected_as(fragment("toUInt64(round(count()*any(_sample_factor)))"), :pageviews),+ selected_as(fragment("any(?)", s.scroll_depth), :scroll_depth),+ selected_as(fragment("any(?)", s.pageleave_visitors), :pageleave_visitors)+ ]+ )+ else+ base_q+ |> select([e], [date(e.timestamp, ^timezone),selected_as(fragment("any(?)", e.hostname), :hostname),selected_as(e.pathname, :page),@@ -427,7 +488,8 @@ defmodule Plausible.Exports do),visitors(e),selected_as(fragment("toUInt64(round(count()*any(_sample_factor)))"), :pageviews)- ]+ ])+ endenddefp export_entry_pages_q(site_id, timezone, date_range) dodiff --git a/lib/plausible/imported/csv_importer.ex b/lib/plausible/imported/csv_importer.exindex ff763e304b..277a9dd8da 100644--- a/lib/plausible/imported/csv_importer.ex+++ b/lib/plausible/imported/csv_importer.ex@@ -161,7 +161,7 @@ defmodule Plausible.Imported.CSVImporter do"imported_operating_systems" =>"date Date, operating_system String, operating_system_version String, visitors UInt64, visits UInt64, visit_duration UInt64, bounces UInt32, pageviews UInt64","imported_pages" =>- "date Date, hostname String, page String, visits UInt64, visitors UInt64, pageviews UInt64",+ "date Date, hostname String, page String, visits UInt64, visitors UInt64, pageviews UInt64, scroll_depth Nullable(UInt64), pageleave_visitors UInt64","imported_sources" =>"date Date, source String, referrer String, utm_source String, utm_medium String, utm_campaign String, utm_content String, utm_term String, pageviews UInt64, visitors UInt64, visits UInt64, visit_duration UInt64, bounces UInt32","imported_visitors" =>diff --git a/lib/plausible/imported/page.ex b/lib/plausible/imported/page.exindex 4720bef4f4..d4ba204a34 100644--- a/lib/plausible/imported/page.ex+++ b/lib/plausible/imported/page.ex@@ -15,5 +15,7 @@ defmodule Plausible.Imported.Page dofield :pageviews, Ch, type: "UInt64"field :exits, Ch, type: "UInt64"field :time_on_page, Ch, type: "UInt64"+ field :scroll_depth, Ch, type: "Nullable(UInt64)"+ field :pageleave_visitors, Ch, type: "UInt64"endenddiff --git a/lib/plausible/stats/imported/imported.ex b/lib/plausible/stats/imported/imported.exindex b05e1d2112..9f7b038595 100644--- a/lib/plausible/stats/imported/imported.ex+++ b/lib/plausible/stats/imported/imported.ex@@ -345,6 +345,7 @@ defmodule Plausible.Stats.Imported dodefp can_order_by?(query) doEnum.all?(query.order_by, fn+ {:scroll_depth, _} -> false{metric, _direction} when is_atom(metric) -> metric in query.metrics_ -> trueend)diff --git a/lib/plausible/stats/imported/sql/expression.ex b/lib/plausible/stats/imported/sql/expression.exindex 057bc22d69..d0fbfd4bf7 100644--- a/lib/plausible/stats/imported/sql/expression.ex+++ b/lib/plausible/stats/imported/sql/expression.ex@@ -115,6 +115,13 @@ defmodule Plausible.Stats.Imported.SQL.Expression dowrap_alias([i], %{pageviews: sum(i.pageviews), __internal_visits: sum(i.visits)})end+ defp select_metric(:scroll_depth, "imported_pages") do+ wrap_alias([i], %{+ scroll_depth_sum: sum(i.scroll_depth),+ pageleave_visitors: sum(i.pageleave_visitors)+ })+ end+defp select_metric(_metric, _table), do: %{}def group_imported_by(q, query) do@@ -351,6 +358,19 @@ defmodule Plausible.Stats.Imported.SQL.Expression do|> select_joined_metrics(rest)end+ # The final `scroll_depth` gets selected at a later querybuilding step+ # (in `Plausible.Stats.SQL.SpecialMetrics.add/3`). But in order to avoid+ # having to join with imported data there again, we select the required+ # information from imported data here already.+ def select_joined_metrics(q, [:scroll_depth | rest]) do+ q+ |> select_merge_as([s, i], %{+ __internal_scroll_depth_sum: i.scroll_depth_sum,+ __internal_pageleave_visitors: i.pageleave_visitors+ })+ |> select_joined_metrics(rest)+ end+# Ignored as it's calculated separatelydef select_joined_metrics(q, [metric | rest])when metric in [:conversion_rate, :group_conversion_rate, :percentage] dodiff --git a/lib/plausible/stats/sql/special_metrics.ex b/lib/plausible/stats/sql/special_metrics.exindex b4a615a1cc..15eef2bed9 100644--- a/lib/plausible/stats/sql/special_metrics.ex+++ b/lib/plausible/stats/sql/special_metrics.ex@@ -150,15 +150,12 @@ defmodule Plausible.Stats.SQL.SpecialMetrics dodim_shortnames|> Enum.map(fn dim -> dynamic([p], field(p, ^dim)) end)- scroll_depth_q =+ scroll_depth_sum_q =subquery(max_per_visitor_q)|> select([p], %{- scroll_depth:- fragment(- "if(isFinite(avg(?)), toUInt8(round(avg(?))), NULL)",- p.max_scroll_depth,- p.max_scroll_depth- )+ scroll_depth_sum:+ fragment("if(count(?) = 0, NULL, sum(?))", p.user_id, p.max_scroll_depth),+ pageleave_visitors: fragment("count(?)", p.user_id)})|> select_merge(^dim_select)|> group_by(^dim_group_by)@@ -173,9 +170,55 @@ defmodule Plausible.Stats.SQL.SpecialMetrics do|> Enum.reduce(fn condition, acc -> dynamic([], ^acc and ^condition) end)end- q- |> join(:left, [e], s in subquery(scroll_depth_q), on: ^join_on_dim_condition)- |> select_merge_as([_e, ..., s], %{scroll_depth: fragment("any(?)", s.scroll_depth)})+ joined_q =+ join(q, :left, [e], s in subquery(scroll_depth_sum_q), on: ^join_on_dim_condition)++ if query.include_imported do+ joined_q+ |> select_merge_as([..., s], %{+ scroll_depth:+ fragment(+ """+ case+ when isNotNull(?) AND isNotNull(?) then+ toUInt8(round((? + ?) / (? + ?)))+ when isNotNull(?) then+ toUInt8(round(? / ?))+ when isNotNull(?) then+ toUInt8(round(? / ?))+ else+ NULL+ end+ """,+ # Case 1: Both imported and native scroll depth sums are present+ selected_as(:__internal_scroll_depth_sum),+ s.scroll_depth_sum,+ selected_as(:__internal_scroll_depth_sum),+ s.scroll_depth_sum,+ selected_as(:__internal_pageleave_visitors),+ s.pageleave_visitors,+ # Case 2: Only imported scroll depth sum is present+ selected_as(:__internal_scroll_depth_sum),+ selected_as(:__internal_scroll_depth_sum),+ selected_as(:__internal_pageleave_visitors),+ # Case 3: Only native scroll depth sum is present+ s.scroll_depth_sum,+ s.scroll_depth_sum,+ s.pageleave_visitors+ )+ })+ else+ joined_q+ |> select_merge_as([..., s], %{+ scroll_depth:+ fragment(+ "if(any(?) > 0, toUInt8(round(any(?) / any(?))), NULL)",+ s.pageleave_visitors,+ s.scroll_depth_sum,+ s.pageleave_visitors+ )+ })+ endelseqenddiff --git a/lib/plausible_web/controllers/api/stats_controller.ex b/lib/plausible_web/controllers/api/stats_controller.exindex a4261e2adf..1ab8792499 100644--- a/lib/plausible_web/controllers/api/stats_controller.ex+++ b/lib/plausible_web/controllers/api/stats_controller.ex@@ -393,15 +393,16 @@ defmodule PlausibleWeb.Api.StatsController dodefp fetch_other_top_stats(site, query, current_user) dopage_filter? = Filters.filtering_on_dimension?(query, "event:page")+ scroll_depth_enabled? = scroll_depth_enabled?(site, current_user)metrics = [:visitors, :visits, :pageviews, :sample_percent]metrics =cond do- page_filter? && query.include_imported ->- metrics+ page_filter? && scroll_depth_enabled? && query.include_imported ->+ metrics ++ [:scroll_depth]- page_filter? && scroll_depth_enabled?(site, current_user) ->+ page_filter? && scroll_depth_enabled? ->metrics ++ [:bounce_rate, :scroll_depth, :time_on_page]page_filter? ->@@ -831,7 +832,7 @@ defmodule PlausibleWeb.Api.StatsController doparams = Map.put(params, "property", "event:page")query = Query.from(site, params, debug_metadata(conn))- include_scroll_depth? = !query.include_imported && scroll_depth_enabled?(site, current_user)+ include_scroll_depth? = scroll_depth_enabled?(site, current_user)extra_metrics =cond dodiff --git a/lib/plausible_web/live/csv_export.ex b/lib/plausible_web/live/csv_export.exindex f76741dc74..bdf2b1c6e4 100644--- a/lib/plausible_web/live/csv_export.ex+++ b/lib/plausible_web/live/csv_export.ex@@ -210,11 +210,12 @@ defmodule PlausibleWeb.Live.CSVExport do@impl truedef handle_event("export", _params, socket) do- %{storage: storage, site_id: site_id, email_to: email_to} = socket.assigns+ %{storage: storage, site_id: site_id, email_to: email_to, current_user: current_user} =+ socket.assignsschedule_result =case storage do- "s3" -> Exports.schedule_s3_export(site_id, email_to)+ "s3" -> Exports.schedule_s3_export(site_id, current_user.id, email_to)"local" -> Exports.schedule_local_export(site_id, email_to)enddiff --git a/lib/workers/export_analytics.ex b/lib/workers/export_analytics.exindex 51a1c0af06..d5da9f05d0 100644--- a/lib/workers/export_analytics.ex+++ b/lib/workers/export_analytics.ex@@ -29,11 +29,13 @@ defmodule Plausible.Workers.ExportAnalytics do"site_id" => site_id} = args+ current_user_id = args["current_user_id"]+site = Plausible.Repo.get!(Plausible.Site, site_id)%Date.Range{} = date_range = Exports.date_range(site.id, site.timezone)queries =- Exports.export_queries(site_id,+ Exports.export_queries(site_id, current_user_id,date_range: date_range,timezone: site.timezone,extname: ".csv"diff --git a/test/plausible/exports_test.exs b/test/plausible/exports_test.exsindex eebc8d1136..9c320235ff 100644--- a/test/plausible/exports_test.exs+++ b/test/plausible/exports_test.exs@@ -7,7 +7,7 @@ defmodule Plausible.ExportsTest dodescribe "export_queries/2" dotest "returns named ecto queries" do- queries = Plausible.Exports.export_queries(_site_id = 1)+ queries = Plausible.Exports.export_queries(_site_id = 1, nil)assert queries |> Map.values() |> Enum.all?(&match?(%Ecto.Query{}, &1))assert Map.keys(queries) == [@@ -26,7 +26,7 @@ defmodule Plausible.ExportsTest dotest "with date range" doqueries =- Plausible.Exports.export_queries(_site_id = 1,+ Plausible.Exports.export_queries(_site_id = 1, nil,date_range: Date.range(~D[2023-01-01], ~D[2024-03-12]))@@ -45,10 +45,7 @@ defmodule Plausible.ExportsTest doendtest "with custom extension" do- queries =- Plausible.Exports.export_queries(_site_id = 1,- extname: ".ch"- )+ queries = Plausible.Exports.export_queries(_site_id = 1, nil, extname: ".ch")assert Map.keys(queries) == ["imported_browsers.ch",diff --git a/test/plausible/imported/csv_importer_test.exs b/test/plausible/imported/csv_importer_test.exsindex 7e5831a952..efc4b05443 100644--- a/test/plausible/imported/csv_importer_test.exs+++ b/test/plausible/imported/csv_importer_test.exs@@ -363,6 +363,71 @@ defmodule Plausible.Imported.CSVImporterTest doassert Plausible.Stats.Clickhouse.imported_pageview_count(site) == 99end+ test "imports scroll_depth as null when the column does not exist in pages CSV",+ %{site: site, user: user} = ctx do+ _ = ctx++ pages_csv =+ %{+ name: "imported_pages_20211230_20220101.csv",+ body: """+ "date","visitors","pageviews","hostname","page"+ "2021-12-30",1,1,"lucky.numbers.com","/14776416252794997127"+ "2021-12-30",1,1,"lucky.numbers.com","/14776416252794997127"+ "2021-12-30",6,6,"lucky.numbers.com","/14776416252794997127"+ "2021-12-30",1,1,"lucky.numbers.com","/9102354072466236765"+ "2021-12-30",1,1,"lucky.numbers.com","/7478911940502018071"+ "2021-12-30",1,1,"lucky.numbers.com","/6402607186523575652"+ "2021-12-30",2,2,"lucky.numbers.com","/9962503789684934900"+ "2021-12-30",8,10,"lucky.numbers.com","/13595620304963848161"+ "2021-12-30",2,2,"lucky.numbers.com","/17019199732013993436"+ "2021-12-30",32,33,"lucky.numbers.com","/9874837495456455794"+ "2021-12-31",4,4,"lucky.numbers.com","/14776416252794997127"+ "2021-12-31",1,1,"lucky.numbers.com","/8738789417178304429"+ "2021-12-31",1,1,"lucky.numbers.com","/7445073500314667742"+ "2021-12-31",1,1,"lucky.numbers.com","/4897404798407749335"+ "2021-12-31",1,2,"lucky.numbers.com","/11263893625781431659"+ "2022-01-01",2,2,"lucky.numbers.com","/5878724061840196349"+ """+ }++ uploads =+ on_ee do+ %{s3_url: s3_url} = Plausible.S3.import_presign_upload(site.id, pages_csv.name)+ [bucket, key] = String.split(URI.parse(s3_url).path, "/", parts: 2)+ ExAws.request!(ExAws.S3.put_object(bucket, key, pages_csv.body))+ %{"filename" => pages_csv.name, "s3_url" => s3_url}+ else+ local_path = Path.join(ctx.tmp_dir, pages_csv.name)+ File.write!(local_path, pages_csv.body)+ %{"filename" => pages_csv.name, "local_path" => local_path}+ end+ |> List.wrap()++ date_range = CSVImporter.date_range(uploads)++ {:ok, _job} =+ CSVImporter.new_import(site, user,+ start_date: date_range.first,+ end_date: date_range.last,+ uploads: uploads,+ storage: on_ee(do: "s3", else: "local")+ )++ assert %{success: 1} = Oban.drain_queue(queue: :analytics_imports, with_safety?: false)++ assert %SiteImport{+ start_date: ~D[2021-12-30],+ end_date: ~D[2022-01-01],+ source: :csv,+ status: :completed+ } = Repo.get_by!(SiteImport, site_id: site.id)++ q = from(i in "imported_pages", where: i.site_id == ^site.id, select: i.scroll_depth)++ assert List.duplicate(nil, 16) == Plausible.IngestRepo.all(q)+ end+test "accepts cells without quotes", %{site: site, user: user} = ctx do_ = ctx@@ -524,7 +589,8 @@ defmodule Plausible.Imported.CSVImporterTest do# export archive to s3on_ee do- assert {:ok, _job} = Plausible.Exports.schedule_s3_export(exported_site.id, user.email)+ assert {:ok, _job} =+ Plausible.Exports.schedule_s3_export(exported_site.id, nil, user.email)elseassert {:ok, %{args: %{"local_path" => local_path}}} =Plausible.Exports.schedule_local_export(exported_site.id, user.email)@@ -1016,6 +1082,152 @@ defmodule Plausible.Imported.CSVImporterTest doassert exported["conversion_rate"] == imported["conversion_rate"]end)end++ @tag :tmp_dir+ test "scroll_depth", %{conn: conn, user: user, tmp_dir: tmp_dir} do+ exported_site = new_site(owner: user)+ imported_site = new_site(owner: user)++ t0 = ~N[2020-01-01 00:00:00]+ [t1, t2, t3] = for i <- 1..3, do: NaiveDateTime.add(t0, i, :minute)++ stats =+ [+ build(:pageview, user_id: 12, pathname: "/blog", timestamp: t0),+ build(:pageleave, user_id: 12, pathname: "/blog", timestamp: t1, scroll_depth: 20),+ build(:pageview, user_id: 12, pathname: "/another", timestamp: t1),+ build(:pageleave, user_id: 12, pathname: "/another", timestamp: t2, scroll_depth: 24),+ build(:pageview, user_id: 34, pathname: "/blog", timestamp: t0),+ build(:pageleave, user_id: 34, pathname: "/blog", timestamp: t1, scroll_depth: 17),+ build(:pageview, user_id: 34, pathname: "/another", timestamp: t1),+ build(:pageleave, user_id: 34, pathname: "/another", timestamp: t2, scroll_depth: 26),+ build(:pageview, user_id: 34, pathname: "/blog", timestamp: t2),+ build(:pageleave, user_id: 34, pathname: "/blog", timestamp: t3, scroll_depth: 60),+ build(:pageview, user_id: 56, pathname: "/blog", timestamp: t0),+ build(:pageleave, user_id: 56, pathname: "/blog", timestamp: t1, scroll_depth: 100),+ build(:pageview, user_id: 78, pathname: "/", timestamp: t0),+ build(:pageleave, user_id: 78, pathname: "/", timestamp: t1, scroll_depth: 20),+ build(:pageview, pathname: "/", timestamp: t1),+ build(:pageview, pathname: "/blog", timestamp: NaiveDateTime.add(t0, 1, :day))+ ]+ |> Enum.map(fn event -> Map.put(event, :hostname, "csv.test") end)++ populate_stats(exported_site, stats)++ # export archive to s3+ on_ee do+ assert {:ok, _job} =+ Plausible.Exports.schedule_s3_export(exported_site.id, nil, user.email)+ else+ assert {:ok, %{args: %{"local_path" => local_path}}} =+ Plausible.Exports.schedule_local_export(exported_site.id, user.email)+ end++ assert %{success: 1} = Oban.drain_queue(queue: :analytics_exports, with_safety: false)++ # download archive+ on_ee do+ ExAws.request!(+ ExAws.S3.download_file(+ Plausible.S3.exports_bucket(),+ to_string(exported_site.id),+ Path.join(tmp_dir, "plausible-export.zip")+ )+ )+ else+ File.rename!(local_path, Path.join(tmp_dir, "plausible-export.zip"))+ end++ # unzip archive+ {:ok, files} =+ :zip.unzip(to_charlist(Path.join(tmp_dir, "plausible-export.zip")), cwd: tmp_dir)++ # upload csvs+ uploads =+ Enum.map(files, fn file ->+ on_ee do+ %{s3_url: s3_url} = Plausible.S3.import_presign_upload(imported_site.id, file)+ [bucket, key] = String.split(URI.parse(s3_url).path, "/", parts: 2)+ content = File.read!(file)+ ExAws.request!(ExAws.S3.put_object(bucket, key, content))+ %{"filename" => Path.basename(file), "s3_url" => s3_url}+ else+ %{+ "filename" => Path.basename(file),+ "local_path" => file+ }+ end+ end)++ # run importer+ date_range = CSVImporter.date_range(uploads)++ {:ok, _job} =+ CSVImporter.new_import(imported_site, user,+ start_date: date_range.first,+ end_date: date_range.last,+ uploads: uploads,+ storage: on_ee(do: "s3", else: "local")+ )++ assert %{success: 1} = Oban.drain_queue(queue: :analytics_imports, with_safety: false)++ # validate import+ assert %SiteImport{+ start_date: ~D[2020-01-01],+ end_date: ~D[2020-01-02],+ source: :csv,+ status: :completed+ } = Repo.get_by!(SiteImport, site_id: imported_site.id)++ assert Plausible.Stats.Clickhouse.imported_pageview_count(exported_site) == 0+ assert Plausible.Stats.Clickhouse.imported_pageview_count(imported_site) == 9++ # assert on the actual rows that got imported into the imported_pages table+ imported_data =+ from(i in "imported_pages",+ where: i.site_id == ^imported_site.id,+ select: %{+ date: i.date,+ page: i.page,+ scroll_depth: i.scroll_depth,+ pageleave_visitors: i.pageleave_visitors+ }+ )+ |> Plausible.IngestRepo.all()++ assert %{date: ~D[2020-01-01], page: "/", scroll_depth: 20, pageleave_visitors: 1} in imported_data++ assert %{date: ~D[2020-01-01], page: "/another", scroll_depth: 50, pageleave_visitors: 2} in imported_data++ assert %{date: ~D[2020-01-01], page: "/blog", scroll_depth: 180, pageleave_visitors: 3} in imported_data++ assert %{date: ~D[2020-01-02], page: "/blog", scroll_depth: nil, pageleave_visitors: 0} in imported_data++ # assert via stats queries that scroll_depth from imported+ # data matches the scroll_depth from native data+ expected_results = [+ %{"dimensions" => ["/"], "metrics" => [20]},+ %{"dimensions" => ["/another"], "metrics" => [25]},+ %{"dimensions" => ["/blog"], "metrics" => [60]}+ ]++ query_scroll_depth_per_page = fn conn, site ->+ post(conn, "/api/v2/query-internal-test", %{+ "site_id" => site.domain,+ "metrics" => ["scroll_depth"],+ "date_range" => "all",+ "order_by" => [["scroll_depth", "asc"]],+ "include" => %{"imports" => true},+ "dimensions" => ["event:page"]+ })+ |> json_response(200)+ |> Map.fetch!("results")+ end++ assert query_scroll_depth_per_page.(conn, exported_site) == expected_results+ assert query_scroll_depth_per_page.(conn, imported_site) == expected_results+ endenddefp clean_buckets(_context) dodiff --git a/test/plausible_web/controllers/api/stats_controller/imported_test.exs b/test/plausible_web/controllers/api/stats_controller/imported_test.exsindex 4e36ba8eff..048fc2ccbf 100644--- a/test/plausible_web/controllers/api/stats_controller/imported_test.exs+++ b/test/plausible_web/controllers/api/stats_controller/imported_test.exs@@ -882,6 +882,7 @@ defmodule PlausibleWeb.Api.StatsController.ImportedTest do"time_on_page" => 40,"visitors" => 3,"pageviews" => 4,+ "scroll_depth" => nil,"name" => "/some-other-page"},%{@@ -889,6 +890,7 @@ defmodule PlausibleWeb.Api.StatsController.ImportedTest do"time_on_page" => 800.0,"visitors" => 2,"pageviews" => 2,+ "scroll_depth" => nil,"name" => "/"}]diff --git a/test/plausible_web/controllers/api/stats_controller/main_graph_test.exs b/test/plausible_web/controllers/api/stats_controller/main_graph_test.exsindex 47704e6167..38a05f725b 100644--- a/test/plausible_web/controllers/api/stats_controller/main_graph_test.exs+++ b/test/plausible_web/controllers/api/stats_controller/main_graph_test.exs@@ -633,6 +633,51 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest doassert plot == [40, 20, 0, 0, 0, 0, 0]end++ test "returns scroll depth per day with imported data", %{conn: conn, site: site} do+ site_import = insert(:site_import, site: site)++ populate_stats(site, site_import.id, [+ # 2020-01-01 - only native data+ build(:pageview, user_id: 12, timestamp: ~N[2020-01-01 00:00:00]),+ build(:pageleave, user_id: 12, timestamp: ~N[2020-01-01 00:01:00], scroll_depth: 20),+ build(:pageview, user_id: 34, timestamp: ~N[2020-01-01 00:00:00]),+ build(:pageleave, user_id: 34, timestamp: ~N[2020-01-01 00:01:00], scroll_depth: 17),+ build(:pageview, user_id: 34, timestamp: ~N[2020-01-01 00:02:00]),+ build(:pageleave, user_id: 34, timestamp: ~N[2020-01-01 00:03:00], scroll_depth: 60),+ # 2020-01-02 - both imported and native data+ build(:pageview, user_id: 56, timestamp: ~N[2020-01-02 00:00:00]),+ build(:pageleave, user_id: 56, timestamp: ~N[2020-01-02 00:01:00], scroll_depth: 20),+ build(:imported_pages,+ date: ~D[2020-01-02],+ page: "/",+ visitors: 1,+ scroll_depth: 40,+ pageleave_visitors: 1+ ),+ # 2020-01-03 - only imported data+ build(:imported_pages,+ date: ~D[2020-01-03],+ page: "/",+ visitors: 1,+ scroll_depth: 90,+ pageleave_visitors: 1+ ),+ build(:imported_pages, date: ~D[2020-01-03], page: "/", visitors: 100, scroll_depth: nil)+ ])++ filters = Jason.encode!([[:is, "event:page", ["/"]]])++ conn =+ get(+ conn,+ "/api/stats/#{site.domain}/main-graph?period=7d&date=2020-01-07&metric=scroll_depth&filters=#{filters}&with_imported=true"+ )++ assert %{"plot" => plot} = json_response(conn, 200)++ assert plot == [40, 30, 90, 0, 0, 0, 0]+ endenddescribe "GET /api/stats/main-graph - conversion_rate plot" dodiff --git a/test/plausible_web/controllers/api/stats_controller/pages_test.exs b/test/plausible_web/controllers/api/stats_controller/pages_test.exsindex 9661f1edd9..6a70985668 100644--- a/test/plausible_web/controllers/api/stats_controller/pages_test.exs+++ b/test/plausible_web/controllers/api/stats_controller/pages_test.exs@@ -648,6 +648,169 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do]end+ test "calculates scroll_depth from native and imported data combined", %{+ conn: conn,+ site: site+ } do+ populate_stats(site, [+ build(:pageview, user_id: @user_id, pathname: "/blog", timestamp: ~N[2020-01-01 00:00:00]),+ build(:pageleave,+ user_id: @user_id,+ pathname: "/blog",+ timestamp: ~N[2020-01-01 00:00:00],+ scroll_depth: 80+ ),+ build(:imported_pages,+ date: ~D[2020-01-01],+ visitors: 3,+ pageviews: 3,+ time_on_page: 90,+ page: "/blog",+ scroll_depth: 120,+ pageleave_visitors: 3+ )+ ])++ conn =+ get(+ conn,+ "/api/stats/#{site.domain}/pages?period=day&date=2020-01-01&detailed=true&with_imported=true&order_by=#{Jason.encode!([["scroll_depth", "desc"]])}"+ )++ assert json_response(conn, 200)["results"] == [+ %{+ "name" => "/blog",+ "visitors" => 4,+ "pageviews" => 4,+ "bounce_rate" => 100,+ "time_on_page" => 30.0,+ "scroll_depth" => 50+ }+ ]+ end++ test "handles missing scroll_depth data from native and imported sources", %{+ conn: conn,+ site: site+ } do+ populate_stats(site, [+ build(:pageview,+ user_id: @user_id,+ pathname: "/native-and-imported",+ timestamp: ~N[2020-01-01 00:00:00]+ ),+ build(:pageleave,+ user_id: @user_id,+ pathname: "/native-and-imported",+ timestamp: ~N[2020-01-01 00:01:00],+ scroll_depth: 80+ ),+ build(:pageview,+ user_id: @user_id,+ pathname: "/native-only",+ timestamp: ~N[2020-01-01 00:01:00]+ ),+ build(:pageleave,+ user_id: @user_id,+ pathname: "/native-only",+ timestamp: ~N[2020-01-01 00:02:00],+ scroll_depth: 40+ ),+ build(:imported_pages,+ date: ~D[2020-01-01],+ visitors: 4,+ pageviews: 4,+ time_on_page: 180,+ page: "/native-and-imported",+ scroll_depth: 120,+ pageleave_visitors: 3+ ),+ build(:imported_pages,+ date: ~D[2020-01-01],+ visitors: 20,+ pageviews: 30,+ time_on_page: 300,+ page: "/imported-only",+ scroll_depth: 100,+ pageleave_visitors: 10+ )+ ])++ conn =+ get(+ conn,+ "/api/stats/#{site.domain}/pages?period=day&date=2020-01-01&detailed=true&with_imported=true&order_by=#{Jason.encode!([["scroll_depth", "desc"]])}"+ )++ assert json_response(conn, 200)["results"] == [+ %{+ "name" => "/native-and-imported",+ "visitors" => 5,+ "pageviews" => 5,+ "bounce_rate" => 0,+ "time_on_page" => 48,+ "scroll_depth" => 50+ },+ %{+ "name" => "/native-only",+ "visitors" => 1,+ "pageviews" => 1,+ "bounce_rate" => 0,+ "time_on_page" => nil,+ "scroll_depth" => 40+ },+ %{+ "name" => "/imported-only",+ "visitors" => 20,+ "pageviews" => 30,+ "bounce_rate" => 0,+ "time_on_page" => 10.0,+ "scroll_depth" => 10+ }+ ]+ end++ test "can query scroll depth only from imported data, ignoring rows where scroll depth doesn't exist",+ %{+ conn: conn,+ site: site+ } do+ populate_stats(site, [+ build(:imported_pages,+ date: ~D[2020-01-01],+ visitors: 10,+ pageviews: 10,+ page: "/blog",+ scroll_depth: 100,+ pageleave_visitors: 10+ ),+ build(:imported_pages,+ date: ~D[2020-01-01],+ visitors: 100,+ pageviews: 150,+ page: "/blog",+ scroll_depth: nil+ )+ ])++ conn =+ get(+ conn,+ "/api/stats/#{site.domain}/pages?period=7d&date=2020-01-02&detailed=true&with_imported=true"+ )++ assert json_response(conn, 200)["results"] == [+ %{+ "name" => "/blog",+ "visitors" => 110,+ "pageviews" => 160,+ "bounce_rate" => 0,+ "time_on_page" => 0.125,+ "scroll_depth" => 10+ }+ ]+ end+test "can filter using the | (OR) filter",%{conn: conn, site: site} dopopulate_stats(site, [@@ -1263,6 +1426,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do"time_on_page" => 800.0,"visitors" => 3,"pageviews" => 3,+ "scroll_depth" => nil,"name" => "/"},%{@@ -1270,6 +1434,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do"time_on_page" => 60,"visitors" => 2,"pageviews" => 2,+ "scroll_depth" => nil,"name" => "/some-other-page"}]@@ -1341,7 +1506,8 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do"name" => "/","pageviews" => 4,"time_on_page" => 90.0,- "visitors" => 4+ "visitors" => 4,+ "scroll_depth" => nil}]end@@ -1390,14 +1556,16 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do"name" => "/","pageviews" => 4,"time_on_page" => 90.0,- "visitors" => 4+ "visitors" => 4,+ "scroll_depth" => nil},%{"bounce_rate" => 100,"name" => "/a","pageviews" => 1,"time_on_page" => 10.0,- "visitors" => 1+ "visitors" => 1,+ "scroll_depth" => nil}]end@@ -1446,14 +1614,16 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do"name" => "/aaa","pageviews" => 4,"time_on_page" => 90.0,- "visitors" => 4+ "visitors" => 4,+ "scroll_depth" => nil},%{"bounce_rate" => 100,"name" => "/a","pageviews" => 1,"time_on_page" => 10.0,- "visitors" => 1+ "visitors" => 1,+ "scroll_depth" => nil}]enddiff --git a/test/plausible_web/controllers/api/stats_controller/top_stats_test.exs b/test/plausible_web/controllers/api/stats_controller/top_stats_test.exsindex f5061b6e9c..ebc7252684 100644--- a/test/plausible_web/controllers/api/stats_controller/top_stats_test.exs+++ b/test/plausible_web/controllers/api/stats_controller/top_stats_test.exs@@ -603,7 +603,7 @@ defmodule PlausibleWeb.Api.StatsController.TopStatsTest do]end- test ":is filter on page returns only visitors, visits and pageviews", %{+ test ":is filter on page returns only visitors, visits, pageviews and scroll_depth", %{conn: conn,site: site} do@@ -638,7 +638,8 @@ defmodule PlausibleWeb.Api.StatsController.TopStatsTest doassert res["top_stats"] == [%{"name" => "Unique visitors", "value" => 2, "graph_metric" => "visitors"},%{"name" => "Total visits", "value" => 4, "graph_metric" => "visits"},- %{"name" => "Total pageviews", "value" => 36, "graph_metric" => "pageviews"}+ %{"name" => "Total pageviews", "value" => 36, "graph_metric" => "pageviews"},+ %{"name" => "Scroll depth", "value" => nil, "graph_metric" => "scroll_depth"}]endend@@ -989,6 +990,41 @@ defmodule PlausibleWeb.Api.StatsController.TopStatsTest do]end+ test "returns scroll_depth with a page filter with imported data", %{conn: conn, site: site} do+ site_import = insert(:site_import, site: site)++ populate_stats(site, site_import.id, [+ build(:pageview, user_id: 123, timestamp: ~N[2021-01-01 00:00:00]),+ build(:pageleave, user_id: 123, timestamp: ~N[2021-01-01 00:00:10], scroll_depth: 40),+ build(:pageview, user_id: 123, timestamp: ~N[2021-01-01 00:00:10]),+ build(:pageleave, user_id: 123, timestamp: ~N[2021-01-01 00:00:20], scroll_depth: 60),+ build(:pageview, user_id: 456, timestamp: ~N[2021-01-01 00:00:00]),+ build(:pageleave, user_id: 456, timestamp: ~N[2021-01-01 00:00:10], scroll_depth: 80),+ build(:imported_pages,+ page: "/",+ date: ~D[2021-01-01],+ visitors: 8,+ scroll_depth: 410,+ pageleave_visitors: 8+ ),+ build(:imported_pages, page: "/", date: ~D[2021-01-02], visitors: 100, scroll_depth: nil)+ ])++ filters = Jason.encode!([[:is, "event:page", ["/"]]])++ conn =+ get(+ conn,+ "/api/stats/#{site.domain}/top-stats?period=7d&date=2021-01-07&filters=#{filters}&with_imported=true"+ )++ res = json_response(conn, 200)++ assert %{"name" => "Scroll depth", "value" => 55, "graph_metric" => "scroll_depth"} in res[+ "top_stats"+ ]+ end+test "contains filter", %{conn: conn, site: site} dopopulate_stats(site, [build(:pageview, pathname: "/some-blog-post"),diff --git a/test/plausible_web/controllers/stats_controller_test.exs b/test/plausible_web/controllers/stats_controller_test.exsindex 586cb5b6b6..743927ffb7 100644--- a/test/plausible_web/controllers/stats_controller_test.exs+++ b/test/plausible_web/controllers/stats_controller_test.exs@@ -547,8 +547,15 @@ defmodule PlausibleWeb.StatsControllerTest do{~c"pages.csv", data} ->assert parse_csv(data) == [- ["name", "visitors", "pageviews", "bounce_rate", "time_on_page"],- ["/test", "1", "1", "0.0", "10.0"],+ [+ "name",+ "visitors",+ "pageviews",+ "bounce_rate",+ "time_on_page",+ "scroll_depth"+ ],+ ["/test", "1", "1", "0.0", "10.0", ""],[""]]