Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.
TypeScript analyticsanalytics-dashboardclickhouseelixirgoogle-analyticsmarketingopen-source-analyticsphoenixplausible-analyticspostgresqlprivacyprivacy-friendlyself-hostedsimple-analyticsstatisticstailwindcssweb-analyticswebsite-analyticswebsite-statswebsite-tracking
Scroll depth: Rename imported_pages columns to reflect whats desired (#5071)
Karl-Aksel Puulmann89a6b771326e
diff --git a/lib/plausible/exports.ex b/lib/plausible/exports.exindex f4a39635fd..6cb93f3739 100644--- a/lib/plausible/exports.ex+++ b/lib/plausible/exports.ex@@ -428,7 +428,8 @@ defmodule Plausible.Exports doorder_by: selected_as(:date))- if include_scroll_depth? do+ # :TODO: To be removed in the next PR+ if include_scroll_depth? and false domax_scroll_depth_per_visitor_q =from(e in "events_v2",where: ^export_filter(site_id, date_range),diff --git a/lib/plausible/imported/csv_importer.ex b/lib/plausible/imported/csv_importer.exindex 6a62d01c79..ff763e304b 100644--- a/lib/plausible/imported/csv_importer.ex+++ b/lib/plausible/imported/csv_importer.ex@@ -5,7 +5,6 @@ defmodule Plausible.Imported.CSVImporter do"""use Plausible.Imported.Importer- import Ecto.Query, only: [from: 2]@impl truedef name(), do: :csv@@ -55,26 +54,6 @@ defmodule Plausible.Imported.CSVImporter do{:error, Exception.message(e)}end- def on_success(site_import, _extra_data) do- has_scroll_depth? =- Plausible.ClickhouseRepo.exists?(- from(i in "imported_pages",- where: i.site_id == ^site_import.site_id,- where: i.import_id == ^site_import.id,- where: not is_nil(i.scroll_depth),- select: 1- )- )-- if has_scroll_depth? do- site_import- |> Ecto.Changeset.change(%{has_scroll_depth: true})- |> Plausible.Repo.update!()- end-- :ok- end-defp import_s3(ch, site_import, uploads) do%{id: import_id,@@ -182,7 +161,7 @@ defmodule Plausible.Imported.CSVImporter do"imported_operating_systems" =>"date Date, operating_system String, operating_system_version String, visitors UInt64, visits UInt64, visit_duration UInt64, bounces UInt32, pageviews UInt64","imported_pages" =>- "date Date, hostname String, page String, visits UInt64, visitors UInt64, pageviews UInt64, scroll_depth Nullable(UInt64), pageleave_visitors UInt64",+ "date Date, hostname String, page String, visits UInt64, visitors UInt64, pageviews UInt64","imported_sources" =>"date Date, source String, referrer String, utm_source String, utm_medium String, utm_campaign String, utm_content String, utm_term String, pageviews UInt64, visitors UInt64, visits UInt64, visit_duration UInt64, bounces UInt32","imported_visitors" =>diff --git a/lib/plausible/imported/page.ex b/lib/plausible/imported/page.exindex d4ba204a34..747c5e0306 100644--- a/lib/plausible/imported/page.ex+++ b/lib/plausible/imported/page.ex@@ -15,7 +15,7 @@ defmodule Plausible.Imported.Page dofield :pageviews, Ch, type: "UInt64"field :exits, Ch, type: "UInt64"field :time_on_page, Ch, type: "UInt64"- field :scroll_depth, Ch, type: "Nullable(UInt64)"- field :pageleave_visitors, Ch, type: "UInt64"+ # field :scroll_depth, Ch, type: "Nullable(UInt64)"+ # field :pageleave_visitors, Ch, type: "UInt64"endenddiff --git a/lib/plausible/stats/imported/sql/expression.ex b/lib/plausible/stats/imported/sql/expression.exindex d0fbfd4bf7..69b8dbf34b 100644--- a/lib/plausible/stats/imported/sql/expression.ex+++ b/lib/plausible/stats/imported/sql/expression.ex@@ -115,12 +115,12 @@ defmodule Plausible.Stats.Imported.SQL.Expression dowrap_alias([i], %{pageviews: sum(i.pageviews), __internal_visits: sum(i.visits)})end- defp select_metric(:scroll_depth, "imported_pages") do- wrap_alias([i], %{- scroll_depth_sum: sum(i.scroll_depth),- pageleave_visitors: sum(i.pageleave_visitors)- })- end+ # defp select_metric(:scroll_depth, "imported_pages") do+ # wrap_alias([i], %{+ # scroll_depth_sum: sum(i.scroll_depth),+ # pageleave_visitors: sum(i.pageleave_visitors)+ # })+ # enddefp select_metric(_metric, _table), do: %{}@@ -364,10 +364,6 @@ defmodule Plausible.Stats.Imported.SQL.Expression do# information from imported data here already.def select_joined_metrics(q, [:scroll_depth | rest]) doq- |> select_merge_as([s, i], %{- __internal_scroll_depth_sum: i.scroll_depth_sum,- __internal_pageleave_visitors: i.pageleave_visitors- })|> select_joined_metrics(rest)enddiff --git a/lib/plausible/stats/sql/special_metrics.ex b/lib/plausible/stats/sql/special_metrics.exindex 39d8a7ffa3..8fd32065f2 100644--- a/lib/plausible/stats/sql/special_metrics.ex+++ b/lib/plausible/stats/sql/special_metrics.ex@@ -180,27 +180,12 @@ defmodule Plausible.Stats.SQL.SpecialMetrics dofragment("""case- when isNotNull(?) AND isNotNull(?) then- toUInt8(round((? + ?) / (? + ?)))- when isNotNull(?) then- toUInt8(round(? / ?))when isNotNull(?) thentoUInt8(round(? / ?))elseNULLend""",- # Case 1: Both imported and native scroll depth sums are present- selected_as(:__internal_scroll_depth_sum),- s.scroll_depth_sum,- selected_as(:__internal_scroll_depth_sum),- s.scroll_depth_sum,- selected_as(:__internal_pageleave_visitors),- s.pageleave_visitors,- # Case 2: Only imported scroll depth sum is present- selected_as(:__internal_scroll_depth_sum),- selected_as(:__internal_scroll_depth_sum),- selected_as(:__internal_pageleave_visitors),# Case 3: Only native scroll depth sum is presents.scroll_depth_sum,s.scroll_depth_sum,diff --git a/priv/ingest_repo/migrations/20250212100953_imported_pages_new_scroll_depth_columns.exs b/priv/ingest_repo/migrations/20250212100953_imported_pages_new_scroll_depth_columns.exsnew file mode 100644index 0000000000..f5dae39a32--- /dev/null+++ b/priv/ingest_repo/migrations/20250212100953_imported_pages_new_scroll_depth_columns.exs@@ -0,0 +1,33 @@+defmodule Plausible.IngestRepo.Migrations.ImportedPagesNewScrollDepthColumns do+ use Ecto.Migration++ @on_cluster Plausible.MigrationUtils.on_cluster_statement("imported_pages")++ def up do+ execute """+ ALTER TABLE imported_pages #{@on_cluster}+ ADD COLUMN total_scroll_depth UInt64+ """++ execute """+ ALTER TABLE imported_pages #{@on_cluster}+ ADD COLUMN total_scroll_depth_visits UInt64+ """++ execute """+ ALTER TABLE imported_pages+ #{@on_cluster}+ DROP COLUMN scroll_depth+ """++ execute """+ ALTER TABLE imported_pages+ #{@on_cluster}+ DROP COLUMN pageleave_visitors+ """+ end++ def down do+ raise "irreversible"+ end+enddiff --git a/test/plausible/imported/csv_importer_test.exs b/test/plausible/imported/csv_importer_test.exsindex d6a6920e56..d508e3c40b 100644--- a/test/plausible/imported/csv_importer_test.exs+++ b/test/plausible/imported/csv_importer_test.exs@@ -363,6 +363,7 @@ defmodule Plausible.Imported.CSVImporterTest doassert Plausible.Stats.Clickhouse.imported_pageview_count(site) == 99end+ @tag skip: "To be re-enabled in the next PR"test "imports scroll_depth as null when the column does not exist in pages CSV",%{site: site, user: user} = ctx do_ = ctx@@ -423,9 +424,9 @@ defmodule Plausible.Imported.CSVImporterTest dostatus: :completed} = Repo.get_by!(SiteImport, site_id: site.id)- q = from(i in "imported_pages", where: i.site_id == ^site.id, select: i.scroll_depth)+ # q = from(i in "imported_pages", where: i.site_id == ^site.id, select: i.scroll_depth)- assert List.duplicate(nil, 16) == Plausible.IngestRepo.all(q)+ # assert List.duplicate(nil, 16) == Plausible.IngestRepo.all(q)endtest "accepts cells without quotes", %{site: site, user: user} = ctx do@@ -1032,6 +1033,7 @@ defmodule Plausible.Imported.CSVImporterTest doend@tag :tmp_dir+ @tag skip: "To be re-enabled in the next PR"test "scroll_depth", %{conn: conn, user: user, tmp_dir: tmp_dir} doexported_site = new_site(owner: user)imported_site = new_site(owner: user)@@ -1152,6 +1154,7 @@ defmodule Plausible.Imported.CSVImporterTest doend@tag :tmp_dir+ @tag skip: "To be re-enabled in the next PR"test "does not include scroll depth without existing engagement data", %{user: user,tmp_dir: tmp_dirdiff --git a/test/plausible_web/controllers/api/external_stats_controller/query_imported_test.exs b/test/plausible_web/controllers/api/external_stats_controller/query_imported_test.exsindex b15c3251dc..9557b9f35e 100644--- a/test/plausible_web/controllers/api/external_stats_controller/query_imported_test.exs+++ b/test/plausible_web/controllers/api/external_stats_controller/query_imported_test.exs@@ -1524,6 +1524,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryImportedTest dorefute meta["metric_warnings"]["scroll_depth"]end+ @tag skip: "To be re-enabled in the next PR"test "does not return warning when imported scroll depth exists", %{conn: conn,site: sitediff --git a/test/plausible_web/controllers/api/stats_controller/main_graph_test.exs b/test/plausible_web/controllers/api/stats_controller/main_graph_test.exsindex 582da9990a..fbdfaccdd5 100644--- a/test/plausible_web/controllers/api/stats_controller/main_graph_test.exs+++ b/test/plausible_web/controllers/api/stats_controller/main_graph_test.exs@@ -634,6 +634,7 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest doassert plot == [40, 20, 0, 0, 0, 0, 0]end+ @tag skip: "To be re-enabled in the next PR"test "returns scroll depth per day with imported data", %{conn: conn, site: site} dosite_import = insert(:site_import, site: site)diff --git a/test/plausible_web/controllers/api/stats_controller/pages_test.exs b/test/plausible_web/controllers/api/stats_controller/pages_test.exsindex 05b9d9b21a..d4d363b0d3 100644--- a/test/plausible_web/controllers/api/stats_controller/pages_test.exs+++ b/test/plausible_web/controllers/api/stats_controller/pages_test.exs@@ -676,6 +676,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do}end+ @tag skip: "To be re-enabled in the next PR"test "calculates scroll_depth from native and imported data combined", %{conn: conn,site: site@@ -717,6 +718,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do]end+ @tag skip: "To be re-enabled in the next PR"test "handles missing scroll_depth data from native and imported sources", %{conn: conn,site: site@@ -798,6 +800,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do]end+ @tag skip: "To be re-enabled in the next PR"test "can query scroll depth only from imported data, ignoring rows where scroll depth doesn't exist",%{conn: conn,@@ -1137,6 +1140,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do]end+ @tag skip: "To be re-enabled in the next PR"test "returns scroll depth warning code", %{conn: conn, site: site} doPlausible.Sites.set_scroll_depth_visible_at(site)@@ -1149,6 +1153,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do"no_imported_scroll_depth"end+ @tag skip: "To be re-enabled in the next PR"test "returns imported pages with a pageview goal filter", %{conn: conn, site: site} doinsert(:goal, site: site, page_path: "/blog**")diff --git a/test/plausible_web/controllers/api/stats_controller/top_stats_test.exs b/test/plausible_web/controllers/api/stats_controller/top_stats_test.exsindex 43c3c97c3c..d0fcfd9134 100644--- a/test/plausible_web/controllers/api/stats_controller/top_stats_test.exs+++ b/test/plausible_web/controllers/api/stats_controller/top_stats_test.exs@@ -1018,6 +1018,7 @@ defmodule PlausibleWeb.Api.StatsController.TopStatsTest do]end+ @tag skip: "To be re-enabled in the next PR"test "returns scroll_depth with a page filter with imported data", %{conn: conn, site: site} dosite_import =insert(:site_import, site: site, start_date: ~D[2021-01-01], has_scroll_depth: true)