Skip to content

Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.

Scroll depth: Rename imported_pages columns to reflect whats desired (#5071)

Karl-Aksel Puulmann89a6b771326e
diff --git a/lib/plausible/exports.ex b/lib/plausible/exports.ex
index f4a39635fd..6cb93f3739 100644
--- a/lib/plausible/exports.ex
+++ b/lib/plausible/exports.ex
@@ -428,7 +428,8 @@ defmodule Plausible.Exports do
order_by: selected_as(:date)
)
- if include_scroll_depth? do
+ # :TODO: To be removed in the next PR
+ if include_scroll_depth? and false do
max_scroll_depth_per_visitor_q =
from(e in "events_v2",
where: ^export_filter(site_id, date_range),
diff --git a/lib/plausible/imported/csv_importer.ex b/lib/plausible/imported/csv_importer.ex
index 6a62d01c79..ff763e304b 100644
--- a/lib/plausible/imported/csv_importer.ex
+++ b/lib/plausible/imported/csv_importer.ex
@@ -5,7 +5,6 @@ defmodule Plausible.Imported.CSVImporter do
"""
use Plausible.Imported.Importer
- import Ecto.Query, only: [from: 2]
@impl true
def name(), do: :csv
@@ -55,26 +54,6 @@ defmodule Plausible.Imported.CSVImporter do
{:error, Exception.message(e)}
end
- def on_success(site_import, _extra_data) do
- has_scroll_depth? =
- Plausible.ClickhouseRepo.exists?(
- from(i in "imported_pages",
- where: i.site_id == ^site_import.site_id,
- where: i.import_id == ^site_import.id,
- where: not is_nil(i.scroll_depth),
- select: 1
- )
- )
-
- if has_scroll_depth? do
- site_import
- |> Ecto.Changeset.change(%{has_scroll_depth: true})
- |> Plausible.Repo.update!()
- end
-
- :ok
- end
-
defp import_s3(ch, site_import, uploads) do
%{
id: import_id,
@@ -182,7 +161,7 @@ defmodule Plausible.Imported.CSVImporter do
"imported_operating_systems" =>
"date Date, operating_system String, operating_system_version String, visitors UInt64, visits UInt64, visit_duration UInt64, bounces UInt32, pageviews UInt64",
"imported_pages" =>
- "date Date, hostname String, page String, visits UInt64, visitors UInt64, pageviews UInt64, scroll_depth Nullable(UInt64), pageleave_visitors UInt64",
+ "date Date, hostname String, page String, visits UInt64, visitors UInt64, pageviews UInt64",
"imported_sources" =>
"date Date, source String, referrer String, utm_source String, utm_medium String, utm_campaign String, utm_content String, utm_term String, pageviews UInt64, visitors UInt64, visits UInt64, visit_duration UInt64, bounces UInt32",
"imported_visitors" =>
diff --git a/lib/plausible/imported/page.ex b/lib/plausible/imported/page.ex
index d4ba204a34..747c5e0306 100644
--- a/lib/plausible/imported/page.ex
+++ b/lib/plausible/imported/page.ex
@@ -15,7 +15,7 @@ defmodule Plausible.Imported.Page do
field :pageviews, Ch, type: "UInt64"
field :exits, Ch, type: "UInt64"
field :time_on_page, Ch, type: "UInt64"
- field :scroll_depth, Ch, type: "Nullable(UInt64)"
- field :pageleave_visitors, Ch, type: "UInt64"
+ # field :scroll_depth, Ch, type: "Nullable(UInt64)"
+ # field :pageleave_visitors, Ch, type: "UInt64"
end
end
diff --git a/lib/plausible/stats/imported/sql/expression.ex b/lib/plausible/stats/imported/sql/expression.ex
index d0fbfd4bf7..69b8dbf34b 100644
--- a/lib/plausible/stats/imported/sql/expression.ex
+++ b/lib/plausible/stats/imported/sql/expression.ex
@@ -115,12 +115,12 @@ defmodule Plausible.Stats.Imported.SQL.Expression do
wrap_alias([i], %{pageviews: sum(i.pageviews), __internal_visits: sum(i.visits)})
end
- defp select_metric(:scroll_depth, "imported_pages") do
- wrap_alias([i], %{
- scroll_depth_sum: sum(i.scroll_depth),
- pageleave_visitors: sum(i.pageleave_visitors)
- })
- end
+ # defp select_metric(:scroll_depth, "imported_pages") do
+ # wrap_alias([i], %{
+ # scroll_depth_sum: sum(i.scroll_depth),
+ # pageleave_visitors: sum(i.pageleave_visitors)
+ # })
+ # end
defp select_metric(_metric, _table), do: %{}
@@ -364,10 +364,6 @@ defmodule Plausible.Stats.Imported.SQL.Expression do
# information from imported data here already.
def select_joined_metrics(q, [:scroll_depth | rest]) do
q
- |> select_merge_as([s, i], %{
- __internal_scroll_depth_sum: i.scroll_depth_sum,
- __internal_pageleave_visitors: i.pageleave_visitors
- })
|> select_joined_metrics(rest)
end
diff --git a/lib/plausible/stats/sql/special_metrics.ex b/lib/plausible/stats/sql/special_metrics.ex
index 39d8a7ffa3..8fd32065f2 100644
--- a/lib/plausible/stats/sql/special_metrics.ex
+++ b/lib/plausible/stats/sql/special_metrics.ex
@@ -180,27 +180,12 @@ defmodule Plausible.Stats.SQL.SpecialMetrics do
fragment(
"""
case
- when isNotNull(?) AND isNotNull(?) then
- toUInt8(round((? + ?) / (? + ?)))
- when isNotNull(?) then
- toUInt8(round(? / ?))
when isNotNull(?) then
toUInt8(round(? / ?))
else
NULL
end
""",
- # Case 1: Both imported and native scroll depth sums are present
- selected_as(:__internal_scroll_depth_sum),
- s.scroll_depth_sum,
- selected_as(:__internal_scroll_depth_sum),
- s.scroll_depth_sum,
- selected_as(:__internal_pageleave_visitors),
- s.pageleave_visitors,
- # Case 2: Only imported scroll depth sum is present
- selected_as(:__internal_scroll_depth_sum),
- selected_as(:__internal_scroll_depth_sum),
- selected_as(:__internal_pageleave_visitors),
# Case 3: Only native scroll depth sum is present
s.scroll_depth_sum,
s.scroll_depth_sum,
diff --git a/priv/ingest_repo/migrations/20250212100953_imported_pages_new_scroll_depth_columns.exs b/priv/ingest_repo/migrations/20250212100953_imported_pages_new_scroll_depth_columns.exs
new file mode 100644
index 0000000000..f5dae39a32
--- /dev/null
+++ b/priv/ingest_repo/migrations/20250212100953_imported_pages_new_scroll_depth_columns.exs
@@ -0,0 +1,33 @@
+defmodule Plausible.IngestRepo.Migrations.ImportedPagesNewScrollDepthColumns do
+ use Ecto.Migration
+
+ @on_cluster Plausible.MigrationUtils.on_cluster_statement("imported_pages")
+
+ def up do
+ execute """
+ ALTER TABLE imported_pages #{@on_cluster}
+ ADD COLUMN total_scroll_depth UInt64
+ """
+
+ execute """
+ ALTER TABLE imported_pages #{@on_cluster}
+ ADD COLUMN total_scroll_depth_visits UInt64
+ """
+
+ execute """
+ ALTER TABLE imported_pages
+ #{@on_cluster}
+ DROP COLUMN scroll_depth
+ """
+
+ execute """
+ ALTER TABLE imported_pages
+ #{@on_cluster}
+ DROP COLUMN pageleave_visitors
+ """
+ end
+
+ def down do
+ raise "irreversible"
+ end
+end
diff --git a/test/plausible/imported/csv_importer_test.exs b/test/plausible/imported/csv_importer_test.exs
index d6a6920e56..d508e3c40b 100644
--- a/test/plausible/imported/csv_importer_test.exs
+++ b/test/plausible/imported/csv_importer_test.exs
@@ -363,6 +363,7 @@ defmodule Plausible.Imported.CSVImporterTest do
assert Plausible.Stats.Clickhouse.imported_pageview_count(site) == 99
end
+ @tag skip: "To be re-enabled in the next PR"
test "imports scroll_depth as null when the column does not exist in pages CSV",
%{site: site, user: user} = ctx do
_ = ctx
@@ -423,9 +424,9 @@ defmodule Plausible.Imported.CSVImporterTest do
status: :completed
} = Repo.get_by!(SiteImport, site_id: site.id)
- q = from(i in "imported_pages", where: i.site_id == ^site.id, select: i.scroll_depth)
+ # q = from(i in "imported_pages", where: i.site_id == ^site.id, select: i.scroll_depth)
- assert List.duplicate(nil, 16) == Plausible.IngestRepo.all(q)
+ # assert List.duplicate(nil, 16) == Plausible.IngestRepo.all(q)
end
test "accepts cells without quotes", %{site: site, user: user} = ctx do
@@ -1032,6 +1033,7 @@ defmodule Plausible.Imported.CSVImporterTest do
end
@tag :tmp_dir
+ @tag skip: "To be re-enabled in the next PR"
test "scroll_depth", %{conn: conn, user: user, tmp_dir: tmp_dir} do
exported_site = new_site(owner: user)
imported_site = new_site(owner: user)
@@ -1152,6 +1154,7 @@ defmodule Plausible.Imported.CSVImporterTest do
end
@tag :tmp_dir
+ @tag skip: "To be re-enabled in the next PR"
test "does not include scroll depth without existing engagement data", %{
user: user,
tmp_dir: tmp_dir
diff --git a/test/plausible_web/controllers/api/external_stats_controller/query_imported_test.exs b/test/plausible_web/controllers/api/external_stats_controller/query_imported_test.exs
index b15c3251dc..9557b9f35e 100644
--- a/test/plausible_web/controllers/api/external_stats_controller/query_imported_test.exs
+++ b/test/plausible_web/controllers/api/external_stats_controller/query_imported_test.exs
@@ -1524,6 +1524,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryImportedTest do
refute meta["metric_warnings"]["scroll_depth"]
end
+ @tag skip: "To be re-enabled in the next PR"
test "does not return warning when imported scroll depth exists", %{
conn: conn,
site: site
diff --git a/test/plausible_web/controllers/api/stats_controller/main_graph_test.exs b/test/plausible_web/controllers/api/stats_controller/main_graph_test.exs
index 582da9990a..fbdfaccdd5 100644
--- a/test/plausible_web/controllers/api/stats_controller/main_graph_test.exs
+++ b/test/plausible_web/controllers/api/stats_controller/main_graph_test.exs
@@ -634,6 +634,7 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do
assert plot == [40, 20, 0, 0, 0, 0, 0]
end
+ @tag skip: "To be re-enabled in the next PR"
test "returns scroll depth per day with imported data", %{conn: conn, site: site} do
site_import = insert(:site_import, site: site)
diff --git a/test/plausible_web/controllers/api/stats_controller/pages_test.exs b/test/plausible_web/controllers/api/stats_controller/pages_test.exs
index 05b9d9b21a..d4d363b0d3 100644
--- a/test/plausible_web/controllers/api/stats_controller/pages_test.exs
+++ b/test/plausible_web/controllers/api/stats_controller/pages_test.exs
@@ -676,6 +676,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
}
end
+ @tag skip: "To be re-enabled in the next PR"
test "calculates scroll_depth from native and imported data combined", %{
conn: conn,
site: site
@@ -717,6 +718,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
]
end
+ @tag skip: "To be re-enabled in the next PR"
test "handles missing scroll_depth data from native and imported sources", %{
conn: conn,
site: site
@@ -798,6 +800,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
]
end
+ @tag skip: "To be re-enabled in the next PR"
test "can query scroll depth only from imported data, ignoring rows where scroll depth doesn't exist",
%{
conn: conn,
@@ -1137,6 +1140,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
]
end
+ @tag skip: "To be re-enabled in the next PR"
test "returns scroll depth warning code", %{conn: conn, site: site} do
Plausible.Sites.set_scroll_depth_visible_at(site)
@@ -1149,6 +1153,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
"no_imported_scroll_depth"
end
+ @tag skip: "To be re-enabled in the next PR"
test "returns imported pages with a pageview goal filter", %{conn: conn, site: site} do
insert(:goal, site: site, page_path: "/blog**")
diff --git a/test/plausible_web/controllers/api/stats_controller/top_stats_test.exs b/test/plausible_web/controllers/api/stats_controller/top_stats_test.exs
index 43c3c97c3c..d0fcfd9134 100644
--- a/test/plausible_web/controllers/api/stats_controller/top_stats_test.exs
+++ b/test/plausible_web/controllers/api/stats_controller/top_stats_test.exs
@@ -1018,6 +1018,7 @@ defmodule PlausibleWeb.Api.StatsController.TopStatsTest do
]
end
+ @tag skip: "To be re-enabled in the next PR"
test "returns scroll_depth with a page filter with imported data", %{conn: conn, site: site} do
site_import =
insert(:site_import, site: site, start_date: ~D[2021-01-01], has_scroll_depth: true)