Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.
TypeScript analyticsanalytics-dashboardclickhouseelixirgoogle-analyticsmarketingopen-source-analyticsphoenixplausible-analyticspostgresqlprivacyprivacy-friendlyself-hostedsimple-analyticsstatisticstailwindcssweb-analyticswebsite-analyticswebsite-statswebsite-tracking
Revert "Scroll depth: update `site.scroll_depth_visible_at` in ingestion (#5035)" (#5042)
Cenk Kücükb4ec09c6cb6b
diff --git a/lib/plausible/application.ex b/lib/plausible/application.exindex 98e2cfec58..1e0e6a23c5 100644--- a/lib/plausible/application.ex+++ b/lib/plausible/application.ex@@ -120,7 +120,6 @@ defmodule Plausible.Application dosetup_geolocation()Location.load_all()Plausible.Ingestion.Source.init()- Plausible.Ingestion.ScrollDepthVisibleAt.init()Plausible.Geo.await_loader()Supervisor.start_link(List.flatten(children), opts)diff --git a/lib/plausible/exports.ex b/lib/plausible/exports.exindex 70f15b2b8f..f082e5f5f2 100644--- a/lib/plausible/exports.ex+++ b/lib/plausible/exports.ex@@ -418,6 +418,8 @@ defmodule Plausible.Exports dosite = Plausible.Repo.get(Plausible.Site, site_id)current_user = current_user_id && Plausible.Repo.get(Plausible.Auth.User, current_user_id)+ include_scroll_depth? = Plausible.Stats.ScrollDepth.check_feature_visible!(site, current_user)+base_q =from(e in sampled("events_v2"),where: ^export_filter(site_id, date_range),@@ -426,7 +428,7 @@ defmodule Plausible.Exports doorder_by: selected_as(:date))- if Plausible.Stats.ScrollDepth.feature_visible?(site, current_user) do+ if include_scroll_depth? domax_scroll_depth_per_visitor_q =from(e in "events_v2",where: ^export_filter(site_id, date_range),diff --git a/lib/plausible/ingestion/event.ex b/lib/plausible/ingestion/event.exindex 6b28d130c8..e113ae8047 100644--- a/lib/plausible/ingestion/event.ex+++ b/lib/plausible/ingestion/event.ex@@ -130,8 +130,7 @@ defmodule Plausible.Ingestion.Event doput_user_id: &put_user_id/2,validate_clickhouse_event: &validate_clickhouse_event/2,register_session: ®ister_session/2,- write_to_buffer: &write_to_buffer/2,- register_scroll_depth_visible: ®ister_scroll_depth_visible/2+ write_to_buffer: &write_to_buffer/2]end@@ -406,15 +405,6 @@ defmodule Plausible.Ingestion.Event doeventend- defp register_scroll_depth_visible(%__MODULE__{clickhouse_event: ev} = event, _context) do- if is_nil(event.site.scroll_depth_visible_at) and ev.name in ["pageleave", "engagement"] and- not is_nil(ev.scroll_depth) and ev.scroll_depth >= 0 and ev.scroll_depth <= 100 do- Plausible.Ingestion.ScrollDepthVisibleAt.mark_scroll_depth_visible(event.site.id)- end-- event- end-@click_id_params ["gclid", "gbraid", "wbraid", "msclkid", "fbclid", "twclid"]defp get_click_id_param(nil), do: nildiff --git a/lib/plausible/ingestion/scroll_depth_visible_at.ex b/lib/plausible/ingestion/scroll_depth_visible_at.exdeleted file mode 100644index 215c9e880c..0000000000--- a/lib/plausible/ingestion/scroll_depth_visible_at.ex+++ /dev/null@@ -1,44 +0,0 @@-defmodule Plausible.Ingestion.ScrollDepthVisibleAt do- @moduledoc """- Module that updates the `scroll_depth_visible_at` column for a site when needed.-- This is called in a hot loop in ingestion, so it:- 1. Only updates the database once per site async (if SiteCache doesn't know about visibility yet)- 2. Does not retry the update if it fails, to be retried on server restart- """-- require Logger- alias Plausible.Repo-- import Ecto.Query-- def init() do- :ets.new(__MODULE__, [- :named_table,- :set,- :public,- {:read_concurrency, true},- {:write_concurrency, true}- ])- end-- def mark_scroll_depth_visible(site_id) do- if :ets.insert_new(__MODULE__, {site_id}) do- Task.start(fn -> attempt_update_repo(site_id) end)- end- end-- defp attempt_update_repo(site_id) do- Repo.update_all(- from(s in Plausible.Site, where: s.id == ^site_id and is_nil(s.scroll_depth_visible_at)),- set: [- scroll_depth_visible_at: DateTime.utc_now()- ]- )- rescue- error ->- Logger.error(- "Error updating scroll_depth_visible_at for site #{site_id}: #{inspect(error)}. This will not be retried until server restart."- )- end-enddiff --git a/lib/plausible/site/cache.ex b/lib/plausible/site/cache.exindex 8cfba51967..1514bc006a 100644--- a/lib/plausible/site/cache.ex+++ b/lib/plausible/site/cache.ex@@ -31,7 +31,6 @@ defmodule Plausible.Site.Cache dodomain_changed_fromingest_rate_limit_scale_secondsingest_rate_limit_threshold- scroll_depth_visible_at)a@impl truediff --git a/lib/plausible/stats/scroll_depth.ex b/lib/plausible/stats/scroll_depth.exindex 4ebd80fe7c..246c8a8527 100644--- a/lib/plausible/stats/scroll_depth.ex+++ b/lib/plausible/stats/scroll_depth.ex@@ -3,6 +3,11 @@ defmodule Plausible.Stats.ScrollDepth doModule to check whether the scroll depth metric is available and visible for a site."""+ import Ecto.Query+ require Logger++ alias Plausible.ClickhouseRepo+def feature_available?(site, user) doFunWithFlags.enabled?(:scroll_depth, for: user) ||FunWithFlags.enabled?(:scroll_depth, for: site)@@ -11,4 +16,48 @@ defmodule Plausible.Stats.ScrollDepth dodef feature_visible?(site, user) dofeature_available?(site, user) && not is_nil(site.scroll_depth_visible_at)end++ @doc """+ Checks whether the scroll depth feature is visible for a site and updates the site record if it is.++ Note this function queries ClickHouse and may take a while to complete.+ """+ def check_feature_visible!(site, user) do+ cond do+ not feature_available?(site, user) ->+ false++ not is_nil(site.scroll_depth_visible_at) ->+ true++ is_nil(site.scroll_depth_visible_at) ->+ visible? = has_scroll_data_last_30d?(site)++ if visible? do+ Plausible.Sites.set_scroll_depth_visible_at(site)+ end++ visible?+ end+ end++ defp has_scroll_data_last_30d?(site) do+ try do+ ClickhouseRepo.exists?(+ from(e in "events_v2",+ where:+ e.site_id == ^site.id and+ e.name == "pageleave" and+ e.timestamp >= fragment("toStartOfDay(now() - toIntervalDay(30))") and+ e.scroll_depth > 0 and e.scroll_depth <= 100+ )+ )+ rescue+ # Avoid propagating error to the user, bringing down the site.+ error ->+ Logger.error("Error checking scroll data for site #{site.id}: #{inspect(error)}")++ false+ end+ endenddiff --git a/lib/plausible_web/controllers/stats_controller.ex b/lib/plausible_web/controllers/stats_controller.exindex 4df58abbf2..8d5dc4b8cb 100644--- a/lib/plausible_web/controllers/stats_controller.ex+++ b/lib/plausible_web/controllers/stats_controller.ex@@ -45,7 +45,7 @@ defmodule PlausibleWeb.StatsController douse Plausible.Repoalias Plausible.Sites- alias Plausible.Stats.{Filters, Query, ScrollDepth}+ alias Plausible.Stats.{Filters, Query}alias PlausibleWeb.Apiplug(PlausibleWeb.Plugs.AuthorizeSiteAccess when action in [:stats, :csv_export])@@ -59,6 +59,9 @@ defmodule PlausibleWeb.StatsController dodogfood_page_path = if demo, do: "/#{site.domain}", else: "/:dashboard"skip_to_dashboard? = conn.params["skip_to_dashboard"] == "true"+ scroll_depth_visible? =+ Plausible.Stats.ScrollDepth.check_feature_visible!(site, current_user)+cond do(stats_start_date && can_see_stats?) || (can_see_stats? && skip_to_dashboard?) ->conn@@ -69,7 +72,7 @@ defmodule PlausibleWeb.StatsController dorevenue_goals: list_revenue_goals(site),funnels: list_funnels(site),has_props: Plausible.Props.configured?(site),- scroll_depth_visible: ScrollDepth.feature_visible?(site, current_user),+ scroll_depth_visible: scroll_depth_visible?,stats_start_date: stats_start_date,native_stats_start_date: NaiveDateTime.to_date(site.native_stats_start_at),title: title(conn, site),@@ -347,6 +350,9 @@ defmodule PlausibleWeb.StatsController doshared_link = Plausible.Repo.preload(shared_link, site: :owner)stats_start_date = Plausible.Sites.stats_start_date(shared_link.site)+ scroll_depth_visible? =+ Plausible.Stats.ScrollDepth.check_feature_visible!(shared_link.site, current_user)+conn|> put_resp_header("x-robots-tag", "noindex, nofollow")|> delete_resp_header("x-frame-options")@@ -356,7 +362,7 @@ defmodule PlausibleWeb.StatsController dorevenue_goals: list_revenue_goals(shared_link.site),funnels: list_funnels(shared_link.site),has_props: Plausible.Props.configured?(shared_link.site),- scroll_depth_visible: ScrollDepth.feature_visible?(shared_link.site, current_user),+ scroll_depth_visible: scroll_depth_visible?,stats_start_date: stats_start_date,native_stats_start_date: NaiveDateTime.to_date(shared_link.site.native_stats_start_at),title: title(conn, shared_link.site),diff --git a/test/plausible/imported/csv_importer_test.exs b/test/plausible/imported/csv_importer_test.exsindex 41745a8f59..a205355be5 100644--- a/test/plausible/imported/csv_importer_test.exs+++ b/test/plausible/imported/csv_importer_test.exs@@ -1065,7 +1065,6 @@ defmodule Plausible.Imported.CSVImporterTest do|> Enum.map(fn event -> Map.put(event, :hostname, "csv.test") end)populate_stats(exported_site, stats)- Plausible.Sites.set_scroll_depth_visible_at(exported_site)initial_context = %{user: user,@@ -1082,6 +1081,9 @@ defmodule Plausible.Imported.CSVImporterTest do|> upload_csvs()|> run_import()+ assert %NaiveDateTime{} =+ Plausible.Repo.reload!(exported_site).scroll_depth_visible_at+assert %SiteImport{start_date: start_date,end_date: end_date,diff --git a/test/plausible/ingestion/scroll_depth_visible_at_test.exs b/test/plausible/ingestion/scroll_depth_visible_at_test.exsdeleted file mode 100644index 8eb23c7ef6..0000000000--- a/test/plausible/ingestion/scroll_depth_visible_at_test.exs+++ /dev/null@@ -1,15 +0,0 @@-defmodule Plausible.Ingestion.ScrollDepthVisibleAtTest do- use Plausible.DataCase- use Plausible.Teams.Test-- test "mark_scroll_depth_visible" do- site = new_site()- Plausible.Ingestion.ScrollDepthVisibleAt.mark_scroll_depth_visible(site.id)-- assert Plausible.TestUtils.eventually(fn ->- site = Plausible.Repo.reload!(site)- success = not is_nil(site.scroll_depth_visible_at)- {success, success}- end)- end-enddiff --git a/test/plausible_web/controllers/api/external_controller_test.exs b/test/plausible_web/controllers/api/external_controller_test.exsindex 75ecca940d..c4d0e52b23 100644--- a/test/plausible_web/controllers/api/external_controller_test.exs+++ b/test/plausible_web/controllers/api/external_controller_test.exs@@ -3,8 +3,6 @@ defmodule PlausibleWeb.Api.ExternalControllerTest douse Plausible.ClickhouseRepouse Plausible.Teams.Test- alias Plausible.Repo-@user_agent "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.110 Safari/537.36"@user_agent_mobile "Mozilla/5.0 (Linux; Android 6.0; U007 Pro Build/MRA58K; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/44.0.2403.119 Mobile Safari/537.36"@user_agent_tablet "Mozilla/5.0 (Linux; U; Android 4.2.2; it-it; Surfing TAB B 9.7 3G Build/JDQ39) AppleWebKit/534.30 (KHTML, like Gecko) Version/4.0 Mobile Safari/534.30"@@ -1293,7 +1291,6 @@ defmodule PlausibleWeb.Api.ExternalControllerTest dopageleave = get_events(site) |> Enum.find(&(&1.name == "pageleave"))assert pageleave.scroll_depth == 255- assert not scroll_depth_visible_at_set?(site)endtest "sd field is ignored if name is not pageleave", %{conn: conn, site: site} do@@ -1301,7 +1298,6 @@ defmodule PlausibleWeb.Api.ExternalControllerTest dopost(conn, "/api/event", %{n: "custom_e", u: "https://test.com", d: site.domain, sd: 10})assert [%{scroll_depth: 0}, %{scroll_depth: 0}] = get_events(site)- assert not scroll_depth_visible_at_set?(site)endtest "ingests valid scroll_depth for a pageleave", %{conn: conn, site: site} do@@ -1311,7 +1307,6 @@ defmodule PlausibleWeb.Api.ExternalControllerTest dopageleave = get_events(site) |> Enum.find(&(&1.name == "pageleave"))assert pageleave.scroll_depth == 25- assert scroll_depth_visible_at_set?(site)endtest "ingests scroll_depth as 100 when sd > 100", %{conn: conn, site: site} do@@ -1321,7 +1316,6 @@ defmodule PlausibleWeb.Api.ExternalControllerTest dopageleave = get_events(site) |> Enum.find(&(&1.name == "pageleave"))assert pageleave.scroll_depth == 100- assert scroll_depth_visible_at_set?(site)endtest "ingests scroll_depth as 255 when sd is a string", %{conn: conn, site: site} do@@ -1331,7 +1325,6 @@ defmodule PlausibleWeb.Api.ExternalControllerTest dopageleave = get_events(site) |> Enum.find(&(&1.name == "pageleave"))assert pageleave.scroll_depth == 255- assert not scroll_depth_visible_at_set?(site)endtest "ingests scroll_depth as 255 when sd is a negative integer", %{conn: conn, site: site} do@@ -1341,7 +1334,6 @@ defmodule PlausibleWeb.Api.ExternalControllerTest dopageleave = get_events(site) |> Enum.find(&(&1.name == "pageleave"))assert pageleave.scroll_depth == 255- assert not scroll_depth_visible_at_set?(site)endtest "ingests valid scroll_depth for a engagement event", %{conn: conn, site: site} do@@ -1351,23 +1343,6 @@ defmodule PlausibleWeb.Api.ExternalControllerTest doevent = get_events(site) |> Enum.find(&(&1.name == "engagement"))assert event.scroll_depth == 25- assert scroll_depth_visible_at_set?(site)- end-- test "does not update scroll_depth_visible_at twice", %{conn: conn, site: site} do- post(conn, "/api/event", %{n: "pageview", u: "https://test.com", d: site.domain})- post(conn, "/api/event", %{n: "pageleave", u: "https://test.com", d: site.domain, sd: 25})- Plausible.Event.WriteBuffer.flush()-- site1 = Repo.reload!(site)- assert not is_nil(site1.scroll_depth_visible_at)-- post(conn, "/api/event", %{n: "pageleave", u: "https://test.com", d: site.domain, sd: 50})- Plausible.Event.WriteBuffer.flush()-- site2 = Repo.reload!(site)- assert not is_nil(site2.scroll_depth_visible_at)- assert site1.scroll_depth_visible_at == site2.scroll_depth_visible_atendend@@ -3609,9 +3584,4 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do))end-- defp scroll_depth_visible_at_set?(site) do- site = Repo.reload!(site)- not is_nil(site.scroll_depth_visible_at)- endenddiff --git a/test/plausible_web/controllers/stats_controller_test.exs b/test/plausible_web/controllers/stats_controller_test.exsindex 999c0d3818..5fd9f620a6 100644--- a/test/plausible_web/controllers/stats_controller_test.exs+++ b/test/plausible_web/controllers/stats_controller_test.exs@@ -85,6 +85,7 @@ defmodule PlausibleWeb.StatsControllerTest dopopulate_stats(site, [build(:pageview)])+ # No pageleaves yet - `scroll_depth_visible_at` will remain `nil`html =conn|> get("/#{site.domain}")@@ -92,8 +93,15 @@ defmodule PlausibleWeb.StatsControllerTest doassert text_of_attr(html, @react_container, "data-scroll-depth-visible") == "false"- Plausible.Sites.set_scroll_depth_visible_at(site)+ site = Repo.reload!(site)+ assert is_nil(site.scroll_depth_visible_at)+ populate_stats(site, [+ build(:pageview, user_id: 123),+ build(:pageleave, user_id: 123, scroll_depth: 20)+ ])++ # Pageleaves exist now - `scroll_depth_visible_at` gets set to `utc_now`html =conn|> get("/#{site.domain}")@@ -1090,6 +1098,7 @@ defmodule PlausibleWeb.StatsControllerTest dosite = insert(:site)link = insert(:shared_link, site: site)+ # No pageleaves yet - `scroll_depth_visible_at` will remain `nil`html =conn|> get("/share/#{site.domain}/?auth=#{link.slug}")@@ -1097,8 +1106,15 @@ defmodule PlausibleWeb.StatsControllerTest doassert text_of_attr(html, @react_container, "data-scroll-depth-visible") == "false"- Plausible.Sites.set_scroll_depth_visible_at(site)+ site = Repo.reload!(site)+ assert is_nil(site.scroll_depth_visible_at)++ populate_stats(site, [+ build(:pageview, user_id: 123),+ build(:pageleave, user_id: 123, scroll_depth: 20)+ ])+ # Pageleaves exist now - `scroll_depth_visible_at` gets set to `utc_now`html =conn|> get("/share/#{site.domain}/?auth=#{link.slug}")