Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.
TypeScript analyticsanalytics-dashboardclickhouseelixirgoogle-analyticsmarketingopen-source-analyticsphoenixplausible-analyticspostgresqlprivacyprivacy-friendlyself-hostedsimple-analyticsstatisticstailwindcssweb-analyticswebsite-analyticswebsite-statswebsite-tracking
Reduce site deletions impact on ClickHouse (#6591)
Adam8d54c4d57794
diff --git a/config/runtime.exs b/config/runtime.exsindex abc25e0f31..315f27de92 100644--- a/config/runtime.exs+++ b/config/runtime.exs@@ -705,12 +705,12 @@ config :plausible, Plausible.AsyncInsertRepo,materialized_views_ignore_errors: 1]-config :plausible, Plausible.ImportDeletionRepo,+config :plausible, Plausible.DeletionRepo,queue_target: 500,queue_interval: 2000,url: ch_db_url,transport_opts: ch_transport_opts,- pool_size: 1+ pool_size: 2config :plausible, Plausible.Ingestion.Persistor,backend: persistor_backend,@@ -848,10 +848,10 @@ cloud_cron = [{"0 0 * * *", Plausible.Workers.LockSites},# Daily at 8{"0 8 * * *", Plausible.Workers.AcceptTrafficUntil},- # First sunday of the month, 4:00 UTC- {"0 4 1-7 * SUN", Plausible.Workers.ClickhouseCleanSites},- # Daily at 4:00 UTC- {"0 4 * * *", Plausible.Workers.SetLegacyTimeOnPageCutoff},+ # Every Tuesday, 3:00 UTC+ {"0 3 * * TUE", Plausible.Workers.ClickhouseCleanSites},+ # Daily at 5:00 UTC+ {"0 5 * * *", Plausible.Workers.SetLegacyTimeOnPageCutoff},# Daily at 2:00 UTC{"0 2 * * *", Plausible.Workers.ScoreTrialProspects}]diff --git a/lib/plausible/application.ex b/lib/plausible/application.exindex 265341480d..9cf02cd280 100644--- a/lib/plausible/application.ex+++ b/lib/plausible/application.ex@@ -44,7 +44,7 @@ defmodule Plausible.Application doPlausible.ClickhouseRepo,Plausible.IngestRepo,Plausible.AsyncInsertRepo,- Plausible.ImportDeletionRepo,+ Plausible.DeletionRepo,Plausible.Cache.Adapter.child_spec(:customer_currency, :cache_customer_currency,ttl_check_interval: :timer.minutes(5),n_lock_partitions: 1,diff --git a/lib/plausible/cache.ex b/lib/plausible/cache.exindex 496c02784f..2bbefbcc93 100644--- a/lib/plausible/cache.ex+++ b/lib/plausible/cache.ex@@ -83,6 +83,19 @@ defmodule Plausible.Cache do:okend+ @spec broadcast_delete(any(), Keyword.t()) :: :ok+ def broadcast_delete(key, opts \\ []) do+ cache_name = Keyword.get(opts, :cache_name, name())+ multicall_timeout = Keyword.get(opts, :multicall_timeout, :timer.seconds(5))++ {:ok, _} =+ Task.start(fn ->+ :rpc.multicall(Adapter, :delete, [cache_name, key], multicall_timeout)+ end)++ :ok+ end+@spec get(any(), Keyword.t()) :: any() | nildef get(key, opts \\ [])diff --git a/lib/plausible/import_deletion_repo.ex b/lib/plausible/deletion_repo.exsimilarity index 77%rename from lib/plausible/import_deletion_repo.exrename to lib/plausible/deletion_repo.exindex 3c8a012d8a..feac404edd 100644--- a/lib/plausible/import_deletion_repo.ex+++ b/lib/plausible/deletion_repo.ex@@ -1,4 +1,4 @@-defmodule Plausible.ImportDeletionRepo do+defmodule Plausible.DeletionRepo do@moduledoc """A dedicated repo for import related mutations"""@@ -9,7 +9,7 @@ defmodule Plausible.ImportDeletionRepo dodefmacro __using__(_) doquote do- alias Plausible.ImportDeletionRepo+ alias Plausible.DeletionRepoimport Ectoimport Ecto.Query, only: [from: 1, from: 2]enddiff --git a/lib/plausible/pending_stats_deletion.ex b/lib/plausible/pending_stats_deletion.exnew file mode 100644index 0000000000..9cedc6433f--- /dev/null+++ b/lib/plausible/pending_stats_deletion.ex@@ -0,0 +1,18 @@+defmodule Plausible.PendingStatsDeletion do+ @moduledoc """+ Schema for tracking pending deletions from ClickHouse+ """++ use Ecto.Schema++ @reasons [:user_request]++ @type t() :: %__MODULE__{}++ schema "pending_stats_deletions" do+ field :site_id, :integer+ field :reason, Ecto.Enum, values: @reasons, default: :user_request++ timestamps()+ end+enddiff --git a/lib/plausible/pending_stats_deletions.ex b/lib/plausible/pending_stats_deletions.exnew file mode 100644index 0000000000..ab16905673--- /dev/null+++ b/lib/plausible/pending_stats_deletions.ex@@ -0,0 +1,103 @@+defmodule Plausible.PendingStatsDeletions do+ @moduledoc """+ Context for pending stats deletions+ """++ import Ecto.Query++ alias Plausible.ClickhouseRepo+ alias Plausible.PendingStatsDeletion+ alias Plausible.Repo+ alias Plausible.Site++ @spec store(Site.t(), atom()) :: {:ok, PendingStatsDeletion.t()}+ def store(%Site{} = site, reason \\ :user_request) do+ Repo.insert(%PendingStatsDeletion{site_id: site.id, reason: reason})+ end++ @spec list(atom()) :: [pos_integer()]+ def list(reason \\ :user_request) do+ from(p in PendingStatsDeletion,+ where: p.reason == ^reason,+ distinct: true,+ order_by: p.site_id,+ select: p.site_id+ )+ |> Repo.all()+ end++ @spec clear([pos_integer()], atom()) :: {non_neg_integer(), nil}+ def clear(site_ids, reason \\ :user_request)+ def clear([], _reason), do: {0, nil}++ def clear(site_ids, reason) do+ Repo.delete_all(+ from(p in PendingStatsDeletion, where: p.site_id in ^site_ids and p.reason == ^reason)+ )+ end++ # Temporary. Bridges sites deleted before pending stats deletion tracking+ # existed. Finds sites with orphaned ClickHouse data (no matching Postgres+ # site) and records a pending deletion for each, so `ClickhouseCleanSites`+ # picks them up via `list/1`. Safe to run more than once. Remove+ # once it's been run in every environment.+ @spec backfill_orphaned_sites() :: {:ok, non_neg_integer()}+ def backfill_orphaned_sites() do+ already_tracked = MapSet.new(list())+ now = NaiveDateTime.utc_now(:second)++ records =+ orphaned_clickhouse_site_ids()+ |> Enum.reject(&(&1 in already_tracked))+ |> Enum.map(fn site_id ->+ %{site_id: site_id, reason: :user_request, inserted_at: now, updated_at: now}+ end)++ {count, _} = Repo.insert_all(PendingStatsDeletion, records)++ {:ok, count}+ end++ @all_stats_tables [+ "events_v2",+ "sessions_v2",+ "imported_browsers",+ "imported_devices",+ "imported_entry_pages",+ "imported_exit_pages",+ "imported_locations",+ "imported_operating_systems",+ "imported_pages",+ "imported_custom_events",+ "imported_sources",+ "imported_visitors"+ ]++ defp orphaned_clickhouse_site_ids() do+ pg_site_ids =+ from(s in Site.regular(), select: s.id)+ |> Repo.all()+ |> MapSet.new()++ {:ok, ch} =+ Ch.start_link(ClickhouseRepo.get_config_without_ch_query_execution_timeout())++ query =+ Enum.map_join(+ @all_stats_tables,+ "\nUNION DISTINCT\n",+ &"SELECT site_id FROM #{&1} GROUP BY site_id"+ )++ %Ch.Result{columns: ["site_id"], rows: rows} =+ DBConnection.run(+ ch,+ fn conn -> Ch.query!(conn, query, [], timeout: :infinity) end,+ timeout: :infinity+ )++ ch_site_ids = rows |> MapSet.new(fn [site_id] -> site_id end)++ MapSet.difference(ch_site_ids, pg_site_ids) |> MapSet.to_list()+ end+enddiff --git a/lib/plausible/purge.ex b/lib/plausible/purge.exindex 6e4b56f159..4d80235547 100644--- a/lib/plausible/purge.ex+++ b/lib/plausible/purge.ex@@ -48,7 +48,7 @@ defmodule Plausible.Purge dodef delete_imported_stats!(%Plausible.Site{} = site) doEnum.each(Plausible.Imported.tables(), fn table ->sql = "ALTER TABLE #{table} DELETE WHERE site_id = {$0:UInt64}"- Ecto.Adapters.SQL.query!(Plausible.ImportDeletionRepo, sql, [site.id])+ Ecto.Adapters.SQL.query!(Plausible.DeletionRepo, sql, [site.id])end)Plausible.Sites.clear_stats_start_date!(site)@@ -71,7 +71,7 @@ defmodule Plausible.Purge doEnum.each(Plausible.Imported.tables(), fn table ->sql = "ALTER TABLE #{table} DELETE WHERE site_id = {$0:UInt64} AND import_id = {$1:UInt64}"- Ecto.Adapters.SQL.query!(Plausible.ImportDeletionRepo, sql, [site.id, import_id])+ Ecto.Adapters.SQL.query!(Plausible.DeletionRepo, sql, [site.id, import_id])end)Plausible.Sites.clear_stats_start_date!(site)diff --git a/lib/plausible/site/cache.ex b/lib/plausible/site/cache.exindex 2c559f66ca..ea8a44787a 100644--- a/lib/plausible/site/cache.ex+++ b/lib/plausible/site/cache.ex@@ -60,7 +60,8 @@ defmodule Plausible.Site.Cache do@impl truedef get_from_source(domain) do- query = from s in base_db_query(), where: s.domain == ^domain+ query =+ from s in base_db_query(), where: s.domain == ^domain or s.domain_changed_from == ^domaincase Plausible.Repo.one(query) do{_, _, site = %Site{}} -> %Site{site | from_cache?: false}diff --git a/lib/plausible/site/removal.ex b/lib/plausible/site/removal.exindex 1d1f27512f..931503e272 100644--- a/lib/plausible/site/removal.ex+++ b/lib/plausible/site/removal.ex@@ -4,16 +4,19 @@ defmodule Plausible.Site.Removal do"""use Plausible+ alias Plausible.PendingStatsDeletionsalias Plausible.Repoalias Plausible.Teamsimport Ecto.Query- @spec run(Plausible.Site.t()) :: {:ok, map()}- def run(site) do+ @spec run(Plausible.Site.t(), Keyword.t()) :: {:ok, map()}+ def run(site, opts \\ []) doRepo.transaction(fn ->site = Repo.preload(site, :team)+ {:ok, pending_stats_deletion} = PendingStatsDeletions.store(site)+result = Repo.delete_all(from(s in Plausible.Site, where: s.domain == ^site.domain))Teams.Memberships.prune_guests(site.team)@@ -27,7 +30,14 @@ defmodule Plausible.Site.Removal doPlausible.Billing.SiteLocker.update_for(site.team, send_email?: false)end- %{delete_all: result}+ cache_opts = Keyword.take(opts, [:cache_name, :multicall_timeout])+ :ok = Plausible.Site.Cache.broadcast_delete(site.domain, cache_opts)++ if site.domain_changed_from do+ :ok = Plausible.Site.Cache.broadcast_delete(site.domain_changed_from, cache_opts)+ end++ %{delete_all: result, pending_stats_deletion: pending_stats_deletion}end)endenddiff --git a/lib/plausible/sites.ex b/lib/plausible/sites.exindex d936a883a8..724a5b2909 100644--- a/lib/plausible/sites.ex+++ b/lib/plausible/sites.ex@@ -398,13 +398,7 @@ defmodule Plausible.Sites doenddef ensure_stats_start_date(%Site{} = site) do- start_date =- [- Plausible.Imported.earliest_import_start_date(site),- native_stats_start_date(site)- ]- |> Enum.reject(&is_nil/1)- |> Enum.min(Date, fn -> nil end)+ start_date = compute_stats_start_date(site)if start_date dosite@@ -421,6 +415,15 @@ defmodule Plausible.Sites doPlausible.Stats.Clickhouse.pageview_start_date_local(site)end+ defp compute_stats_start_date(site) do+ [+ Plausible.Imported.earliest_import_start_date(site),+ native_stats_start_date(site)+ ]+ |> Enum.reject(&is_nil/1)+ |> Enum.min(Date, fn -> nil end)+ end+def has_stats?(site) do!!ensure_stats_start_date(site).stats_start_dateenddiff --git a/lib/plausible/telemetry/plausible_metrics.ex b/lib/plausible/telemetry/plausible_metrics.exindex 6ed2543ea4..3e2ca772f7 100644--- a/lib/plausible/telemetry/plausible_metrics.ex+++ b/lib/plausible/telemetry/plausible_metrics.ex@@ -219,6 +219,26 @@ defmodule Plausible.PromEx.Plugins.PlausibleMetrics dometric_prefix ++ [:verification, :unhandled],event_name: InstallationSupport.Verification.Checks.telemetry_event_unhandled())+ ),+ distribution(+ metric_prefix ++ [:clickhouse_clean_sites, :stage, :duration],+ event_name: Plausible.Workers.ClickhouseCleanSites.telemetry_stage_duration(),+ reporter_options: [+ buckets: [10, 50, 100, 500, 1_000, 5_000, 10_000, 30_000, 60_000, 300_000, 600_000]+ ],+ unit: {:native, :millisecond},+ measurement: :duration,+ tags: [:stage]+ ),+ last_value(+ metric_prefix ++ [:clickhouse_clean_sites, :run, :sites_count],+ event_name: Plausible.Workers.ClickhouseCleanSites.telemetry_run_event(),+ measurement: :sites_count+ ),+ last_value(+ metric_prefix ++ [:clickhouse_clean_sites, :run, :partitions_count],+ event_name: Plausible.Workers.ClickhouseCleanSites.telemetry_run_event(),+ measurement: :partitions_count)]|> Enum.concat(persistor_metrics(metric_prefix))diff --git a/lib/workers/clickhouse_clean_sites.ex b/lib/workers/clickhouse_clean_sites.exindex a425fdfa58..234f80a467 100644--- a/lib/workers/clickhouse_clean_sites.ex+++ b/lib/workers/clickhouse_clean_sites.ex@@ -1,24 +1,25 @@defmodule Plausible.Workers.ClickhouseCleanSites do@moduledoc """- Cleans deleted site data from ClickHouse asynchronously.+ Cleans deleted site data from ClickHouse asynchronously, using lightweight+ deletes (except for ingest counters).++ Deletes against `events_v2` and `sessions_v2` scoped to relevant+ partitions only. The remaining tables aren't partitioned,+ so they're cleared in one go.We batch up data deletions from ClickHouse as deleting a single site isjust as expensive as deleting many."""- use Plausible.Repo- use Plausible.ClickhouseRepo- use Plausible.IngestRepo+ use Plausible.DeletionRepouse Oban.Worker, queue: :clickhouse_clean_sites- import Ecto.Query+ alias Plausible.PendingStatsDeletions+ alias Plausible.PromEx.Plugins.PlausibleMetricsrequire Logger- @tables_to_clear [- "events_v2",- "sessions_v2",- "ingest_counters",+ @unpartitioned_tables ["imported_browsers","imported_devices","imported_entry_pages",@@ -31,51 +32,109 @@ defmodule Plausible.Workers.ClickhouseCleanSites do"imported_visitors"]+ # ingest_counters has a projection (`ingest_counters_site_traffic_projection`),+ # and ClickHouse refuses lightweight deletes against tables with projections+ # - fall back to a mutation which rebuilds the projection.+ @mutation_only_tables ["ingest_counters"]+@settings if Mix.env() in [:test, :ce_test, :e2e_test], do: [mutations_sync: 2], else: []+ @spec telemetry_run_event() :: [atom()]+ def telemetry_run_event(), do: [:plausible, :clickhouse_clean_sites, :run]++ @spec telemetry_stage_duration() :: [atom()]+ def telemetry_stage_duration(), do: [:plausible, :clickhouse_clean_sites, :stage]+def perform(_job) do- deleted_sites = get_deleted_sites_with_clickhouse_data()-- if not Enum.empty?(deleted_sites) do- Logger.notice(- "Clearing ClickHouse data for the following #{length(deleted_sites)} sites which have been deleted: #{inspect(deleted_sites)}"- )-- for table <- @tables_to_clear do- IngestRepo.query!(- "ALTER TABLE {$0:Identifier} DELETE WHERE site_id IN {$1:Array(UInt64)}",- [table, deleted_sites],- settings: @settings+ case measure_stage("list_pending_deletions", fn -> PendingStatsDeletions.list() end) do+ [] ->+ :ok++ site_ids ->+ partition_ids_events =+ measure_stage("get_partition_ids_events", fn -> partition_ids("events_v2", site_ids) end)++ partition_ids_sessions =+ measure_stage("get_partition_ids_sessions", fn ->+ partition_ids("sessions_v2", site_ids)+ end)++ :telemetry.execute(telemetry_run_event(), %{+ sites_count: length(site_ids),+ partitions_count_events: length(partition_ids_events),+ partitions_count_sessions: length(partition_ids_sessions)+ })++ Logger.warning(+ "Clearing ClickHouse data for #{length(site_ids)} sites across #{length(partition_ids_events)}/#{length(partition_ids_sessions)} partitions")- end++ measure_stage("events_deletion", fn ->+ for partition_id <- partition_ids_events do+ clear_partitioned_table!("events_v2", partition_id, site_ids)+ end+ end)++ measure_stage("sessions_deletion", fn ->+ for partition_id <- partition_ids_sessions do+ clear_partitioned_table!("sessions_v2", partition_id, site_ids)+ end+ end)++ measure_stage("unpartitioned_tables", fn ->+ for table <- @unpartitioned_tables do+ clear_unpartitioned_table!(table, site_ids)+ end+ end)++ measure_stage("mutation_only_tables", fn ->+ for table <- @mutation_only_tables do+ clear_table_via_mutation!(table, site_ids)+ end+ end)++ measure_stage("clear_pending_deletions", fn ->+ PendingStatsDeletions.clear(site_ids)+ end)++ :okend+ end++ defp partition_ids(table, site_ids) do+ from(e in table,+ where: e.site_id in ^site_ids,+ distinct: true,+ select: field(e, :_partition_id)+ )+ |> DeletionRepo.all()+ end++ defp measure_stage(stage, fun) do+ PlausibleMetrics.measure_duration(telemetry_stage_duration(), fun, %{stage: stage})+ end++ defp clear_partitioned_table!(table, partition_id, site_ids) do+ DeletionRepo.query!(+ "DELETE FROM {$0:Identifier} IN PARTITION ID {$1:String} WHERE site_id IN {$2:Array(UInt64)}",+ [table, partition_id, site_ids],+ settings: @settings+ )+ end- :ok+ defp clear_unpartitioned_table!(table, site_ids) do+ DeletionRepo.query!(+ "DELETE FROM {$0:Identifier} WHERE site_id IN {$1:Array(UInt64)}",+ [table, site_ids],+ settings: @settings+ )end- def get_deleted_sites_with_clickhouse_data() do- pg_sites =- from(s in Plausible.Site.regular(), select: s.id)- |> Plausible.Repo.all()- |> MapSet.new()-- {:ok, ch} =- Plausible.ClickhouseRepo.get_config_without_ch_query_execution_timeout()- |> Ch.start_link()-- %Ch.Result{columns: ["site_id"], rows: rows} =- DBConnection.run(- ch,- fn conn ->- Ch.query!(conn, "FROM events_v2 SELECT site_id GROUP BY site_id", [],- timeout: :infinity- )- end,- timeout: :infinity- )-- ch_sites = rows |> MapSet.new(fn [site_id] -> site_id end)-- MapSet.difference(ch_sites, pg_sites) |> MapSet.to_list()+ defp clear_table_via_mutation!(table, site_ids) do+ DeletionRepo.query!(+ "ALTER TABLE {$0:Identifier} DELETE WHERE site_id IN {$1:Array(UInt64)}",+ [table, site_ids],+ settings: @settings+ )endenddiff --git a/test/plausible/cache_test.exs b/test/plausible/cache_test.exsindex 6ed629f20d..d23dc37233 100644--- a/test/plausible/cache_test.exs+++ b/test/plausible/cache_test.exs@@ -133,6 +133,16 @@ defmodule Plausible.CacheTest do{ExampleCache.get("item1", cache_name: test, force?: true) == :item1, :ok}end)end++ test "broadcast_delete deletes from local cache", %{test: test} do+ {:ok, _} = start_test_cache(test)+ :item1 = ExampleCache.put("item1", :item1, cache_name: test)+ :ok = ExampleCache.broadcast_delete("item1", cache_name: test)++ assert eventually(fn ->+ {ExampleCache.get("item1", cache_name: test, force?: true) == nil, :ok}+ end)+ endenddescribe "warming the cache" dodiff --git a/test/plausible/pending_stats_deletions_test.exs b/test/plausible/pending_stats_deletions_test.exsnew file mode 100644index 0000000000..f25e1ab216--- /dev/null+++ b/test/plausible/pending_stats_deletions_test.exs@@ -0,0 +1,142 @@+defmodule Plausible.PendingStatsDeletionsTest do+ use Plausible.DataCase, async: true++ alias Plausible.PendingStatsDeletion+ alias Plausible.PendingStatsDeletions++ describe "store/2" do+ test "creates a record for the site" do+ site = new_site()++ assert {:ok, %PendingStatsDeletion{} = pending_deletion} = PendingStatsDeletions.store(site)++ assert pending_deletion.site_id == site.id+ assert pending_deletion.reason == :user_request++ assert Repo.get_by(PendingStatsDeletion, site_id: site.id) == pending_deletion+ end++ test "stores a record even if the site has no stats" do+ site = new_site()++ assert {:ok, %PendingStatsDeletion{}} = PendingStatsDeletions.store(site)++ assert Repo.get_by(PendingStatsDeletion, site_id: site.id)+ end++ test "defaults reason to :user_request" do+ site = new_site()++ assert {:ok, pending_deletion} = PendingStatsDeletions.store(site)+ assert pending_deletion.reason == :user_request+ end++ test "accepts an explicit reason" do+ site = new_site()++ assert {:ok, pending_deletion} = PendingStatsDeletions.store(site, :user_request)+ assert pending_deletion.reason == :user_request+ end+ end++ describe "list/1" do+ test "returns an empty list when there are no pending stats deletions" do+ assert PendingStatsDeletions.list() == []+ end++ test "returns the distinct site_ids" do+ insert(:pending_stats_deletion, site_id: 1, reason: :user_request)+ insert(:pending_stats_deletion, site_id: 1, reason: :user_request)+ insert(:pending_stats_deletion, site_id: 2, reason: :user_request)++ assert PendingStatsDeletions.list() == [1, 2]+ end++ test "only considers records with the given reason" do+ insert(:pending_stats_deletion, site_id: 1, reason: :user_request)++ assert PendingStatsDeletions.list(:user_request) == [1]+ end+ end++ describe "clear/2" do+ test "removes records for the given site_ids and reason" do+ insert(:pending_stats_deletion, site_id: 1, reason: :user_request)+ insert(:pending_stats_deletion, site_id: 2, reason: :user_request)++ assert {1, nil} = PendingStatsDeletions.clear([1])++ assert PendingStatsDeletions.list() == [2]+ end++ test "removes all matching records regardless of how many accumulated for a site" do+ insert(:pending_stats_deletion, site_id: 1, reason: :user_request)+ insert(:pending_stats_deletion, site_id: 1, reason: :user_request)++ assert {2, nil} = PendingStatsDeletions.clear([1])+ assert PendingStatsDeletions.list() == []+ end++ test "does not query the database when given an empty list of site_ids" do+ insert(:pending_stats_deletion, site_id: 1, reason: :user_request)++ assert {0, nil} = PendingStatsDeletions.clear([])+ assert Repo.get_by(PendingStatsDeletion, site_id: 1)+ end+ end++ describe "backfill_orphaned_sites/0" do+ test "records a pending stats deletion for a site with clickhouse data but no postgres row" do+ orphaned_site = new_site()++ populate_stats(orphaned_site, [+ build(:pageview, timestamp: ~N[2020-01-01 12:00:00])+ ])++ Repo.delete!(orphaned_site)++ assert {:ok, count} = PendingStatsDeletions.backfill_orphaned_sites()+ assert count >= 1++ assert Repo.get_by(PendingStatsDeletion, site_id: orphaned_site.id, reason: :user_request)+ end++ test "detects sites with only imported stats, not just events_v2/sessions_v2" do+ orphaned_site = new_site()++ populate_stats(orphaned_site, [+ build(:imported_visitors)+ ])++ Repo.delete!(orphaned_site)++ assert {:ok, _} = PendingStatsDeletions.backfill_orphaned_sites()++ assert Repo.get_by(PendingStatsDeletion, site_id: orphaned_site.id)+ end++ test "does not record anything for a site that still exists in postgres" do+ site = new_site()+ populate_stats(site, [build(:pageview)])++ PendingStatsDeletions.backfill_orphaned_sites()++ refute Repo.get_by(PendingStatsDeletion, site_id: site.id)+ end++ test "does not create a duplicate record when run more than once" do+ orphaned_site = new_site()+ populate_stats(orphaned_site, [build(:pageview)])+ Repo.delete!(orphaned_site)++ assert {:ok, _} = PendingStatsDeletions.backfill_orphaned_sites()+ assert {:ok, _} = PendingStatsDeletions.backfill_orphaned_sites()++ matching_records =+ from(p in PendingStatsDeletion, where: p.site_id == ^orphaned_site.id)+ |> Repo.aggregate(:count)++ assert matching_records == 1+ end+ end+enddiff --git a/test/plausible/site/site_removal_test.exs b/test/plausible/site/site_removal_test.exsindex c1e780cd83..11f7cfa15f 100644--- a/test/plausible/site/site_removal_test.exs+++ b/test/plausible/site/site_removal_test.exs@@ -2,6 +2,7 @@ defmodule Plausible.Site.SiteRemovalTest douse Plausible.DataCase, async: trueuse Oban.Testing, repo: Plausible.Repo+ alias Plausible.PendingStatsDeletionalias Plausible.Site.Removalalias Plausible.Sites@@ -12,6 +13,33 @@ defmodule Plausible.Site.SiteRemovalTest dorefute Sites.get_by_domain(site.domain)end+ test "site deletion stores a pending stats deletion record" do+ site = new_site()++ populate_stats(site, [+ build(:pageview, timestamp: ~N[2020-01-01 12:00:00]),+ build(:pageview, timestamp: ~N[2020-01-10 12:00:00])+ ])++ assert {:ok, context} = Removal.run(site)++ assert %PendingStatsDeletion{} = pending_deletion = context.pending_stats_deletion+ assert pending_deletion.site_id == site.id+ assert pending_deletion.reason == :user_request++ assert Repo.get_by(PendingStatsDeletion, site_id: site.id)+ end++ test "site deletion stores a pending stats deletion record even if the site has no stats" do+ site = new_site()++ assert {:ok, context} = Removal.run(site)++ assert %PendingStatsDeletion{site_id: site_id} = context.pending_stats_deletion+ assert site_id == site.id+ assert Repo.get_by(PendingStatsDeletion, site_id: site.id)+ end+test "site deletion prunes team guest memberships" doowner = new_user()site = new_site(owner: owner)@@ -86,4 +114,24 @@ defmodule Plausible.Site.SiteRemovalTest doassert Plausible.ConsolidatedView.get(team)endend++ test "site is removed from sites cache upon deletion", %{test: test} do+ {:ok, _} = start_test_cache(test)+ site = new_site(domain_changed_from: "#{test}")++ Plausible.Site.Cache.refresh_all(cache_name: test)++ assert Plausible.Site.Cache.get(site.domain, cache_name: test, force?: true)+ assert Plausible.Site.Cache.get(site.domain_changed_from, cache_name: test, force?: true)++ assert {:ok, _} = Removal.run(site, cache_name: test)++ refute Plausible.Site.Cache.get(site.domain, cache_name: test, force?: true)+ refute Plausible.Site.Cache.get(site.domain_changed_from, cache_name: test, force?: true)+ end++ defp start_test_cache(cache_name) do+ %{start: {m, f, a}} = Plausible.Site.Cache.child_spec(cache_name: cache_name)+ apply(m, f, a)+ endenddiff --git a/test/plausible/sites_test.exs b/test/plausible/sites_test.exsindex 8007f9f638..e530d8722e 100644--- a/test/plausible/sites_test.exs+++ b/test/plausible/sites_test.exs@@ -145,13 +145,13 @@ defmodule Plausible.SitesTest dodescribe "ensure_stats_start_date" dotest "is nil if site has no stats" do- site = insert(:site)+ site = new_site()assert Sites.ensure_stats_start_date(site).stats_start_date == nilendtest "is date if site does have stats" do- site = insert(:site)+ site = new_site()populate_stats(site, [build(:pageview)@@ -162,7 +162,7 @@ defmodule Plausible.SitesTest doendtest "memoizes value of start date" do- site = insert(:site)+ site = new_site()assert site.stats_start_date == nil@@ -219,13 +219,13 @@ defmodule Plausible.SitesTest dodescribe "native_stats_start_date" dotest "is nil if site has no stats" do- site = insert(:site)+ site = new_site()assert Sites.native_stats_start_date(site) == nilendtest "is date if site does have stats" do- site = insert(:site)+ site = new_site()populate_stats(site, [build(:pageview)@@ -235,7 +235,7 @@ defmodule Plausible.SitesTest doendtest "ignores imported stats" do- site = insert(:site)+ site = new_site()insert(:site_import, site: site)assert Sites.native_stats_start_date(site) == nil@@ -244,13 +244,13 @@ defmodule Plausible.SitesTest dodescribe "has_stats?" dotest "is false if site has no stats" do- site = insert(:site)+ site = new_site()refute Sites.has_stats?(site)endtest "is true if site has stats" do- site = insert(:site)+ site = new_site()populate_stats(site, [build(:pageview)diff --git a/test/support/factory.ex b/test/support/factory.exindex e534bb7251..66d9ff45b3 100644--- a/test/support/factory.ex+++ b/test/support/factory.ex@@ -106,6 +106,12 @@ defmodule Plausible.Factory do}end+ def pending_stats_deletion_factory do+ %Plausible.PendingStatsDeletion{+ site_id: sequence(:pending_stats_deletion_site_id, &(&1 + 1))+ }+ end+def ch_session_factory dohostname = sequence(:domain, &"example-#{&1}.com")diff --git a/test/workers/clickhouse_clean_sites_test.exs b/test/workers/clickhouse_clean_sites_test.exsindex 18cc2c96f7..5106b7735d 100644--- a/test/workers/clickhouse_clean_sites_test.exs+++ b/test/workers/clickhouse_clean_sites_test.exs@@ -2,12 +2,16 @@ defmodule Plausible.Workers.ClickhouseCleanSitesTest douse Plausible.DataCaseimport Plausible.Factory+ alias Plausible.Ingestion.Counters.Record+ alias Plausible.PendingStatsDeletionalias Plausible.Workers.ClickhouseCleanSites+ @moduletag :capture_log+@tag :slow- test "deletes data from events and sessions tables" do- site = insert(:site)- deleted_site = insert(:site)+ test "deletes data from events and sessions tables for sites pending stats deletion" do+ site = new_site()+ deleted_site = new_site()populate_stats(site, [build(:pageview)@@ -27,17 +31,10 @@ defmodule Plausible.Workers.ClickhouseCleanSitesTest dobuild(:imported_operating_systems)])- Repo.delete!(deleted_site)-- assert Enum.member?(- ClickhouseCleanSites.get_deleted_sites_with_clickhouse_data(),- deleted_site.id- )+ insert_ingest_counter(site.id)+ insert_ingest_counter(deleted_site.id)- assert not Enum.member?(- ClickhouseCleanSites.get_deleted_sites_with_clickhouse_data(),- site.id- )+ assert {:ok, _} = Plausible.Site.Removal.run(deleted_site)ClickhouseCleanSites.perform(nil)@@ -52,17 +49,102 @@ defmodule Plausible.Workers.ClickhouseCleanSitesTest doassert_count(deleted_site, "imported_devices", 0)assert_count(deleted_site, "imported_browsers", 0)assert_count(deleted_site, "imported_operating_systems", 0)+ # ingest_counters has a projection, so it's cleared via a mutation rather+ # than a lightweight delete - regression coverage for that special case+ assert_count(deleted_site, "ingest_counters", 0)assert_count(site, "events_v2", 1)assert_count(site, "sessions_v2", 1)+ assert_count(site, "ingest_counters", 1)- assert not Enum.member?(- ClickhouseCleanSites.get_deleted_sites_with_clickhouse_data(),- deleted_site.id- )+ refute Repo.get_by(PendingStatsDeletion, site_id: deleted_site.id)+ end++ @tag :slow+ test "deletes data spanning multiple monthly partitions" do+ deleted_site = new_site()++ populate_stats(deleted_site, [+ build(:pageview, timestamp: ~N[2020-01-15 12:00:00]),+ build(:pageview, timestamp: ~N[2020-03-15 12:00:00])+ ])++ assert {:ok, _} = Plausible.Site.Removal.run(deleted_site)++ ClickhouseCleanSites.perform(nil)++ assert_count(deleted_site, "events_v2", 0)+ assert_count(deleted_site, "sessions_v2", 0)+ end++ @tag :slow+ test "emits telemetry for the run and for each stage", %{test: test} do+ test_pid = self()++ telemetry_run = ClickhouseCleanSites.telemetry_run_event()+ telemetry_stage = ClickhouseCleanSites.telemetry_stage_duration()++ :telemetry.attach_many(+ "#{test}-telemetry-handler",+ [telemetry_run, telemetry_stage],+ fn event, measurements, metadata, _ ->+ send(test_pid, {:telemetry_handled, event, measurements, metadata})+ end,+ %{}+ )++ deleted_site = new_site()++ populate_stats(deleted_site, [+ build(:pageview, timestamp: ~N[2020-01-15 12:00:00]),+ build(:pageview, timestamp: ~N[2020-03-15 12:00:00])+ ])++ assert {:ok, _} = Plausible.Site.Removal.run(deleted_site)++ ClickhouseCleanSites.perform(nil)++ assert_receive {:telemetry_handled, ^telemetry_stage, %{duration: _},+ %{stage: "list_pending_deletions"}}++ assert_receive {:telemetry_handled, ^telemetry_stage, %{duration: _},+ %{stage: "get_partition_ids_events"}}++ assert_receive {:telemetry_handled, ^telemetry_stage, %{duration: _},+ %{stage: "get_partition_ids_sessions"}}++ # only January and March actually have data, February skipped+ assert_receive {:telemetry_handled, ^telemetry_run,+ %{sites_count: 1, partitions_count_events: 2, partitions_count_sessions: 2},+ %{}}++ assert_receive {:telemetry_handled, ^telemetry_stage, %{duration: _},+ %{stage: "events_deletion"}}++ assert_receive {:telemetry_handled, ^telemetry_stage, %{duration: _},+ %{stage: "sessions_deletion"}}++ assert_receive {:telemetry_handled, ^telemetry_stage, %{duration: _},+ %{stage: "mutation_only_tables"}}++ assert_receive {:telemetry_handled, ^telemetry_stage, %{duration: _},+ %{stage: "clear_pending_deletions"}}enddef assert_count(site, table, expected_count) doq = from(e in table, select: %{count: fragment("count()")}, where: e.site_id == ^site.id)await_clickhouse_count(q, expected_count)end++ defp insert_ingest_counter(site_id) do+ Plausible.IngestRepo.insert_all(Record, [+ %{+ event_timebucket: DateTime.utc_now() |> DateTime.truncate(:second),+ site_id: site_id,+ domain: "example.com",+ metric: "pageview",+ value: 1,+ tracker_script_version: 0+ }+ ])+ endend