Skip to content

Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.

Reduce site deletions impact on ClickHouse (#6591)

Adam8d54c4d57794
diff --git a/config/runtime.exs b/config/runtime.exs
index abc25e0f31..315f27de92 100644
--- a/config/runtime.exs
+++ b/config/runtime.exs
@@ -705,12 +705,12 @@ config :plausible, Plausible.AsyncInsertRepo,
materialized_views_ignore_errors: 1
]
-config :plausible, Plausible.ImportDeletionRepo,
+config :plausible, Plausible.DeletionRepo,
queue_target: 500,
queue_interval: 2000,
url: ch_db_url,
transport_opts: ch_transport_opts,
- pool_size: 1
+ pool_size: 2
config :plausible, Plausible.Ingestion.Persistor,
backend: persistor_backend,
@@ -848,10 +848,10 @@ cloud_cron = [
{"0 0 * * *", Plausible.Workers.LockSites},
# Daily at 8
{"0 8 * * *", Plausible.Workers.AcceptTrafficUntil},
- # First sunday of the month, 4:00 UTC
- {"0 4 1-7 * SUN", Plausible.Workers.ClickhouseCleanSites},
- # Daily at 4:00 UTC
- {"0 4 * * *", Plausible.Workers.SetLegacyTimeOnPageCutoff},
+ # Every Tuesday, 3:00 UTC
+ {"0 3 * * TUE", Plausible.Workers.ClickhouseCleanSites},
+ # Daily at 5:00 UTC
+ {"0 5 * * *", Plausible.Workers.SetLegacyTimeOnPageCutoff},
# Daily at 2:00 UTC
{"0 2 * * *", Plausible.Workers.ScoreTrialProspects}
]
diff --git a/lib/plausible/application.ex b/lib/plausible/application.ex
index 265341480d..9cf02cd280 100644
--- a/lib/plausible/application.ex
+++ b/lib/plausible/application.ex
@@ -44,7 +44,7 @@ defmodule Plausible.Application do
Plausible.ClickhouseRepo,
Plausible.IngestRepo,
Plausible.AsyncInsertRepo,
- Plausible.ImportDeletionRepo,
+ Plausible.DeletionRepo,
Plausible.Cache.Adapter.child_spec(:customer_currency, :cache_customer_currency,
ttl_check_interval: :timer.minutes(5),
n_lock_partitions: 1,
diff --git a/lib/plausible/cache.ex b/lib/plausible/cache.ex
index 496c02784f..2bbefbcc93 100644
--- a/lib/plausible/cache.ex
+++ b/lib/plausible/cache.ex
@@ -83,6 +83,19 @@ defmodule Plausible.Cache do
:ok
end
+ @spec broadcast_delete(any(), Keyword.t()) :: :ok
+ def broadcast_delete(key, opts \\ []) do
+ cache_name = Keyword.get(opts, :cache_name, name())
+ multicall_timeout = Keyword.get(opts, :multicall_timeout, :timer.seconds(5))
+
+ {:ok, _} =
+ Task.start(fn ->
+ :rpc.multicall(Adapter, :delete, [cache_name, key], multicall_timeout)
+ end)
+
+ :ok
+ end
+
@spec get(any(), Keyword.t()) :: any() | nil
def get(key, opts \\ [])
diff --git a/lib/plausible/import_deletion_repo.ex b/lib/plausible/deletion_repo.ex
similarity index 77%
rename from lib/plausible/import_deletion_repo.ex
rename to lib/plausible/deletion_repo.ex
index 3c8a012d8a..feac404edd 100644
--- a/lib/plausible/import_deletion_repo.ex
+++ b/lib/plausible/deletion_repo.ex
@@ -1,4 +1,4 @@
-defmodule Plausible.ImportDeletionRepo do
+defmodule Plausible.DeletionRepo do
@moduledoc """
A dedicated repo for import related mutations
"""
@@ -9,7 +9,7 @@ defmodule Plausible.ImportDeletionRepo do
defmacro __using__(_) do
quote do
- alias Plausible.ImportDeletionRepo
+ alias Plausible.DeletionRepo
import Ecto
import Ecto.Query, only: [from: 1, from: 2]
end
diff --git a/lib/plausible/pending_stats_deletion.ex b/lib/plausible/pending_stats_deletion.ex
new file mode 100644
index 0000000000..9cedc6433f
--- /dev/null
+++ b/lib/plausible/pending_stats_deletion.ex
@@ -0,0 +1,18 @@
+defmodule Plausible.PendingStatsDeletion do
+ @moduledoc """
+ Schema for tracking pending deletions from ClickHouse
+ """
+
+ use Ecto.Schema
+
+ @reasons [:user_request]
+
+ @type t() :: %__MODULE__{}
+
+ schema "pending_stats_deletions" do
+ field :site_id, :integer
+ field :reason, Ecto.Enum, values: @reasons, default: :user_request
+
+ timestamps()
+ end
+end
diff --git a/lib/plausible/pending_stats_deletions.ex b/lib/plausible/pending_stats_deletions.ex
new file mode 100644
index 0000000000..ab16905673
--- /dev/null
+++ b/lib/plausible/pending_stats_deletions.ex
@@ -0,0 +1,103 @@
+defmodule Plausible.PendingStatsDeletions do
+ @moduledoc """
+ Context for pending stats deletions
+ """
+
+ import Ecto.Query
+
+ alias Plausible.ClickhouseRepo
+ alias Plausible.PendingStatsDeletion
+ alias Plausible.Repo
+ alias Plausible.Site
+
+ @spec store(Site.t(), atom()) :: {:ok, PendingStatsDeletion.t()}
+ def store(%Site{} = site, reason \\ :user_request) do
+ Repo.insert(%PendingStatsDeletion{site_id: site.id, reason: reason})
+ end
+
+ @spec list(atom()) :: [pos_integer()]
+ def list(reason \\ :user_request) do
+ from(p in PendingStatsDeletion,
+ where: p.reason == ^reason,
+ distinct: true,
+ order_by: p.site_id,
+ select: p.site_id
+ )
+ |> Repo.all()
+ end
+
+ @spec clear([pos_integer()], atom()) :: {non_neg_integer(), nil}
+ def clear(site_ids, reason \\ :user_request)
+ def clear([], _reason), do: {0, nil}
+
+ def clear(site_ids, reason) do
+ Repo.delete_all(
+ from(p in PendingStatsDeletion, where: p.site_id in ^site_ids and p.reason == ^reason)
+ )
+ end
+
+ # Temporary. Bridges sites deleted before pending stats deletion tracking
+ # existed. Finds sites with orphaned ClickHouse data (no matching Postgres
+ # site) and records a pending deletion for each, so `ClickhouseCleanSites`
+ # picks them up via `list/1`. Safe to run more than once. Remove
+ # once it's been run in every environment.
+ @spec backfill_orphaned_sites() :: {:ok, non_neg_integer()}
+ def backfill_orphaned_sites() do
+ already_tracked = MapSet.new(list())
+ now = NaiveDateTime.utc_now(:second)
+
+ records =
+ orphaned_clickhouse_site_ids()
+ |> Enum.reject(&(&1 in already_tracked))
+ |> Enum.map(fn site_id ->
+ %{site_id: site_id, reason: :user_request, inserted_at: now, updated_at: now}
+ end)
+
+ {count, _} = Repo.insert_all(PendingStatsDeletion, records)
+
+ {:ok, count}
+ end
+
+ @all_stats_tables [
+ "events_v2",
+ "sessions_v2",
+ "imported_browsers",
+ "imported_devices",
+ "imported_entry_pages",
+ "imported_exit_pages",
+ "imported_locations",
+ "imported_operating_systems",
+ "imported_pages",
+ "imported_custom_events",
+ "imported_sources",
+ "imported_visitors"
+ ]
+
+ defp orphaned_clickhouse_site_ids() do
+ pg_site_ids =
+ from(s in Site.regular(), select: s.id)
+ |> Repo.all()
+ |> MapSet.new()
+
+ {:ok, ch} =
+ Ch.start_link(ClickhouseRepo.get_config_without_ch_query_execution_timeout())
+
+ query =
+ Enum.map_join(
+ @all_stats_tables,
+ "\nUNION DISTINCT\n",
+ &"SELECT site_id FROM #{&1} GROUP BY site_id"
+ )
+
+ %Ch.Result{columns: ["site_id"], rows: rows} =
+ DBConnection.run(
+ ch,
+ fn conn -> Ch.query!(conn, query, [], timeout: :infinity) end,
+ timeout: :infinity
+ )
+
+ ch_site_ids = rows |> MapSet.new(fn [site_id] -> site_id end)
+
+ MapSet.difference(ch_site_ids, pg_site_ids) |> MapSet.to_list()
+ end
+end
diff --git a/lib/plausible/purge.ex b/lib/plausible/purge.ex
index 6e4b56f159..4d80235547 100644
--- a/lib/plausible/purge.ex
+++ b/lib/plausible/purge.ex
@@ -48,7 +48,7 @@ defmodule Plausible.Purge do
def delete_imported_stats!(%Plausible.Site{} = site) do
Enum.each(Plausible.Imported.tables(), fn table ->
sql = "ALTER TABLE #{table} DELETE WHERE site_id = {$0:UInt64}"
- Ecto.Adapters.SQL.query!(Plausible.ImportDeletionRepo, sql, [site.id])
+ Ecto.Adapters.SQL.query!(Plausible.DeletionRepo, sql, [site.id])
end)
Plausible.Sites.clear_stats_start_date!(site)
@@ -71,7 +71,7 @@ defmodule Plausible.Purge do
Enum.each(Plausible.Imported.tables(), fn table ->
sql = "ALTER TABLE #{table} DELETE WHERE site_id = {$0:UInt64} AND import_id = {$1:UInt64}"
- Ecto.Adapters.SQL.query!(Plausible.ImportDeletionRepo, sql, [site.id, import_id])
+ Ecto.Adapters.SQL.query!(Plausible.DeletionRepo, sql, [site.id, import_id])
end)
Plausible.Sites.clear_stats_start_date!(site)
diff --git a/lib/plausible/site/cache.ex b/lib/plausible/site/cache.ex
index 2c559f66ca..ea8a44787a 100644
--- a/lib/plausible/site/cache.ex
+++ b/lib/plausible/site/cache.ex
@@ -60,7 +60,8 @@ defmodule Plausible.Site.Cache do
@impl true
def get_from_source(domain) do
- query = from s in base_db_query(), where: s.domain == ^domain
+ query =
+ from s in base_db_query(), where: s.domain == ^domain or s.domain_changed_from == ^domain
case Plausible.Repo.one(query) do
{_, _, site = %Site{}} -> %Site{site | from_cache?: false}
diff --git a/lib/plausible/site/removal.ex b/lib/plausible/site/removal.ex
index 1d1f27512f..931503e272 100644
--- a/lib/plausible/site/removal.ex
+++ b/lib/plausible/site/removal.ex
@@ -4,16 +4,19 @@ defmodule Plausible.Site.Removal do
"""
use Plausible
+ alias Plausible.PendingStatsDeletions
alias Plausible.Repo
alias Plausible.Teams
import Ecto.Query
- @spec run(Plausible.Site.t()) :: {:ok, map()}
- def run(site) do
+ @spec run(Plausible.Site.t(), Keyword.t()) :: {:ok, map()}
+ def run(site, opts \\ []) do
Repo.transaction(fn ->
site = Repo.preload(site, :team)
+ {:ok, pending_stats_deletion} = PendingStatsDeletions.store(site)
+
result = Repo.delete_all(from(s in Plausible.Site, where: s.domain == ^site.domain))
Teams.Memberships.prune_guests(site.team)
@@ -27,7 +30,14 @@ defmodule Plausible.Site.Removal do
Plausible.Billing.SiteLocker.update_for(site.team, send_email?: false)
end
- %{delete_all: result}
+ cache_opts = Keyword.take(opts, [:cache_name, :multicall_timeout])
+ :ok = Plausible.Site.Cache.broadcast_delete(site.domain, cache_opts)
+
+ if site.domain_changed_from do
+ :ok = Plausible.Site.Cache.broadcast_delete(site.domain_changed_from, cache_opts)
+ end
+
+ %{delete_all: result, pending_stats_deletion: pending_stats_deletion}
end)
end
end
diff --git a/lib/plausible/sites.ex b/lib/plausible/sites.ex
index d936a883a8..724a5b2909 100644
--- a/lib/plausible/sites.ex
+++ b/lib/plausible/sites.ex
@@ -398,13 +398,7 @@ defmodule Plausible.Sites do
end
def ensure_stats_start_date(%Site{} = site) do
- start_date =
- [
- Plausible.Imported.earliest_import_start_date(site),
- native_stats_start_date(site)
- ]
- |> Enum.reject(&is_nil/1)
- |> Enum.min(Date, fn -> nil end)
+ start_date = compute_stats_start_date(site)
if start_date do
site
@@ -421,6 +415,15 @@ defmodule Plausible.Sites do
Plausible.Stats.Clickhouse.pageview_start_date_local(site)
end
+ defp compute_stats_start_date(site) do
+ [
+ Plausible.Imported.earliest_import_start_date(site),
+ native_stats_start_date(site)
+ ]
+ |> Enum.reject(&is_nil/1)
+ |> Enum.min(Date, fn -> nil end)
+ end
+
def has_stats?(site) do
!!ensure_stats_start_date(site).stats_start_date
end
diff --git a/lib/plausible/telemetry/plausible_metrics.ex b/lib/plausible/telemetry/plausible_metrics.ex
index 6ed2543ea4..3e2ca772f7 100644
--- a/lib/plausible/telemetry/plausible_metrics.ex
+++ b/lib/plausible/telemetry/plausible_metrics.ex
@@ -219,6 +219,26 @@ defmodule Plausible.PromEx.Plugins.PlausibleMetrics do
metric_prefix ++ [:verification, :unhandled],
event_name: InstallationSupport.Verification.Checks.telemetry_event_unhandled()
)
+ ),
+ distribution(
+ metric_prefix ++ [:clickhouse_clean_sites, :stage, :duration],
+ event_name: Plausible.Workers.ClickhouseCleanSites.telemetry_stage_duration(),
+ reporter_options: [
+ buckets: [10, 50, 100, 500, 1_000, 5_000, 10_000, 30_000, 60_000, 300_000, 600_000]
+ ],
+ unit: {:native, :millisecond},
+ measurement: :duration,
+ tags: [:stage]
+ ),
+ last_value(
+ metric_prefix ++ [:clickhouse_clean_sites, :run, :sites_count],
+ event_name: Plausible.Workers.ClickhouseCleanSites.telemetry_run_event(),
+ measurement: :sites_count
+ ),
+ last_value(
+ metric_prefix ++ [:clickhouse_clean_sites, :run, :partitions_count],
+ event_name: Plausible.Workers.ClickhouseCleanSites.telemetry_run_event(),
+ measurement: :partitions_count
)
]
|> Enum.concat(persistor_metrics(metric_prefix))
diff --git a/lib/workers/clickhouse_clean_sites.ex b/lib/workers/clickhouse_clean_sites.ex
index a425fdfa58..234f80a467 100644
--- a/lib/workers/clickhouse_clean_sites.ex
+++ b/lib/workers/clickhouse_clean_sites.ex
@@ -1,24 +1,25 @@
defmodule Plausible.Workers.ClickhouseCleanSites do
@moduledoc """
- Cleans deleted site data from ClickHouse asynchronously.
+ Cleans deleted site data from ClickHouse asynchronously, using lightweight
+ deletes (except for ingest counters).
+
+ Deletes against `events_v2` and `sessions_v2` scoped to relevant
+ partitions only. The remaining tables aren't partitioned,
+ so they're cleared in one go.
We batch up data deletions from ClickHouse as deleting a single site is
just as expensive as deleting many.
"""
- use Plausible.Repo
- use Plausible.ClickhouseRepo
- use Plausible.IngestRepo
+ use Plausible.DeletionRepo
use Oban.Worker, queue: :clickhouse_clean_sites
- import Ecto.Query
+ alias Plausible.PendingStatsDeletions
+ alias Plausible.PromEx.Plugins.PlausibleMetrics
require Logger
- @tables_to_clear [
- "events_v2",
- "sessions_v2",
- "ingest_counters",
+ @unpartitioned_tables [
"imported_browsers",
"imported_devices",
"imported_entry_pages",
@@ -31,51 +32,109 @@ defmodule Plausible.Workers.ClickhouseCleanSites do
"imported_visitors"
]
+ # ingest_counters has a projection (`ingest_counters_site_traffic_projection`),
+ # and ClickHouse refuses lightweight deletes against tables with projections
+ # - fall back to a mutation which rebuilds the projection.
+ @mutation_only_tables ["ingest_counters"]
+
@settings if Mix.env() in [:test, :ce_test, :e2e_test], do: [mutations_sync: 2], else: []
+ @spec telemetry_run_event() :: [atom()]
+ def telemetry_run_event(), do: [:plausible, :clickhouse_clean_sites, :run]
+
+ @spec telemetry_stage_duration() :: [atom()]
+ def telemetry_stage_duration(), do: [:plausible, :clickhouse_clean_sites, :stage]
+
def perform(_job) do
- deleted_sites = get_deleted_sites_with_clickhouse_data()
-
- if not Enum.empty?(deleted_sites) do
- Logger.notice(
- "Clearing ClickHouse data for the following #{length(deleted_sites)} sites which have been deleted: #{inspect(deleted_sites)}"
- )
-
- for table <- @tables_to_clear do
- IngestRepo.query!(
- "ALTER TABLE {$0:Identifier} DELETE WHERE site_id IN {$1:Array(UInt64)}",
- [table, deleted_sites],
- settings: @settings
+ case measure_stage("list_pending_deletions", fn -> PendingStatsDeletions.list() end) do
+ [] ->
+ :ok
+
+ site_ids ->
+ partition_ids_events =
+ measure_stage("get_partition_ids_events", fn -> partition_ids("events_v2", site_ids) end)
+
+ partition_ids_sessions =
+ measure_stage("get_partition_ids_sessions", fn ->
+ partition_ids("sessions_v2", site_ids)
+ end)
+
+ :telemetry.execute(telemetry_run_event(), %{
+ sites_count: length(site_ids),
+ partitions_count_events: length(partition_ids_events),
+ partitions_count_sessions: length(partition_ids_sessions)
+ })
+
+ Logger.warning(
+ "Clearing ClickHouse data for #{length(site_ids)} sites across #{length(partition_ids_events)}/#{length(partition_ids_sessions)} partitions"
)
- end
+
+ measure_stage("events_deletion", fn ->
+ for partition_id <- partition_ids_events do
+ clear_partitioned_table!("events_v2", partition_id, site_ids)
+ end
+ end)
+
+ measure_stage("sessions_deletion", fn ->
+ for partition_id <- partition_ids_sessions do
+ clear_partitioned_table!("sessions_v2", partition_id, site_ids)
+ end
+ end)
+
+ measure_stage("unpartitioned_tables", fn ->
+ for table <- @unpartitioned_tables do
+ clear_unpartitioned_table!(table, site_ids)
+ end
+ end)
+
+ measure_stage("mutation_only_tables", fn ->
+ for table <- @mutation_only_tables do
+ clear_table_via_mutation!(table, site_ids)
+ end
+ end)
+
+ measure_stage("clear_pending_deletions", fn ->
+ PendingStatsDeletions.clear(site_ids)
+ end)
+
+ :ok
end
+ end
+
+ defp partition_ids(table, site_ids) do
+ from(e in table,
+ where: e.site_id in ^site_ids,
+ distinct: true,
+ select: field(e, :_partition_id)
+ )
+ |> DeletionRepo.all()
+ end
+
+ defp measure_stage(stage, fun) do
+ PlausibleMetrics.measure_duration(telemetry_stage_duration(), fun, %{stage: stage})
+ end
+
+ defp clear_partitioned_table!(table, partition_id, site_ids) do
+ DeletionRepo.query!(
+ "DELETE FROM {$0:Identifier} IN PARTITION ID {$1:String} WHERE site_id IN {$2:Array(UInt64)}",
+ [table, partition_id, site_ids],
+ settings: @settings
+ )
+ end
- :ok
+ defp clear_unpartitioned_table!(table, site_ids) do
+ DeletionRepo.query!(
+ "DELETE FROM {$0:Identifier} WHERE site_id IN {$1:Array(UInt64)}",
+ [table, site_ids],
+ settings: @settings
+ )
end
- def get_deleted_sites_with_clickhouse_data() do
- pg_sites =
- from(s in Plausible.Site.regular(), select: s.id)
- |> Plausible.Repo.all()
- |> MapSet.new()
-
- {:ok, ch} =
- Plausible.ClickhouseRepo.get_config_without_ch_query_execution_timeout()
- |> Ch.start_link()
-
- %Ch.Result{columns: ["site_id"], rows: rows} =
- DBConnection.run(
- ch,
- fn conn ->
- Ch.query!(conn, "FROM events_v2 SELECT site_id GROUP BY site_id", [],
- timeout: :infinity
- )
- end,
- timeout: :infinity
- )
-
- ch_sites = rows |> MapSet.new(fn [site_id] -> site_id end)
-
- MapSet.difference(ch_sites, pg_sites) |> MapSet.to_list()
+ defp clear_table_via_mutation!(table, site_ids) do
+ DeletionRepo.query!(
+ "ALTER TABLE {$0:Identifier} DELETE WHERE site_id IN {$1:Array(UInt64)}",
+ [table, site_ids],
+ settings: @settings
+ )
end
end
diff --git a/test/plausible/cache_test.exs b/test/plausible/cache_test.exs
index 6ed629f20d..d23dc37233 100644
--- a/test/plausible/cache_test.exs
+++ b/test/plausible/cache_test.exs
@@ -133,6 +133,16 @@ defmodule Plausible.CacheTest do
{ExampleCache.get("item1", cache_name: test, force?: true) == :item1, :ok}
end)
end
+
+ test "broadcast_delete deletes from local cache", %{test: test} do
+ {:ok, _} = start_test_cache(test)
+ :item1 = ExampleCache.put("item1", :item1, cache_name: test)
+ :ok = ExampleCache.broadcast_delete("item1", cache_name: test)
+
+ assert eventually(fn ->
+ {ExampleCache.get("item1", cache_name: test, force?: true) == nil, :ok}
+ end)
+ end
end
describe "warming the cache" do
diff --git a/test/plausible/pending_stats_deletions_test.exs b/test/plausible/pending_stats_deletions_test.exs
new file mode 100644
index 0000000000..f25e1ab216
--- /dev/null
+++ b/test/plausible/pending_stats_deletions_test.exs
@@ -0,0 +1,142 @@
+defmodule Plausible.PendingStatsDeletionsTest do
+ use Plausible.DataCase, async: true
+
+ alias Plausible.PendingStatsDeletion
+ alias Plausible.PendingStatsDeletions
+
+ describe "store/2" do
+ test "creates a record for the site" do
+ site = new_site()
+
+ assert {:ok, %PendingStatsDeletion{} = pending_deletion} = PendingStatsDeletions.store(site)
+
+ assert pending_deletion.site_id == site.id
+ assert pending_deletion.reason == :user_request
+
+ assert Repo.get_by(PendingStatsDeletion, site_id: site.id) == pending_deletion
+ end
+
+ test "stores a record even if the site has no stats" do
+ site = new_site()
+
+ assert {:ok, %PendingStatsDeletion{}} = PendingStatsDeletions.store(site)
+
+ assert Repo.get_by(PendingStatsDeletion, site_id: site.id)
+ end
+
+ test "defaults reason to :user_request" do
+ site = new_site()
+
+ assert {:ok, pending_deletion} = PendingStatsDeletions.store(site)
+ assert pending_deletion.reason == :user_request
+ end
+
+ test "accepts an explicit reason" do
+ site = new_site()
+
+ assert {:ok, pending_deletion} = PendingStatsDeletions.store(site, :user_request)
+ assert pending_deletion.reason == :user_request
+ end
+ end
+
+ describe "list/1" do
+ test "returns an empty list when there are no pending stats deletions" do
+ assert PendingStatsDeletions.list() == []
+ end
+
+ test "returns the distinct site_ids" do
+ insert(:pending_stats_deletion, site_id: 1, reason: :user_request)
+ insert(:pending_stats_deletion, site_id: 1, reason: :user_request)
+ insert(:pending_stats_deletion, site_id: 2, reason: :user_request)
+
+ assert PendingStatsDeletions.list() == [1, 2]
+ end
+
+ test "only considers records with the given reason" do
+ insert(:pending_stats_deletion, site_id: 1, reason: :user_request)
+
+ assert PendingStatsDeletions.list(:user_request) == [1]
+ end
+ end
+
+ describe "clear/2" do
+ test "removes records for the given site_ids and reason" do
+ insert(:pending_stats_deletion, site_id: 1, reason: :user_request)
+ insert(:pending_stats_deletion, site_id: 2, reason: :user_request)
+
+ assert {1, nil} = PendingStatsDeletions.clear([1])
+
+ assert PendingStatsDeletions.list() == [2]
+ end
+
+ test "removes all matching records regardless of how many accumulated for a site" do
+ insert(:pending_stats_deletion, site_id: 1, reason: :user_request)
+ insert(:pending_stats_deletion, site_id: 1, reason: :user_request)
+
+ assert {2, nil} = PendingStatsDeletions.clear([1])
+ assert PendingStatsDeletions.list() == []
+ end
+
+ test "does not query the database when given an empty list of site_ids" do
+ insert(:pending_stats_deletion, site_id: 1, reason: :user_request)
+
+ assert {0, nil} = PendingStatsDeletions.clear([])
+ assert Repo.get_by(PendingStatsDeletion, site_id: 1)
+ end
+ end
+
+ describe "backfill_orphaned_sites/0" do
+ test "records a pending stats deletion for a site with clickhouse data but no postgres row" do
+ orphaned_site = new_site()
+
+ populate_stats(orphaned_site, [
+ build(:pageview, timestamp: ~N[2020-01-01 12:00:00])
+ ])
+
+ Repo.delete!(orphaned_site)
+
+ assert {:ok, count} = PendingStatsDeletions.backfill_orphaned_sites()
+ assert count >= 1
+
+ assert Repo.get_by(PendingStatsDeletion, site_id: orphaned_site.id, reason: :user_request)
+ end
+
+ test "detects sites with only imported stats, not just events_v2/sessions_v2" do
+ orphaned_site = new_site()
+
+ populate_stats(orphaned_site, [
+ build(:imported_visitors)
+ ])
+
+ Repo.delete!(orphaned_site)
+
+ assert {:ok, _} = PendingStatsDeletions.backfill_orphaned_sites()
+
+ assert Repo.get_by(PendingStatsDeletion, site_id: orphaned_site.id)
+ end
+
+ test "does not record anything for a site that still exists in postgres" do
+ site = new_site()
+ populate_stats(site, [build(:pageview)])
+
+ PendingStatsDeletions.backfill_orphaned_sites()
+
+ refute Repo.get_by(PendingStatsDeletion, site_id: site.id)
+ end
+
+ test "does not create a duplicate record when run more than once" do
+ orphaned_site = new_site()
+ populate_stats(orphaned_site, [build(:pageview)])
+ Repo.delete!(orphaned_site)
+
+ assert {:ok, _} = PendingStatsDeletions.backfill_orphaned_sites()
+ assert {:ok, _} = PendingStatsDeletions.backfill_orphaned_sites()
+
+ matching_records =
+ from(p in PendingStatsDeletion, where: p.site_id == ^orphaned_site.id)
+ |> Repo.aggregate(:count)
+
+ assert matching_records == 1
+ end
+ end
+end
diff --git a/test/plausible/site/site_removal_test.exs b/test/plausible/site/site_removal_test.exs
index c1e780cd83..11f7cfa15f 100644
--- a/test/plausible/site/site_removal_test.exs
+++ b/test/plausible/site/site_removal_test.exs
@@ -2,6 +2,7 @@ defmodule Plausible.Site.SiteRemovalTest do
use Plausible.DataCase, async: true
use Oban.Testing, repo: Plausible.Repo
+ alias Plausible.PendingStatsDeletion
alias Plausible.Site.Removal
alias Plausible.Sites
@@ -12,6 +13,33 @@ defmodule Plausible.Site.SiteRemovalTest do
refute Sites.get_by_domain(site.domain)
end
+ test "site deletion stores a pending stats deletion record" do
+ site = new_site()
+
+ populate_stats(site, [
+ build(:pageview, timestamp: ~N[2020-01-01 12:00:00]),
+ build(:pageview, timestamp: ~N[2020-01-10 12:00:00])
+ ])
+
+ assert {:ok, context} = Removal.run(site)
+
+ assert %PendingStatsDeletion{} = pending_deletion = context.pending_stats_deletion
+ assert pending_deletion.site_id == site.id
+ assert pending_deletion.reason == :user_request
+
+ assert Repo.get_by(PendingStatsDeletion, site_id: site.id)
+ end
+
+ test "site deletion stores a pending stats deletion record even if the site has no stats" do
+ site = new_site()
+
+ assert {:ok, context} = Removal.run(site)
+
+ assert %PendingStatsDeletion{site_id: site_id} = context.pending_stats_deletion
+ assert site_id == site.id
+ assert Repo.get_by(PendingStatsDeletion, site_id: site.id)
+ end
+
test "site deletion prunes team guest memberships" do
owner = new_user()
site = new_site(owner: owner)
@@ -86,4 +114,24 @@ defmodule Plausible.Site.SiteRemovalTest do
assert Plausible.ConsolidatedView.get(team)
end
end
+
+ test "site is removed from sites cache upon deletion", %{test: test} do
+ {:ok, _} = start_test_cache(test)
+ site = new_site(domain_changed_from: "#{test}")
+
+ Plausible.Site.Cache.refresh_all(cache_name: test)
+
+ assert Plausible.Site.Cache.get(site.domain, cache_name: test, force?: true)
+ assert Plausible.Site.Cache.get(site.domain_changed_from, cache_name: test, force?: true)
+
+ assert {:ok, _} = Removal.run(site, cache_name: test)
+
+ refute Plausible.Site.Cache.get(site.domain, cache_name: test, force?: true)
+ refute Plausible.Site.Cache.get(site.domain_changed_from, cache_name: test, force?: true)
+ end
+
+ defp start_test_cache(cache_name) do
+ %{start: {m, f, a}} = Plausible.Site.Cache.child_spec(cache_name: cache_name)
+ apply(m, f, a)
+ end
end
diff --git a/test/plausible/sites_test.exs b/test/plausible/sites_test.exs
index 8007f9f638..e530d8722e 100644
--- a/test/plausible/sites_test.exs
+++ b/test/plausible/sites_test.exs
@@ -145,13 +145,13 @@ defmodule Plausible.SitesTest do
describe "ensure_stats_start_date" do
test "is nil if site has no stats" do
- site = insert(:site)
+ site = new_site()
assert Sites.ensure_stats_start_date(site).stats_start_date == nil
end
test "is date if site does have stats" do
- site = insert(:site)
+ site = new_site()
populate_stats(site, [
build(:pageview)
@@ -162,7 +162,7 @@ defmodule Plausible.SitesTest do
end
test "memoizes value of start date" do
- site = insert(:site)
+ site = new_site()
assert site.stats_start_date == nil
@@ -219,13 +219,13 @@ defmodule Plausible.SitesTest do
describe "native_stats_start_date" do
test "is nil if site has no stats" do
- site = insert(:site)
+ site = new_site()
assert Sites.native_stats_start_date(site) == nil
end
test "is date if site does have stats" do
- site = insert(:site)
+ site = new_site()
populate_stats(site, [
build(:pageview)
@@ -235,7 +235,7 @@ defmodule Plausible.SitesTest do
end
test "ignores imported stats" do
- site = insert(:site)
+ site = new_site()
insert(:site_import, site: site)
assert Sites.native_stats_start_date(site) == nil
@@ -244,13 +244,13 @@ defmodule Plausible.SitesTest do
describe "has_stats?" do
test "is false if site has no stats" do
- site = insert(:site)
+ site = new_site()
refute Sites.has_stats?(site)
end
test "is true if site has stats" do
- site = insert(:site)
+ site = new_site()
populate_stats(site, [
build(:pageview)
diff --git a/test/support/factory.ex b/test/support/factory.ex
index e534bb7251..66d9ff45b3 100644
--- a/test/support/factory.ex
+++ b/test/support/factory.ex
@@ -106,6 +106,12 @@ defmodule Plausible.Factory do
}
end
+ def pending_stats_deletion_factory do
+ %Plausible.PendingStatsDeletion{
+ site_id: sequence(:pending_stats_deletion_site_id, &(&1 + 1))
+ }
+ end
+
def ch_session_factory do
hostname = sequence(:domain, &"example-#{&1}.com")
diff --git a/test/workers/clickhouse_clean_sites_test.exs b/test/workers/clickhouse_clean_sites_test.exs
index 18cc2c96f7..5106b7735d 100644
--- a/test/workers/clickhouse_clean_sites_test.exs
+++ b/test/workers/clickhouse_clean_sites_test.exs
@@ -2,12 +2,16 @@ defmodule Plausible.Workers.ClickhouseCleanSitesTest do
use Plausible.DataCase
import Plausible.Factory
+ alias Plausible.Ingestion.Counters.Record
+ alias Plausible.PendingStatsDeletion
alias Plausible.Workers.ClickhouseCleanSites
+ @moduletag :capture_log
+
@tag :slow
- test "deletes data from events and sessions tables" do
- site = insert(:site)
- deleted_site = insert(:site)
+ test "deletes data from events and sessions tables for sites pending stats deletion" do
+ site = new_site()
+ deleted_site = new_site()
populate_stats(site, [
build(:pageview)
@@ -27,17 +31,10 @@ defmodule Plausible.Workers.ClickhouseCleanSitesTest do
build(:imported_operating_systems)
])
- Repo.delete!(deleted_site)
-
- assert Enum.member?(
- ClickhouseCleanSites.get_deleted_sites_with_clickhouse_data(),
- deleted_site.id
- )
+ insert_ingest_counter(site.id)
+ insert_ingest_counter(deleted_site.id)
- assert not Enum.member?(
- ClickhouseCleanSites.get_deleted_sites_with_clickhouse_data(),
- site.id
- )
+ assert {:ok, _} = Plausible.Site.Removal.run(deleted_site)
ClickhouseCleanSites.perform(nil)
@@ -52,17 +49,102 @@ defmodule Plausible.Workers.ClickhouseCleanSitesTest do
assert_count(deleted_site, "imported_devices", 0)
assert_count(deleted_site, "imported_browsers", 0)
assert_count(deleted_site, "imported_operating_systems", 0)
+ # ingest_counters has a projection, so it's cleared via a mutation rather
+ # than a lightweight delete - regression coverage for that special case
+ assert_count(deleted_site, "ingest_counters", 0)
assert_count(site, "events_v2", 1)
assert_count(site, "sessions_v2", 1)
+ assert_count(site, "ingest_counters", 1)
- assert not Enum.member?(
- ClickhouseCleanSites.get_deleted_sites_with_clickhouse_data(),
- deleted_site.id
- )
+ refute Repo.get_by(PendingStatsDeletion, site_id: deleted_site.id)
+ end
+
+ @tag :slow
+ test "deletes data spanning multiple monthly partitions" do
+ deleted_site = new_site()
+
+ populate_stats(deleted_site, [
+ build(:pageview, timestamp: ~N[2020-01-15 12:00:00]),
+ build(:pageview, timestamp: ~N[2020-03-15 12:00:00])
+ ])
+
+ assert {:ok, _} = Plausible.Site.Removal.run(deleted_site)
+
+ ClickhouseCleanSites.perform(nil)
+
+ assert_count(deleted_site, "events_v2", 0)
+ assert_count(deleted_site, "sessions_v2", 0)
+ end
+
+ @tag :slow
+ test "emits telemetry for the run and for each stage", %{test: test} do
+ test_pid = self()
+
+ telemetry_run = ClickhouseCleanSites.telemetry_run_event()
+ telemetry_stage = ClickhouseCleanSites.telemetry_stage_duration()
+
+ :telemetry.attach_many(
+ "#{test}-telemetry-handler",
+ [telemetry_run, telemetry_stage],
+ fn event, measurements, metadata, _ ->
+ send(test_pid, {:telemetry_handled, event, measurements, metadata})
+ end,
+ %{}
+ )
+
+ deleted_site = new_site()
+
+ populate_stats(deleted_site, [
+ build(:pageview, timestamp: ~N[2020-01-15 12:00:00]),
+ build(:pageview, timestamp: ~N[2020-03-15 12:00:00])
+ ])
+
+ assert {:ok, _} = Plausible.Site.Removal.run(deleted_site)
+
+ ClickhouseCleanSites.perform(nil)
+
+ assert_receive {:telemetry_handled, ^telemetry_stage, %{duration: _},
+ %{stage: "list_pending_deletions"}}
+
+ assert_receive {:telemetry_handled, ^telemetry_stage, %{duration: _},
+ %{stage: "get_partition_ids_events"}}
+
+ assert_receive {:telemetry_handled, ^telemetry_stage, %{duration: _},
+ %{stage: "get_partition_ids_sessions"}}
+
+ # only January and March actually have data, February skipped
+ assert_receive {:telemetry_handled, ^telemetry_run,
+ %{sites_count: 1, partitions_count_events: 2, partitions_count_sessions: 2},
+ %{}}
+
+ assert_receive {:telemetry_handled, ^telemetry_stage, %{duration: _},
+ %{stage: "events_deletion"}}
+
+ assert_receive {:telemetry_handled, ^telemetry_stage, %{duration: _},
+ %{stage: "sessions_deletion"}}
+
+ assert_receive {:telemetry_handled, ^telemetry_stage, %{duration: _},
+ %{stage: "mutation_only_tables"}}
+
+ assert_receive {:telemetry_handled, ^telemetry_stage, %{duration: _},
+ %{stage: "clear_pending_deletions"}}
end
def assert_count(site, table, expected_count) do
q = from(e in table, select: %{count: fragment("count()")}, where: e.site_id == ^site.id)
await_clickhouse_count(q, expected_count)
end
+
+ defp insert_ingest_counter(site_id) do
+ Plausible.IngestRepo.insert_all(Record, [
+ %{
+ event_timebucket: DateTime.utc_now() |> DateTime.truncate(:second),
+ site_id: site_id,
+ domain: "example.com",
+ metric: "pageview",
+ value: 1,
+ tracker_script_version: 0
+ }
+ ])
+ end
end