Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.
TypeScript analyticsanalytics-dashboardclickhouseelixirgoogle-analyticsmarketingopen-source-analyticsphoenixplausible-analyticspostgresqlprivacyprivacy-friendlyself-hostedsimple-analyticsstatisticstailwindcssweb-analyticswebsite-analyticswebsite-statswebsite-tracking
Delay stats deletions (#2632)
Adam Rutkowski8f9f03296818
diff --git a/config/runtime.exs b/config/runtime.exsindex 53ed3ca925..f3337b46ed 100644--- a/config/runtime.exs+++ b/config/runtime.exs@@ -352,7 +352,8 @@ base_queues = [site_setup_emails: 1,clean_email_verification_codes: 1,clean_invitations: 1,- google_analytics_imports: 1+ google_analytics_imports: 1,+ site_stats_removal: 1]cloud_queues = [diff --git a/lib/plausible/purge.ex b/lib/plausible/purge.exindex 3bb91c9a03..13865a7004 100644--- a/lib/plausible/purge.ex+++ b/lib/plausible/purge.ex@@ -9,18 +9,6 @@ defmodule Plausible.Purge do- [Synchronicity of `ALTER` Queries](https://clickhouse.com/docs/en/sql-reference/statements/alter/#synchronicity-of-alter-queries)"""- @doc """- Deletes a site and all associated stats.- """- @spec delete_site!(Plausible.Site.t()) :: :ok- def delete_site!(site) do- delete_native_stats!(site)- delete_imported_stats!(site)- Plausible.Repo.delete!(site)-- :ok- end-@spec delete_imported_stats!(Plausible.Site.t()) :: :ok@doc """Deletes imported stats from Google Analytics, and clears thediff --git a/lib/plausible/site/removal.ex b/lib/plausible/site/removal.exnew file mode 100644index 0000000000..c8bd91ad72--- /dev/null+++ b/lib/plausible/site/removal.ex@@ -0,0 +1,41 @@+defmodule Plausible.Site.Removal do+ @moduledoc """+ A service responsible for site and its stats deletion.+ The site deletion alone is done in postgres and is executed first,+ the latter deletions (events, sessions and imported tables in clickhouse)+ are performed asynchrnounsly via `Plausible.Workers.StatsRemoval`.++ This is to avoid race condition in which the site is deleted, but stats+ writes are pending (either in the buffers or are about to be buffered, due+ to Sites.Cache keeping the now obsolete record until refresh is triggered).+ """+ @stats_deletion_delay_seconds 60 * 20++ alias Plausible.Workers.StatsRemoval+ alias Plausible.Repo+ alias Ecto.Multi++ import Ecto.Query++ @spec stats_deletion_delay_seconds() :: pos_integer()+ def stats_deletion_delay_seconds() do+ @stats_deletion_delay_seconds+ end++ @spec run(String.t()) :: {:ok, map()}+ def run(domain) do+ site_by_domain_q = from s in Plausible.Site, where: s.domain == ^domain++ Multi.new()+ |> Multi.run(:site_id, fn _, _ ->+ {:ok, Repo.one(from s in site_by_domain_q, select: s.id)}+ end)+ |> Multi.delete_all(:delete_all, site_by_domain_q)+ |> Oban.insert(:delayed_metrics_removal, fn %{site_id: site_id} ->+ StatsRemoval.new(%{domain: domain, site_id: site_id},+ schedule_in: stats_deletion_delay_seconds()+ )+ end)+ |> Repo.transaction()+ end+enddiff --git a/lib/plausible_web/controllers/api/external_sites_controller.ex b/lib/plausible_web/controllers/api/external_sites_controller.exindex a754d98faf..57e18e0b97 100644--- a/lib/plausible_web/controllers/api/external_sites_controller.ex+++ b/lib/plausible_web/controllers/api/external_sites_controller.ex@@ -42,7 +42,7 @@ defmodule PlausibleWeb.Api.ExternalSitesController dosite = Sites.get_for_user(conn.assigns[:current_user].id, site_id, [:owner])if site do- Plausible.Purge.delete_site!(site)+ {:ok, _} = Plausible.Site.Removal.run(site.domain)json(conn, %{"deleted" => true})elseH.not_found(conn, "Site could not be found")diff --git a/lib/plausible_web/controllers/auth_controller.ex b/lib/plausible_web/controllers/auth_controller.exindex 51721bbe1c..3ae2de68fd 100644--- a/lib/plausible_web/controllers/auth_controller.ex+++ b/lib/plausible_web/controllers/auth_controller.ex@@ -542,7 +542,7 @@ defmodule PlausibleWeb.AuthController doRepo.delete!(membership)if membership.role == :owner do- Plausible.Purge.delete_site!(membership.site)+ Plausible.Site.Removal.run(membership.site.domain)endenddiff --git a/lib/plausible_web/controllers/site_controller.ex b/lib/plausible_web/controllers/site_controller.exindex b43b7c8c66..145b6fc3c7 100644--- a/lib/plausible_web/controllers/site_controller.ex+++ b/lib/plausible_web/controllers/site_controller.ex@@ -359,10 +359,10 @@ defmodule PlausibleWeb.SiteController dodef delete_site(conn, _params) dosite = conn.assigns[:site]- Plausible.Purge.delete_site!(site)+ Plausible.Site.Removal.run(site.domain)conn- |> put_flash(:success, "Site deleted successfully along with all pageviews")+ |> put_flash(:success, "Your site and page views deletion process has started.")|> redirect(to: "/sites")enddiff --git a/lib/plausible_web/templates/site/settings_danger_zone.html.eex b/lib/plausible_web/templates/site/settings_danger_zone.html.eexindex bf90295131..59d28293d4 100644--- a/lib/plausible_web/templates/site/settings_danger_zone.html.eex+++ b/lib/plausible_web/templates/site/settings_danger_zone.html.eex@@ -38,7 +38,7 @@Delete site</p><p class="text-sm leading-5 text-gray-500 dark:text-gray-200">- Permanently remove all stats and the site configuration too. This process takes a few hours so you won't be able to immediately register the same site again+ Permanently remove all stats and the site configuration too. This process may take up to 48 hours so you won't be able to immediately register the same site again</p></div><%= link "Delete #{@site.domain}", to: "/#{URI.encode_www_form(@site.domain)}", method: :delete, class: "inline-block px-4 py-2 border border-transparent font-medium rounded-md text-red-700 dark:text-red-800 bg-red-100 dark:bg-red-200 hover:bg-red-50 dark:hover:bg-red-300 focus:outline-none focus:border-red-300 focus:ring active:bg-red-200 transition ease-in-out duration-150 sm:text-sm sm:leading-5", data: [confirm: "Deleting the site data cannot be reversed. Are you sure?"] %>diff --git a/lib/workers/stats_removal.ex b/lib/workers/stats_removal.exnew file mode 100644index 0000000000..6ca86ba0d7--- /dev/null+++ b/lib/workers/stats_removal.ex@@ -0,0 +1,59 @@+defmodule Plausible.Workers.StatsRemoval do+ @moduledoc """+ Asynchronous worker firing deletion mutations to clickhouse.+ For now only ALTER TABLE deletions are supported. Experimental+ DELETE FROM support is going to be introduced once production db+ is upgraded.++ At most 3 attempts are made, with 15m backoff value.++ Imported stats tables keep site reference through a numeric id, whilist+ events and sessions store domain as-is - hence two different deletes,+ one of which cannot be performed anymore once the site identifier is permanently+ gone from postgres.+ """+ use Plausible.Repo++ use Oban.Worker,+ queue: :site_stats_removal,+ max_attempts: 3,+ unique: [period: :infinity, fields: [:args]]++ @impl Oban.Worker+ def perform(%{args: args}) do+ domain = Map.fetch!(args, "domain")+ site_id = Map.get(args, "site_id")++ imported_result = delete_imported!(site_id)+ native_result = delete_native!(domain)++ {:ok, Map.merge(imported_result, native_result)}+ end++ @impl Oban.Worker+ def backoff(_job) do+ 15 * 60+ end++ defp delete_imported!(nil) do+ %{}+ end++ defp delete_imported!(id) when is_integer(id) do+ Enum.map(Plausible.Imported.tables(), fn table ->+ sql = "ALTER TABLE #{table} DELETE WHERE site_id = ?"+ {table, Ecto.Adapters.SQL.query!(Plausible.ClickhouseRepo, sql, [id])}+ end)+ |> Enum.into(%{})+ end++ defp delete_native!(domain) do+ events_sql = "ALTER TABLE events DELETE WHERE domain = ?"+ sessions_sql = "ALTER TABLE sessions DELETE WHERE domain = ?"++ %{+ "events" => Ecto.Adapters.SQL.query!(Plausible.ClickhouseRepo, events_sql, [domain]),+ "sessions" => Ecto.Adapters.SQL.query!(Plausible.ClickhouseRepo, sessions_sql, [domain])+ }+ end+enddiff --git a/test/plausible/purge_test.exs b/test/plausible/purge_test.exsindex 1efc98a835..a05fc51e9b 100644--- a/test/plausible/purge_test.exs+++ b/test/plausible/purge_test.exs@@ -20,28 +20,17 @@ defmodule Plausible.PurgeTest do{:ok, %{site: site}}end- defp assert_count(query, expected) do- assert eventually(- fn ->- count = Plausible.ClickhouseRepo.aggregate(query, :count)- {count == expected, count}- end,- 200,- 10- )- end-test "delete_imported_stats!/1 deletes imported data", %{site: site} doEnum.each(Plausible.Imported.tables(), fn table ->query = from(imported in table, where: imported.site_id == ^site.id)- assert_count(query, 1)+ assert await_clickhouse_count(query, 1)end)assert :ok == Plausible.Purge.delete_imported_stats!(site)Enum.each(Plausible.Imported.tables(), fn table ->query = from(imported in table, where: imported.site_id == ^site.id)- assert_count(query, 0)+ assert await_clickhouse_count(query, 0)end)end@@ -52,38 +41,19 @@ defmodule Plausible.PurgeTest dotest "delete_native_stats!/1 deletes native stats", %{site: site} doevents_query = from(s in Plausible.ClickhouseEvent, where: s.domain == ^site.domain)- assert_count(events_query, 1)+ assert await_clickhouse_count(events_query, 1)sessions_query = from(s in Plausible.ClickhouseSession, where: s.domain == ^site.domain)- assert_count(sessions_query, 1)+ assert await_clickhouse_count(sessions_query, 1)assert :ok == Plausible.Purge.delete_native_stats!(site)- assert_count(events_query, 0)- assert_count(sessions_query, 0)+ assert await_clickhouse_count(events_query, 0)+ assert await_clickhouse_count(sessions_query, 0)endtest "delete_native_stats!/1 resets stats_start_date", %{site: site} doassert :ok == Plausible.Purge.delete_native_stats!(site)assert %Plausible.Site{stats_start_date: nil} = Plausible.Repo.reload(site)end-- test "delete_site!/1 deletes the site and all stats", %{site: site} do- events_query = from(s in Plausible.ClickhouseEvent, where: s.domain == ^site.domain)- assert_count(events_query, 1)-- sessions_query = from(s in Plausible.ClickhouseSession, where: s.domain == ^site.domain)- assert_count(sessions_query, 1)-- assert :ok == Plausible.Purge.delete_site!(site)- assert nil == Plausible.Repo.reload(site)-- assert_count(events_query, 0)- assert_count(sessions_query, 0)-- Enum.each(Plausible.Imported.tables(), fn table ->- query = from(imported in table, where: imported.site_id == ^site.id)- assert_count(query, 0)- end)- endenddiff --git a/test/plausible/site/site_removal_test.exs b/test/plausible/site/site_removal_test.exsnew file mode 100644index 0000000000..544acbf17c--- /dev/null+++ b/test/plausible/site/site_removal_test.exs@@ -0,0 +1,116 @@+defmodule Plausible.Site.SiteRemovalTest do+ use Plausible.DataCase, async: true+ use Oban.Testing, repo: Plausible.Repo++ alias Plausible.Site.Removal+ alias Plausible.Sites+ alias Plausible.Workers.StatsRemoval++ describe "execution and scheduling" do+ test "site from postgres is immediately deleted" do+ site = insert(:site)+ assert {:ok, context} = Removal.run(site.domain)+ assert context.delete_all == {1, nil}+ assert context.site_id == site.id+ refute Sites.get_by_domain(site.domain)+ end++ test "deletion is idempotent" do+ assert {:ok, context} = Removal.run("some.example.com")+ assert context.delete_all == {0, nil}+ end++ test "stats deletion job is scheduled when no site exists in postgres" do+ assert {:ok, _} = Removal.run("a.domain.example.com")++ assert_enqueued(+ worker: StatsRemoval,+ args: %{"domain" => "a.domain.example.com", "site_id" => nil}+ )+ end++ test "stats deletion job is scheduled when site exists in postgres" do+ site = insert(:site)+ assert {:ok, _} = Removal.run(site.domain)++ assert_enqueued(+ worker: StatsRemoval,+ args: %{"domain" => site.domain, "site_id" => site.id}+ )+ end++ test "stats deletion is always scheduled ~20m in the future" do+ assert {:ok, _} = Removal.run("foo.example.com")++ in_20m = DateTime.add(DateTime.utc_now(), 1200, :second)++ assert_enqueued(+ worker: StatsRemoval,+ scheduled_at: {in_20m, delta: 5}+ )+ end++ test "stats deletion is always scheduled late enough for sites cache to expire" do+ delay_ms = Removal.stats_deletion_delay_seconds() * 1000+ assert delay_ms > Plausible.Site.Cache.Warmer.interval()+ end+ end++ describe "the background worker" do+ test "the job runs deletes at clickhouse" do+ assert {:ok, %{"events" => r, "sessions" => r}} =+ perform_job(StatsRemoval, %{"domain" => "foo.example.com"})++ assert %Clickhousex.Result{command: :updated} = r++ assert {:ok, %{"events" => r, "sessions" => r, "imported_browsers" => r}} =+ perform_job(StatsRemoval, %{"domain" => "foo.example.com", "site_id" => 777})++ assert %Clickhousex.Result{command: :updated} = r+ end+ end++ describe "integration" do+ setup do+ site = insert(:site, stats_start_date: ~D[2020-01-01])++ populate_stats(site, [+ build(:pageview),+ build(:imported_visitors),+ build(:imported_sources),+ build(:imported_pages),+ build(:imported_entry_pages),+ build(:imported_exit_pages),+ build(:imported_locations),+ build(:imported_devices),+ build(:imported_browsers),+ build(:imported_operating_systems)+ ])++ {:ok, %{site: site}}+ end++ test "the job actually removes stats from clickhouse", %{site: site} do+ Enum.each(Plausible.Imported.tables(), fn table ->+ query = from(imported in table, where: imported.site_id == ^site.id)+ assert await_clickhouse_count(query, 1)+ end)++ events_query = from(s in Plausible.ClickhouseEvent, where: s.domain == ^site.domain)+ assert await_clickhouse_count(events_query, 1)++ sessions_query = from(s in Plausible.ClickhouseSession, where: s.domain == ^site.domain)+ assert await_clickhouse_count(sessions_query, 1)++ perform_job(StatsRemoval, %{"domain" => site.domain, "site_id" => site.id})++ assert await_clickhouse_count(events_query, 0)+ assert await_clickhouse_count(sessions_query, 0)++ Enum.each(Plausible.Imported.tables(), fn table ->+ query = from(imported in table, where: imported.site_id == ^site.id)+ assert await_clickhouse_count(query, 0)+ end)+ end+ end+enddiff --git a/test/support/test_utils.ex b/test/support/test_utils.exindex ed2b41f076..698c4e11bd 100644--- a/test/support/test_utils.ex+++ b/test/support/test_utils.ex@@ -184,4 +184,16 @@ defmodule Plausible.TestUtils doendend)end++ def await_clickhouse_count(query, expected) do+ eventually(+ fn ->+ count = Plausible.ClickhouseRepo.aggregate(query, :count)++ {count == expected, count}+ end,+ 200,+ 10+ )+ endend