Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.
TypeScript analyticsanalytics-dashboardclickhouseelixirgoogle-analyticsmarketingopen-source-analyticsphoenixplausible-analyticspostgresqlprivacyprivacy-friendlyself-hostedsimple-analyticsstatisticstailwindcssweb-analyticswebsite-analyticswebsite-statswebsite-tracking
Worker to clean site data from ClickHouse (#3959)
Karl-Aksel Puulmanna6d4786959e2
diff --git a/lib/workers/clickhouse_clean_sites.ex b/lib/workers/clickhouse_clean_sites.exnew file mode 100644index 0000000000..974dd0d9dc--- /dev/null+++ b/lib/workers/clickhouse_clean_sites.ex@@ -0,0 +1,68 @@+defmodule Plausible.Workers.ClickhouseCleanSites do+ @moduledoc """+ Cleans deleted site data from ClickHouse asynchronously.++ We batch up data deletions from ClickHouse as deleting a single site is+ just as expensive as deleting many.+ """++ use Plausible.Repo+ use Plausible.ClickhouseRepo+ use Plausible.IngestRepo+ use Oban.Worker, queue: :clickhouse_clean_sites++ import Ecto.Query++ require Logger++ @tables_to_clear [+ "events_v2",+ "sessions_v2",+ "ingest_counters",+ "imported_browsers",+ "imported_devices",+ "imported_entry_pages",+ "imported_exit_pages",+ "imported_locations",+ "imported_operating_systems",+ "imported_pages",+ "imported_sources",+ "imported_visitors"+ ]++ @settings if Mix.env() in [:test, :small_test], do: [mutations_sync: 2], else: []++ def perform(_job) do+ deleted_sites = get_deleted_sites_with_clickhouse_data()++ if not Enum.empty?(deleted_sites) do+ Logger.info(+ "Clearing ClickHouse data for the following #{length(deleted_sites)} sites which have been deleted: #{inspect(deleted_sites)}"+ )++ for table <- @tables_to_clear do+ IngestRepo.query!(+ "ALTER TABLE {$0:Identifier} DELETE WHERE site_id IN {$1:Array(UInt64)}",+ [table, deleted_sites],+ settings: @settings+ )+ end+ end++ :ok+ end++ def get_deleted_sites_with_clickhouse_data() do+ pg_sites =+ from(s in Plausible.Site, select: s.id)+ |> Plausible.Repo.all()+ |> MapSet.new()++ ch_sites =+ from(e in "events_v2", group_by: e.site_id, select: e.site_id)+ |> Plausible.ClickhouseRepo.all(timeout: :infinity)+ |> MapSet.new()++ MapSet.difference(ch_sites, pg_sites) |> MapSet.to_list()+ end+enddiff --git a/test/workers/clickhouse_clean_sites_test.exs b/test/workers/clickhouse_clean_sites_test.exsnew file mode 100644index 0000000000..8d85dee2c1--- /dev/null+++ b/test/workers/clickhouse_clean_sites_test.exs@@ -0,0 +1,69 @@+defmodule Plausible.Workers.ClickhouseCleanSitesTest do+ use Plausible.DataCase+ use Plausible.TestUtils+ use Plausible+ import Plausible.Factory++ alias Plausible.Workers.ClickhouseCleanSites++ test "deletes data from events and sessions tables" do+ site = insert(:site)+ deleted_site = insert(:site)++ populate_stats(site, [+ build(:pageview)+ ])++ populate_stats(deleted_site, [+ build(:pageview),+ build(:pageview),+ build(:imported_visitors),+ build(:imported_sources),+ build(:imported_pages),+ build(:imported_entry_pages),+ build(:imported_exit_pages),+ build(:imported_locations),+ build(:imported_devices),+ build(:imported_browsers),+ build(:imported_operating_systems)+ ])++ Repo.delete!(deleted_site)++ assert Enum.member?(+ ClickhouseCleanSites.get_deleted_sites_with_clickhouse_data(),+ deleted_site.id+ )++ assert not Enum.member?(+ ClickhouseCleanSites.get_deleted_sites_with_clickhouse_data(),+ site.id+ )++ ClickhouseCleanSites.perform(nil)++ assert_count(deleted_site, "events_v2", 0)+ assert_count(deleted_site, "sessions_v2", 0)+ assert_count(deleted_site, "imported_visitors", 0)+ assert_count(deleted_site, "imported_sources", 0)+ assert_count(deleted_site, "imported_pages", 0)+ assert_count(deleted_site, "imported_entry_pages", 0)+ assert_count(deleted_site, "imported_exit_pages", 0)+ assert_count(deleted_site, "imported_locations", 0)+ assert_count(deleted_site, "imported_devices", 0)+ assert_count(deleted_site, "imported_browsers", 0)+ assert_count(deleted_site, "imported_operating_systems", 0)+ assert_count(site, "events_v2", 1)+ assert_count(site, "sessions_v2", 1)++ assert not Enum.member?(+ ClickhouseCleanSites.get_deleted_sites_with_clickhouse_data(),+ deleted_site.id+ )+ end++ def assert_count(site, table, expected_count) do+ q = from(e in table, select: %{count: fragment("count()")}, where: e.site_id == ^site.id)+ await_clickhouse_count(q, expected_count)+ end+end