Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.
TypeScript analyticsanalytics-dashboardclickhouseelixirgoogle-analyticsmarketingopen-source-analyticsphoenixplausible-analyticspostgresqlprivacyprivacy-friendlyself-hostedsimple-analyticsstatisticstailwindcssweb-analyticswebsite-analyticswebsite-statswebsite-tracking
Fixes to periodic sites clean up (#6601)
Adamf4d99fd0b11c
diff --git a/lib/plausible/telemetry/plausible_metrics.ex b/lib/plausible/telemetry/plausible_metrics.exindex 3e2ca772f7..524e94f941 100644--- a/lib/plausible/telemetry/plausible_metrics.ex+++ b/lib/plausible/telemetry/plausible_metrics.ex@@ -237,8 +237,9 @@ defmodule Plausible.PromEx.Plugins.PlausibleMetrics do),last_value(metric_prefix ++ [:clickhouse_clean_sites, :run, :partitions_count],- event_name: Plausible.Workers.ClickhouseCleanSites.telemetry_run_event(),- measurement: :partitions_count+ event_name: Plausible.Workers.ClickhouseCleanSites.telemetry_partitions_event(),+ measurement: :count,+ tags: [:table])]|> Enum.concat(persistor_metrics(metric_prefix))diff --git a/lib/workers/clickhouse_clean_sites.ex b/lib/workers/clickhouse_clean_sites.exindex 234f80a467..aa7a093829 100644--- a/lib/workers/clickhouse_clean_sites.ex+++ b/lib/workers/clickhouse_clean_sites.ex@@ -37,11 +37,16 @@ defmodule Plausible.Workers.ClickhouseCleanSites do# - fall back to a mutation which rebuilds the projection.@mutation_only_tables ["ingest_counters"]- @settings if Mix.env() in [:test, :ce_test, :e2e_test], do: [mutations_sync: 2], else: []+ @settings if Mix.env() in [:test, :ce_test, :e2e_test],+ do: [mutations_sync: 2, lightweight_deletes_sync: 2],+ else: [mutations_sync: 0, lightweight_deletes_sync: 0]@spec telemetry_run_event() :: [atom()]def telemetry_run_event(), do: [:plausible, :clickhouse_clean_sites, :run]+ @spec telemetry_partitions_event() :: [atom()]+ def telemetry_partitions_event(), do: [:plausible, :clickhouse_clean_sites, :partitions]+@spec telemetry_stage_duration() :: [atom()]def telemetry_stage_duration(), do: [:plausible, :clickhouse_clean_sites, :stage]@@ -59,11 +64,19 @@ defmodule Plausible.Workers.ClickhouseCleanSites dopartition_ids("sessions_v2", site_ids)end)- :telemetry.execute(telemetry_run_event(), %{- sites_count: length(site_ids),- partitions_count_events: length(partition_ids_events),- partitions_count_sessions: length(partition_ids_sessions)- })+ :telemetry.execute(telemetry_run_event(), %{sites_count: length(site_ids)})++ :telemetry.execute(+ telemetry_partitions_event(),+ %{count: length(partition_ids_events)},+ %{table: "events_v2"}+ )++ :telemetry.execute(+ telemetry_partitions_event(),+ %{count: length(partition_ids_sessions)},+ %{table: "sessions_v2"}+ )Logger.warning("Clearing ClickHouse data for #{length(site_ids)} sites across #{length(partition_ids_events)}/#{length(partition_ids_sessions)} partitions"diff --git a/test/workers/clickhouse_clean_sites_test.exs b/test/workers/clickhouse_clean_sites_test.exsindex 5106b7735d..560dde4741 100644--- a/test/workers/clickhouse_clean_sites_test.exs+++ b/test/workers/clickhouse_clean_sites_test.exs@@ -81,11 +81,12 @@ defmodule Plausible.Workers.ClickhouseCleanSitesTest dotest_pid = self()telemetry_run = ClickhouseCleanSites.telemetry_run_event()+ telemetry_partitions = ClickhouseCleanSites.telemetry_partitions_event()telemetry_stage = ClickhouseCleanSites.telemetry_stage_duration():telemetry.attach_many("#{test}-telemetry-handler",- [telemetry_run, telemetry_stage],+ [telemetry_run, telemetry_partitions, telemetry_stage],fn event, measurements, metadata, _ ->send(test_pid, {:telemetry_handled, event, measurements, metadata})end,@@ -112,10 +113,13 @@ defmodule Plausible.Workers.ClickhouseCleanSitesTest doassert_receive {:telemetry_handled, ^telemetry_stage, %{duration: _},%{stage: "get_partition_ids_sessions"}}+ assert_receive {:telemetry_handled, ^telemetry_run, %{sites_count: 1}, %{}}+# only January and March actually have data, February skipped- assert_receive {:telemetry_handled, ^telemetry_run,- %{sites_count: 1, partitions_count_events: 2, partitions_count_sessions: 2},- %{}}+ assert_receive {:telemetry_handled, ^telemetry_partitions, %{count: 2}, %{table: "events_v2"}}++ assert_receive {:telemetry_handled, ^telemetry_partitions, %{count: 2},+ %{table: "sessions_v2"}}assert_receive {:telemetry_handled, ^telemetry_stage, %{duration: _},%{stage: "events_deletion"}}