Skip to content

Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.

Fixes to periodic sites clean up (#6601)

Adamf4d99fd0b11c
diff --git a/lib/plausible/telemetry/plausible_metrics.ex b/lib/plausible/telemetry/plausible_metrics.ex
index 3e2ca772f7..524e94f941 100644
--- a/lib/plausible/telemetry/plausible_metrics.ex
+++ b/lib/plausible/telemetry/plausible_metrics.ex
@@ -237,8 +237,9 @@ defmodule Plausible.PromEx.Plugins.PlausibleMetrics do
),
last_value(
metric_prefix ++ [:clickhouse_clean_sites, :run, :partitions_count],
- event_name: Plausible.Workers.ClickhouseCleanSites.telemetry_run_event(),
- measurement: :partitions_count
+ event_name: Plausible.Workers.ClickhouseCleanSites.telemetry_partitions_event(),
+ measurement: :count,
+ tags: [:table]
)
]
|> Enum.concat(persistor_metrics(metric_prefix))
diff --git a/lib/workers/clickhouse_clean_sites.ex b/lib/workers/clickhouse_clean_sites.ex
index 234f80a467..aa7a093829 100644
--- a/lib/workers/clickhouse_clean_sites.ex
+++ b/lib/workers/clickhouse_clean_sites.ex
@@ -37,11 +37,16 @@ defmodule Plausible.Workers.ClickhouseCleanSites do
# - fall back to a mutation which rebuilds the projection.
@mutation_only_tables ["ingest_counters"]
- @settings if Mix.env() in [:test, :ce_test, :e2e_test], do: [mutations_sync: 2], else: []
+ @settings if Mix.env() in [:test, :ce_test, :e2e_test],
+ do: [mutations_sync: 2, lightweight_deletes_sync: 2],
+ else: [mutations_sync: 0, lightweight_deletes_sync: 0]
@spec telemetry_run_event() :: [atom()]
def telemetry_run_event(), do: [:plausible, :clickhouse_clean_sites, :run]
+ @spec telemetry_partitions_event() :: [atom()]
+ def telemetry_partitions_event(), do: [:plausible, :clickhouse_clean_sites, :partitions]
+
@spec telemetry_stage_duration() :: [atom()]
def telemetry_stage_duration(), do: [:plausible, :clickhouse_clean_sites, :stage]
@@ -59,11 +64,19 @@ defmodule Plausible.Workers.ClickhouseCleanSites do
partition_ids("sessions_v2", site_ids)
end)
- :telemetry.execute(telemetry_run_event(), %{
- sites_count: length(site_ids),
- partitions_count_events: length(partition_ids_events),
- partitions_count_sessions: length(partition_ids_sessions)
- })
+ :telemetry.execute(telemetry_run_event(), %{sites_count: length(site_ids)})
+
+ :telemetry.execute(
+ telemetry_partitions_event(),
+ %{count: length(partition_ids_events)},
+ %{table: "events_v2"}
+ )
+
+ :telemetry.execute(
+ telemetry_partitions_event(),
+ %{count: length(partition_ids_sessions)},
+ %{table: "sessions_v2"}
+ )
Logger.warning(
"Clearing ClickHouse data for #{length(site_ids)} sites across #{length(partition_ids_events)}/#{length(partition_ids_sessions)} partitions"
diff --git a/test/workers/clickhouse_clean_sites_test.exs b/test/workers/clickhouse_clean_sites_test.exs
index 5106b7735d..560dde4741 100644
--- a/test/workers/clickhouse_clean_sites_test.exs
+++ b/test/workers/clickhouse_clean_sites_test.exs
@@ -81,11 +81,12 @@ defmodule Plausible.Workers.ClickhouseCleanSitesTest do
test_pid = self()
telemetry_run = ClickhouseCleanSites.telemetry_run_event()
+ telemetry_partitions = ClickhouseCleanSites.telemetry_partitions_event()
telemetry_stage = ClickhouseCleanSites.telemetry_stage_duration()
:telemetry.attach_many(
"#{test}-telemetry-handler",
- [telemetry_run, telemetry_stage],
+ [telemetry_run, telemetry_partitions, telemetry_stage],
fn event, measurements, metadata, _ ->
send(test_pid, {:telemetry_handled, event, measurements, metadata})
end,
@@ -112,10 +113,13 @@ defmodule Plausible.Workers.ClickhouseCleanSitesTest do
assert_receive {:telemetry_handled, ^telemetry_stage, %{duration: _},
%{stage: "get_partition_ids_sessions"}}
+ assert_receive {:telemetry_handled, ^telemetry_run, %{sites_count: 1}, %{}}
+
# only January and March actually have data, February skipped
- assert_receive {:telemetry_handled, ^telemetry_run,
- %{sites_count: 1, partitions_count_events: 2, partitions_count_sessions: 2},
- %{}}
+ assert_receive {:telemetry_handled, ^telemetry_partitions, %{count: 2}, %{table: "events_v2"}}
+
+ assert_receive {:telemetry_handled, ^telemetry_partitions, %{count: 2},
+ %{table: "sessions_v2"}}
assert_receive {:telemetry_handled, ^telemetry_stage, %{duration: _},
%{stage: "events_deletion"}}