Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.
TypeScript analyticsanalytics-dashboardclickhouseelixirgoogle-analyticsmarketingopen-source-analyticsphoenixplausible-analyticspostgresqlprivacyprivacy-friendlyself-hostedsimple-analyticsstatisticstailwindcssweb-analyticswebsite-analyticswebsite-statswebsite-tracking
Add script for purging replayed sessions and events by ID and date range (#6556)
Adrian Gruntkowski48f03c7ee5a6
diff --git a/lib/plausible/data_migration/purge_ingest_replay.ex b/lib/plausible/data_migration/purge_ingest_replay.exnew file mode 100644index 0000000000..f25515552a--- /dev/null+++ b/lib/plausible/data_migration/purge_ingest_replay.ex@@ -0,0 +1,123 @@+defmodule Plausible.DataMigration.PurgeIngestReplay do+ @moduledoc """+ Script purging ingest replay.++ The script is used in the ingest replay procedure when a particular replay turns+ out to be incorrect - for instance, due to too much overlap with normally ingestged+ events.++ Three options are required:++ - `session_id` - `replay_session_id` to purge, provided as an integer+ - `from` and `to` - provided as Dates; necessary for reducing scope of database scan;+ the date range is inclusive; to scan a single day, `from` and `to` can be the same;+ maximum range is 3 days+ """++ alias Plausible.IngestRepo++ @settings if Mix.env() in [:test, :ce_test, :e2e_test], do: [mutations_sync: 2], else: []++ @confirmation_phrase "REMOVE REPLAYED INGEST"++ @count_query_sessions """+ SELECT count(*) AS count FROM sessions_v2+ WHERE replay_session_id = {$0:UInt64}+ AND toDate(start) >= {$1:Date}+ AND toDate(start) <= {$2:Date}+ """++ @count_query_events """+ SELECT count(*) AS count FROM events_v2+ WHERE replay_session_id = {$0:UInt64}+ AND toDate(timestamp) >= {$1:Date}+ AND toDate(timestamp) <= {$2:Date}+ """++ @purge_query_sessions """+ ALTER TABLE sessions_v2 DELETE+ WHERE replay_session_id = {$0:UInt64}+ AND toDate(start) >= {$1:Date}+ AND toDate(start) <= {$2:Date}+ """++ @purge_query_events """+ ALTER TABLE events_v2 DELETE+ WHERE replay_session_id = {$0:UInt64}+ AND toDate(timestamp) >= {$1:Date}+ AND toDate(timestamp) <= {$2:Date}+ """++ def run(opts \\ []) do+ from = %Date{} = Keyword.fetch!(opts, :from)+ to = %Date{} = Keyword.fetch!(opts, :to)+ session_id = Keyword.fetch!(opts, :session_id)++ days_scope = Date.diff(to, from)++ if not is_integer(session_id) or session_id <= 0 do+ raise ArgumentError, "Invalid session ID"+ end++ if days_scope < 0 or days_scope > 2 do+ raise ArgumentError,+ "The period declared by inclusive from/to range cannot be longer than 3 days"+ end++ %{rows: [[session_count]]} =+ IngestRepo.query!(+ @count_query_sessions,+ [session_id, from, to],+ @settings+ )++ %{rows: [[event_count]]} =+ IngestRepo.query!(@count_query_events, [session_id, from, to], @settings)++ if session_count > 0 or event_count > 0 do+ IO.puts("""+ About to remove replayed ingest for session ID #{session_id}++ Start date: #{from}+ End date: #{to}++ Sessions found: #{session_count}+ Events found: #{event_count}++ To confirm, please type in:++ #{@confirmation_phrase}++ and hit Enter.++ """)++ confirmation = IO.gets("Confirmation: ")++ if String.trim(confirmation) == @confirmation_phrase do+ run_purge(session_id, from, to)+ else+ IO.puts("Wrong confirmation phrase. Aborting!")+ {:error, :aborted}+ end+ else+ IO.puts("No events found matching criteria. Aborting.")+ {:error, :aborted}+ end+ end++ defp run_purge(session_id, from, to) do+ IO.puts("Purging sessions...")++ IngestRepo.query!(+ @purge_query_sessions,+ [session_id, from, to],+ @settings+ )++ IO.puts("Purging events...")+ IngestRepo.query!(@purge_query_events, [session_id, from, to], @settings)++ IO.puts("Done!")+ end+enddiff --git a/test/plausible/data_migration/purge_ingest_replay_test.exs b/test/plausible/data_migration/purge_ingest_replay_test.exsnew file mode 100644index 0000000000..a12243ed84--- /dev/null+++ b/test/plausible/data_migration/purge_ingest_replay_test.exs@@ -0,0 +1,191 @@+defmodule Plausible.DataMigration.PurgeIngestReplayTest do+ use Plausible++ on_ee do+ use Plausible.DataCase, async: true++ import ExUnit.CaptureIO++ alias Plausible.DataMigration.PurgeIngestReplay++ describe "run/1" do+ test "aborts where there are no replayed events within range" do+ assert capture_io(fn ->+ assert {:error, :aborted} =+ PurgeIngestReplay.run(+ from: ~D[2026-07-26],+ to: ~D[2026-07-27],+ session_id: 123+ )+ end) =~ "No events found matching criteria. Aborting"+ end++ test "purges replayed events in scope" do+ site = new_site()++ replay_session_id = random_id()+ other_replay_session_id = random_id()++ populate_stats(site, [+ # out of scope+ build(:pageview,+ timestamp: ~N[2026-07-25 11:44:56],+ replay_session_id: replay_session_id+ ),+ # different replay+ build(:pageview,+ timestamp: ~N[2026-07-25 11:44:56],+ replay_session_id: other_replay_session_id+ ),+ # non-replay event+ build(:pageview, timestamp: ~N[2026-07-25 11:44:56]),+ # in scope+ build(:pageview,+ timestamp: ~N[2026-07-26 11:44:56],+ replay_session_id: replay_session_id+ ),+ build(:pageview,+ user_id: 456,+ timestamp: ~N[2026-07-26 13:12:33],+ replay_session_id: replay_session_id+ ),+ build(:pageview,+ user_id: 456,+ timestamp: ~N[2026-07-26 13:32:43],+ replay_session_id: replay_session_id+ ),+ build(:pageview,+ timestamp: ~N[2026-07-27 14:44:56],+ replay_session_id: replay_session_id+ )+ ])++ output =+ capture_io("REMOVE REPLAYED INGEST", fn ->+ assert :ok =+ PurgeIngestReplay.run(+ from: ~D[2026-07-26],+ to: ~D[2026-07-27],+ session_id: replay_session_id+ )+ end)++ assert output =~ "About to remove replayed ingest for session ID #{replay_session_id}"+ assert output =~ "Start date: 2026-07-26"+ assert output =~ "End date: 2026-07-27"+ assert output =~ "Sessions found: 3"+ assert output =~ "Events found: 4"+ assert output =~ "Purging sessions..."+ assert output =~ "Purging events..."+ assert output =~ "Done!"++ assert eventually(fn ->+ assert %{rows: [[session_count]]} =+ Plausible.IngestRepo.query!(+ "SELECT count(*) FROM events_v2 WHERE replay_session_id = #{replay_session_id}"+ )++ assert %{rows: [[event_count]]} =+ Plausible.IngestRepo.query!(+ "SELECT count(*) FROM sessions_v2 WHERE replay_session_id = #{replay_session_id}"+ )++ # only out of scope event is left+ {session_count == 1 and event_count == 1, true}+ end)+ end++ test "aborts when no matching replay events are in scope" do+ site = new_site()++ replay_session_id = random_id()+ other_replay_session_id = random_id()++ populate_stats(site, [+ # out of scope+ build(:pageview,+ timestamp: ~N[2026-07-25 11:44:56],+ replay_session_id: replay_session_id+ ),+ # different replay+ build(:pageview,+ timestamp: ~N[2026-07-25 11:44:56],+ replay_session_id: other_replay_session_id+ ),+ # non-replay event+ build(:pageview, timestamp: ~N[2026-07-25 11:44:56])+ ])++ assert capture_io(fn ->+ assert {:error, :aborted} =+ PurgeIngestReplay.run(+ from: ~D[2026-07-26],+ to: ~D[2026-07-27],+ session_id: replay_session_id+ )+ end) =~ "No events found matching criteria. Aborting"+ end++ test "raises on invalid session ID" do+ assert_raise ArgumentError, ~r/Invalid session ID/, fn ->+ PurgeIngestReplay.run(+ from: ~D[2026-07-26],+ to: ~D[2026-07-27],+ session_id: 0+ )++ assert_raise MatchError, fn ->+ PurgeIngestReplay.run(+ from: ~D[2026-07-26],+ to: ~D[2026-07-27],+ session_id: :invalid+ )+ end+ end+ end++ test "raises on invalid session date range" do+ assert_raise ArgumentError,+ ~r/The period declared by inclusive from\/to range cannot be longer than 3 days/,+ fn ->+ PurgeIngestReplay.run(+ from: ~D[2026-07-27],+ to: ~D[2026-07-26],+ session_id: 123+ )+ end++ assert_raise ArgumentError,+ ~r/The period declared by inclusive from\/to range cannot be longer than 3 days/,+ fn ->+ PurgeIngestReplay.run(+ from: ~D[2026-07-26],+ to: ~D[2026-07-29],+ session_id: 123+ )+ end++ assert_raise MatchError, fn ->+ PurgeIngestReplay.run(+ from: ~D[2026-07-26],+ to: :invalid,+ session_id: 123+ )+ end++ assert_raise MatchError, fn ->+ PurgeIngestReplay.run(+ from: :invalid,+ to: ~D[2026-07-29],+ session_id: 123+ )+ end+ end+ end++ defp random_id() do+ :crypto.strong_rand_bytes(8)+ |> :binary.decode_unsigned()+ end+ end+end