Skip to content

Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.

Disable deduplication for improted_* tables in CH (#3429)

Adrian Gruntkowski4cdf843adecc
diff --git a/priv/ingest_repo/migrations/20231017073642_disable_deduplication_window_for_imports.exs b/priv/ingest_repo/migrations/20231017073642_disable_deduplication_window_for_imports.exs
new file mode 100644
index 0000000000..944a900dfc
--- /dev/null
+++ b/priv/ingest_repo/migrations/20231017073642_disable_deduplication_window_for_imports.exs
@@ -0,0 +1,34 @@
+defmodule Plausible.IngestRepo.Migrations.DisableDeduplicationWindowForImports do
+ use Ecto.Migration
+
+ @import_tables ~w(
+ imported_visitors
+ imported_sources
+ imported_pages
+ imported_entry_pages
+ imported_exit_pages
+ imported_locations
+ imported_devices
+ imported_browsers
+ imported_operating_systems
+ )
+
+ def change do
+ cluster_query = "SELECT 1 FROM system.replicas WHERE table = 'imported_visitors'"
+
+ cluster? =
+ case Ecto.Adapters.SQL.query(Plausible.IngestRepo, cluster_query) |> IO.inspect() do
+ {:ok, %{rows: []}} -> false
+ {:ok, _} -> true
+ end
+
+ for table <- @import_tables do
+ execute """
+ ALTER TABLE #{table} #{if cluster?, do: "ON CLUSTER '{cluster}'"} MODIFY SETTING replicated_deduplication_window = 0
+ """,
+ """
+ ALTER TABLE #{table} #{if cluster?, do: "ON CLUSTER '{cluster}'"} MODIFY SETTING replicated_deduplication_window = 100
+ """
+ end
+ end
+end