Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.
TypeScript analyticsanalytics-dashboardclickhouseelixirgoogle-analyticsmarketingopen-source-analyticsphoenixplausible-analyticspostgresqlprivacyprivacy-friendlyself-hostedsimple-analyticsstatisticstailwindcssweb-analyticswebsite-analyticswebsite-statswebsite-tracking
Add data migration for moving sessions_v2 table to VersionedCollapsingMergeTree (#3802)
Karl-Aksel Puulmann8ba2b934b369
diff --git a/lib/plausible/data_migration.ex b/lib/plausible/data_migration.exindex 7655f56e3d..bc41dba9a3 100644--- a/lib/plausible/data_migration.ex+++ b/lib/plausible/data_migration.ex@@ -68,10 +68,15 @@ defmodule Plausible.DataMigration doenddefp do_run(name, query) do- {:ok, res} = @repo.query(query, [], timeout: :infinity)- IO.puts(" #{IO.ANSI.yellow()}#{name} #{IO.ANSI.green()}Done!#{IO.ANSI.reset()}\n")- IO.puts(String.duplicate("-", 78))- {:ok, res}+ case @repo.query(query, [], timeout: :infinity) do+ {:ok, res} ->+ IO.puts(" #{IO.ANSI.yellow()}#{name} #{IO.ANSI.green()}Done!#{IO.ANSI.reset()}\n")+ IO.puts(String.duplicate("-", 78))+ {:ok, res}++ result ->+ result+ endenddefp unwrap_with_io(name, assigns) dodiff --git a/lib/plausible/data_migration/versioned_sessions.ex b/lib/plausible/data_migration/versioned_sessions.exnew file mode 100644index 0000000000..1c6e182998--- /dev/null+++ b/lib/plausible/data_migration/versioned_sessions.ex@@ -0,0 +1,75 @@+defmodule Plausible.DataMigration.VersionedSessions do+ @moduledoc """+ Sessions CollapsingMergeTree -> VersionedCollapsingMergeTree migration, SQL files available at:+ priv/data_migrations/VersionedSessions/sql+ """+ use Plausible.DataMigration, dir: "VersionedSessions", repo: Plausible.IngestRepo++ @suffix_format "{YYYY}{0M}{0D}{h24}{m}{s}"+ @versioned_table_engines [+ "ReplicatedVersionedCollapsingMergeTree",+ "VersionedCollapsingMergeTree"+ ]++ def run(opts \\ []) do+ run_exchange? = Keyword.get(opts, :run_exchange?, true)++ unique_suffix = Timex.now() |> Timex.format!(@suffix_format)++ cluster? = Plausible.MigrationUtils.clustered_table?("sessions_v2")++ {:ok, %{rows: partitions}} = run_sql("list-partitions")+ partitions = Enum.map(partitions, fn [part] -> part end)++ {:ok, %{rows: [[current_table_engine, table_settings]]}} =+ run_sql("get-sessions-table-settings")++ if Enum.member?(@versioned_table_engines, current_table_engine) do+ IO.puts("sessions_v2 table is already versioned, no migration needed")+ else+ {:ok, _} = run_sql("drop-sessions-tmp-table", cluster?: cluster?)++ {:ok, _} =+ run_sql("create-sessions-tmp-table",+ cluster?: cluster?,+ table_settings: table_settings,+ unique_suffix: unique_suffix+ )++ for partition <- partitions do+ {:ok, _} = run_sql("attach-partition", partition: partition)+ end++ if run_exchange? do+ run_exchange(cluster?)+ end++ IO.puts("Migration done!")+ end+ end++ defp run_exchange(cluster?) do+ case run_sql("exchange-sessions-tables", cluster?: cluster?) do+ {:ok, _} ->+ nil++ # Docker containers don't seem to support EXCHANGE TABLE, hack around this with a non-atomic swap+ {:error, %Ch.Error{code: 1}} ->+ IO.puts("Exchanging sessions_v2 and sessions_v2_tmp_versioned non-atomically")++ {:ok, _} =+ run_sql("rename-table",+ from: "sessions_v2",+ to: "sessions_v2_backup",+ cluster?: cluster?+ )++ {:ok, _} =+ run_sql("rename-table",+ from: "sessions_v2_tmp_versioned",+ to: "sessions_v2",+ cluster?: cluster?+ )+ end+ end+enddiff --git a/lib/plausible/migration_utils.ex b/lib/plausible/migration_utils.exindex a4daa72e22..ac416a389a 100644--- a/lib/plausible/migration_utils.ex+++ b/lib/plausible/migration_utils.ex@@ -4,9 +4,13 @@ defmodule Plausible.MigrationUtils do"""def on_cluster_statement(table) do+ if(clustered_table?(table), do: "ON CLUSTER '{cluster}'", else: "")+ end++ def clustered_table?(table) docase Plausible.IngestRepo.query("SELECT 1 FROM system.replicas WHERE table = '#{table}'") do- {:ok, %{rows: []}} -> ""- {:ok, _} -> "ON CLUSTER '{cluster}'"+ {:ok, %{rows: []}} -> false+ {:ok, _} -> trueendendenddiff --git a/priv/data_migrations/VersionedSessions/sql/attach-partition.sql.eex b/priv/data_migrations/VersionedSessions/sql/attach-partition.sql.eexnew file mode 100644index 0000000000..c1d8cfb5f8--- /dev/null+++ b/priv/data_migrations/VersionedSessions/sql/attach-partition.sql.eex@@ -0,0 +1 @@+ALTER TABLE sessions_v2_tmp_versioned ATTACH PARTITION '<%= @partition %>' FROM sessions_v2diff --git a/priv/data_migrations/VersionedSessions/sql/check-replicas.sql.eex b/priv/data_migrations/VersionedSessions/sql/check-replicas.sql.eexnew file mode 100644index 0000000000..367a574df4--- /dev/null+++ b/priv/data_migrations/VersionedSessions/sql/check-replicas.sql.eex@@ -0,0 +1 @@+SELECT 1 FROM system.replicas WHERE table = 'sessions_v2'diff --git a/priv/data_migrations/VersionedSessions/sql/create-sessions-tmp-table.sql.eex b/priv/data_migrations/VersionedSessions/sql/create-sessions-tmp-table.sql.eexnew file mode 100644index 0000000000..dc579c8587--- /dev/null+++ b/priv/data_migrations/VersionedSessions/sql/create-sessions-tmp-table.sql.eex@@ -0,0 +1,13 @@+CREATE TABLE sessions_v2_tmp_versioned+<%= if @cluster? do %>ON CLUSTER '{cluster}'<% end %>+AS sessions_v2+<%= if @cluster? do %>+ENGINE = ReplicatedVersionedCollapsingMergeTree('/clickhouse/{cluster}/tables/{shard}/plausible_prod/sessions_v2_<%= @unique_suffix %>', '{replica}', sign, events)+<% else %>+ENGINE = VersionedCollapsingMergeTree(sign, events)+<% end %>+PARTITION BY toYYYYMM(start)+PRIMARY KEY (site_id, toDate(start), user_id, session_id)+ORDER BY (site_id, toDate(start), user_id, session_id)+SAMPLE BY user_id+<%= @table_settings %>diff --git a/priv/data_migrations/VersionedSessions/sql/drop-sessions-tmp-table.sql.eex b/priv/data_migrations/VersionedSessions/sql/drop-sessions-tmp-table.sql.eexnew file mode 100644index 0000000000..7a967981a1--- /dev/null+++ b/priv/data_migrations/VersionedSessions/sql/drop-sessions-tmp-table.sql.eex@@ -0,0 +1 @@+DROP TABLE IF EXISTS sessions_v2_tmp_versioned <%= if @cluster? do %>ON CLUSTER '{cluster}'<% end %>diff --git a/priv/data_migrations/VersionedSessions/sql/exchange-sessions-tables.sql.eex b/priv/data_migrations/VersionedSessions/sql/exchange-sessions-tables.sql.eexnew file mode 100644index 0000000000..416da043ed--- /dev/null+++ b/priv/data_migrations/VersionedSessions/sql/exchange-sessions-tables.sql.eex@@ -0,0 +1 @@+EXCHANGE TABLES sessions_v2_tmp_versioned AND sessions_v2 <%= if @cluster? do %>ON CLUSTER '{cluster}'<% end %>diff --git a/priv/data_migrations/VersionedSessions/sql/get-sessions-table-settings.sql.eex b/priv/data_migrations/VersionedSessions/sql/get-sessions-table-settings.sql.eexnew file mode 100644index 0000000000..ffe35fb43b--- /dev/null+++ b/priv/data_migrations/VersionedSessions/sql/get-sessions-table-settings.sql.eex@@ -0,0 +1,4 @@+SELECT engine, extract(engine_full, 'SETTINGS .+')+FROM system.tables+WHERE database = currentDatabase()+ AND name = 'sessions_v2'diff --git a/priv/data_migrations/VersionedSessions/sql/list-partitions.sql.eex b/priv/data_migrations/VersionedSessions/sql/list-partitions.sql.eexnew file mode 100644index 0000000000..9868226881--- /dev/null+++ b/priv/data_migrations/VersionedSessions/sql/list-partitions.sql.eex@@ -0,0 +1,5 @@+SELECT distinct(partition)+FROM system.parts+WHERE database = currentDatabase()+ AND table = 'sessions_v2'+ORDER BY partitiondiff --git a/priv/data_migrations/VersionedSessions/sql/rename-table.sql.eex b/priv/data_migrations/VersionedSessions/sql/rename-table.sql.eexnew file mode 100644index 0000000000..ea07c8d93b--- /dev/null+++ b/priv/data_migrations/VersionedSessions/sql/rename-table.sql.eex@@ -0,0 +1 @@+RENAME TABLE <%= @from %> TO <%= @to %> <%= if @cluster? do %>ON CLUSTER '{cluster}'<% end %>