Skip to content

Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.

Add data migration for moving sessions_v2 table to VersionedCollapsingMergeTree (#3802)

Karl-Aksel Puulmann8ba2b934b369
diff --git a/lib/plausible/data_migration.ex b/lib/plausible/data_migration.ex
index 7655f56e3d..bc41dba9a3 100644
--- a/lib/plausible/data_migration.ex
+++ b/lib/plausible/data_migration.ex
@@ -68,10 +68,15 @@ defmodule Plausible.DataMigration do
end
defp do_run(name, query) do
- {:ok, res} = @repo.query(query, [], timeout: :infinity)
- IO.puts(" #{IO.ANSI.yellow()}#{name} #{IO.ANSI.green()}Done!#{IO.ANSI.reset()}\n")
- IO.puts(String.duplicate("-", 78))
- {:ok, res}
+ case @repo.query(query, [], timeout: :infinity) do
+ {:ok, res} ->
+ IO.puts(" #{IO.ANSI.yellow()}#{name} #{IO.ANSI.green()}Done!#{IO.ANSI.reset()}\n")
+ IO.puts(String.duplicate("-", 78))
+ {:ok, res}
+
+ result ->
+ result
+ end
end
defp unwrap_with_io(name, assigns) do
diff --git a/lib/plausible/data_migration/versioned_sessions.ex b/lib/plausible/data_migration/versioned_sessions.ex
new file mode 100644
index 0000000000..1c6e182998
--- /dev/null
+++ b/lib/plausible/data_migration/versioned_sessions.ex
@@ -0,0 +1,75 @@
+defmodule Plausible.DataMigration.VersionedSessions do
+ @moduledoc """
+ Sessions CollapsingMergeTree -> VersionedCollapsingMergeTree migration, SQL files available at:
+ priv/data_migrations/VersionedSessions/sql
+ """
+ use Plausible.DataMigration, dir: "VersionedSessions", repo: Plausible.IngestRepo
+
+ @suffix_format "{YYYY}{0M}{0D}{h24}{m}{s}"
+ @versioned_table_engines [
+ "ReplicatedVersionedCollapsingMergeTree",
+ "VersionedCollapsingMergeTree"
+ ]
+
+ def run(opts \\ []) do
+ run_exchange? = Keyword.get(opts, :run_exchange?, true)
+
+ unique_suffix = Timex.now() |> Timex.format!(@suffix_format)
+
+ cluster? = Plausible.MigrationUtils.clustered_table?("sessions_v2")
+
+ {:ok, %{rows: partitions}} = run_sql("list-partitions")
+ partitions = Enum.map(partitions, fn [part] -> part end)
+
+ {:ok, %{rows: [[current_table_engine, table_settings]]}} =
+ run_sql("get-sessions-table-settings")
+
+ if Enum.member?(@versioned_table_engines, current_table_engine) do
+ IO.puts("sessions_v2 table is already versioned, no migration needed")
+ else
+ {:ok, _} = run_sql("drop-sessions-tmp-table", cluster?: cluster?)
+
+ {:ok, _} =
+ run_sql("create-sessions-tmp-table",
+ cluster?: cluster?,
+ table_settings: table_settings,
+ unique_suffix: unique_suffix
+ )
+
+ for partition <- partitions do
+ {:ok, _} = run_sql("attach-partition", partition: partition)
+ end
+
+ if run_exchange? do
+ run_exchange(cluster?)
+ end
+
+ IO.puts("Migration done!")
+ end
+ end
+
+ defp run_exchange(cluster?) do
+ case run_sql("exchange-sessions-tables", cluster?: cluster?) do
+ {:ok, _} ->
+ nil
+
+ # Docker containers don't seem to support EXCHANGE TABLE, hack around this with a non-atomic swap
+ {:error, %Ch.Error{code: 1}} ->
+ IO.puts("Exchanging sessions_v2 and sessions_v2_tmp_versioned non-atomically")
+
+ {:ok, _} =
+ run_sql("rename-table",
+ from: "sessions_v2",
+ to: "sessions_v2_backup",
+ cluster?: cluster?
+ )
+
+ {:ok, _} =
+ run_sql("rename-table",
+ from: "sessions_v2_tmp_versioned",
+ to: "sessions_v2",
+ cluster?: cluster?
+ )
+ end
+ end
+end
diff --git a/lib/plausible/migration_utils.ex b/lib/plausible/migration_utils.ex
index a4daa72e22..ac416a389a 100644
--- a/lib/plausible/migration_utils.ex
+++ b/lib/plausible/migration_utils.ex
@@ -4,9 +4,13 @@ defmodule Plausible.MigrationUtils do
"""
def on_cluster_statement(table) do
+ if(clustered_table?(table), do: "ON CLUSTER '{cluster}'", else: "")
+ end
+
+ def clustered_table?(table) do
case Plausible.IngestRepo.query("SELECT 1 FROM system.replicas WHERE table = '#{table}'") do
- {:ok, %{rows: []}} -> ""
- {:ok, _} -> "ON CLUSTER '{cluster}'"
+ {:ok, %{rows: []}} -> false
+ {:ok, _} -> true
end
end
end
diff --git a/priv/data_migrations/VersionedSessions/sql/attach-partition.sql.eex b/priv/data_migrations/VersionedSessions/sql/attach-partition.sql.eex
new file mode 100644
index 0000000000..c1d8cfb5f8
--- /dev/null
+++ b/priv/data_migrations/VersionedSessions/sql/attach-partition.sql.eex
@@ -0,0 +1 @@
+ALTER TABLE sessions_v2_tmp_versioned ATTACH PARTITION '<%= @partition %>' FROM sessions_v2
diff --git a/priv/data_migrations/VersionedSessions/sql/check-replicas.sql.eex b/priv/data_migrations/VersionedSessions/sql/check-replicas.sql.eex
new file mode 100644
index 0000000000..367a574df4
--- /dev/null
+++ b/priv/data_migrations/VersionedSessions/sql/check-replicas.sql.eex
@@ -0,0 +1 @@
+SELECT 1 FROM system.replicas WHERE table = 'sessions_v2'
diff --git a/priv/data_migrations/VersionedSessions/sql/create-sessions-tmp-table.sql.eex b/priv/data_migrations/VersionedSessions/sql/create-sessions-tmp-table.sql.eex
new file mode 100644
index 0000000000..dc579c8587
--- /dev/null
+++ b/priv/data_migrations/VersionedSessions/sql/create-sessions-tmp-table.sql.eex
@@ -0,0 +1,13 @@
+CREATE TABLE sessions_v2_tmp_versioned
+<%= if @cluster? do %>ON CLUSTER '{cluster}'<% end %>
+AS sessions_v2
+<%= if @cluster? do %>
+ENGINE = ReplicatedVersionedCollapsingMergeTree('/clickhouse/{cluster}/tables/{shard}/plausible_prod/sessions_v2_<%= @unique_suffix %>', '{replica}', sign, events)
+<% else %>
+ENGINE = VersionedCollapsingMergeTree(sign, events)
+<% end %>
+PARTITION BY toYYYYMM(start)
+PRIMARY KEY (site_id, toDate(start), user_id, session_id)
+ORDER BY (site_id, toDate(start), user_id, session_id)
+SAMPLE BY user_id
+<%= @table_settings %>
diff --git a/priv/data_migrations/VersionedSessions/sql/drop-sessions-tmp-table.sql.eex b/priv/data_migrations/VersionedSessions/sql/drop-sessions-tmp-table.sql.eex
new file mode 100644
index 0000000000..7a967981a1
--- /dev/null
+++ b/priv/data_migrations/VersionedSessions/sql/drop-sessions-tmp-table.sql.eex
@@ -0,0 +1 @@
+DROP TABLE IF EXISTS sessions_v2_tmp_versioned <%= if @cluster? do %>ON CLUSTER '{cluster}'<% end %>
diff --git a/priv/data_migrations/VersionedSessions/sql/exchange-sessions-tables.sql.eex b/priv/data_migrations/VersionedSessions/sql/exchange-sessions-tables.sql.eex
new file mode 100644
index 0000000000..416da043ed
--- /dev/null
+++ b/priv/data_migrations/VersionedSessions/sql/exchange-sessions-tables.sql.eex
@@ -0,0 +1 @@
+EXCHANGE TABLES sessions_v2_tmp_versioned AND sessions_v2 <%= if @cluster? do %>ON CLUSTER '{cluster}'<% end %>
diff --git a/priv/data_migrations/VersionedSessions/sql/get-sessions-table-settings.sql.eex b/priv/data_migrations/VersionedSessions/sql/get-sessions-table-settings.sql.eex
new file mode 100644
index 0000000000..ffe35fb43b
--- /dev/null
+++ b/priv/data_migrations/VersionedSessions/sql/get-sessions-table-settings.sql.eex
@@ -0,0 +1,4 @@
+SELECT engine, extract(engine_full, 'SETTINGS .+')
+FROM system.tables
+WHERE database = currentDatabase()
+ AND name = 'sessions_v2'
diff --git a/priv/data_migrations/VersionedSessions/sql/list-partitions.sql.eex b/priv/data_migrations/VersionedSessions/sql/list-partitions.sql.eex
new file mode 100644
index 0000000000..9868226881
--- /dev/null
+++ b/priv/data_migrations/VersionedSessions/sql/list-partitions.sql.eex
@@ -0,0 +1,5 @@
+SELECT distinct(partition)
+FROM system.parts
+WHERE database = currentDatabase()
+ AND table = 'sessions_v2'
+ORDER BY partition
diff --git a/priv/data_migrations/VersionedSessions/sql/rename-table.sql.eex b/priv/data_migrations/VersionedSessions/sql/rename-table.sql.eex
new file mode 100644
index 0000000000..ea07c8d93b
--- /dev/null
+++ b/priv/data_migrations/VersionedSessions/sql/rename-table.sql.eex
@@ -0,0 +1 @@
+RENAME TABLE <%= @from %> TO <%= @to %> <%= if @cluster? do %>ON CLUSTER '{cluster}'<% end %>