Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.
TypeScript analyticsanalytics-dashboardclickhouseelixirgoogle-analyticsmarketingopen-source-analyticsphoenixplausible-analyticspostgresqlprivacyprivacy-friendlyself-hostedsimple-analyticsstatisticstailwindcssweb-analyticswebsite-analyticswebsite-statswebsite-tracking
Mutation to populate event session columns (#3844)
Karl-Aksel Puulmann26d41ddbb97b
diff --git a/lib/plausible/data_migration.ex b/lib/plausible/data_migration.exindex bc41dba9a3..838b101ad5 100644--- a/lib/plausible/data_migration.ex+++ b/lib/plausible/data_migration.ex@@ -62,13 +62,13 @@ defmodule Plausible.DataMigration do|> EEx.eval_file(assigns: assigns)end- def run_sql(name, assigns \\ []) do+ def run_sql(name, assigns \\ [], options \\ []) doquery = unwrap(name, assigns)- do_run(name, query)+ do_run(name, query, options)end- defp do_run(name, query) do- case @repo.query(query, [], timeout: :infinity) do+ defp do_run(name, query, options \\ []) do+ case @repo.query(query, [], [timeout: :infinity] ++ options) do{:ok, res} ->IO.puts(" #{IO.ANSI.yellow()}#{name} #{IO.ANSI.green()}Done!#{IO.ANSI.reset()}\n")IO.puts(String.duplicate("-", 78))diff --git a/lib/plausible/data_migration/populate_event_session_columns.ex b/lib/plausible/data_migration/populate_event_session_columns.exnew file mode 100644index 0000000000..659ca8c831--- /dev/null+++ b/lib/plausible/data_migration/populate_event_session_columns.ex@@ -0,0 +1,166 @@+defmodule Plausible.DataMigration.PopulateEventSessionColumns do+ @moduledoc """+ Populates event session columns with data from sessions table.++ Run via: ./bin/plausible rpc "Plausible.DataMigration.PopulateEventSessionColumns.run"+ Kill via: ./bin/plausible rpc "Plausible.DataMigration.PopulateEventSessionColumns.kill"+ Monitor via ./bin/plausible rpc "Plausible.DataMigration.PopulateEventSessionColumns.report_progress"++ Suggested to run in a screen/tmux session to be able to easily monitor++ SQL files available at: priv/data_migrations/PopulateEventSessionColumns/sql+ """+ use Plausible.DataMigration, dir: "PopulateEventSessionColumns", repo: Plausible.IngestRepo++ require Logger++ # See https://clickhouse.com/docs/en/sql-reference/dictionaries#cache for meaning of these defaults+ @default_dictionary_config %{+ lifetime: 600_000,+ size_in_cells: 1_000_000,+ max_threads_for_updates: 6+ }++ def run(opts \\ []) do+ cluster? = Plausible.MigrationUtils.clustered_table?("sessions_v2")++ {:ok, _} =+ run_sql("create-sessions-dictionary",+ cluster?: cluster?,+ dictionary_connection_params:+ Keyword.get(opts, :dictionary_connection_string, dictionary_connection_params()),+ dictionary_config: dictionary_config(opts)+ )++ {partitions, _, _, _} = get_partitions(opts)++ IO.puts("Starting mutation on #{length(partitions)} partition(s)")++ for partition <- partitions do+ {:ok, _} =+ run_sql("update-table", [cluster?: cluster?, partition: partition],+ settings: [allow_nondeterministic_mutations: 1]+ )+ end++ wait_until_mutations_complete(opts)++ IO.puts("Mutations seem done, cleaning up!")+ {:ok, _} = run_sql("drop-sessions-dictionary", cluster?: cluster?)+ end++ def kill(opts \\ []) do+ cluster? = Plausible.MigrationUtils.clustered_table?("events_v2")++ report_progress(opts)++ IO.puts("Killing running mutations")+ {:ok, _} = run_sql("kill-running-mutations", cluster?: cluster?)+ end++ def wait_until_mutations_complete(opts \\ []) do+ Process.sleep(5_000)+ in_progress? = report_progress(opts)++ if in_progress? do+ wait_until_mutations_complete(opts)+ end+ end++ def report_progress(opts \\ []) do+ {partitions, parts, min_partition, max_partition} = get_partitions(opts)++ {:ok, %{rows: mutation_results}} =+ run_sql("get-mutations-progress",+ min_partition: min_partition,+ max_partition: max_partition+ )++ [+ [+ mutations,+ parts_to_do,+ running_for,+ total_size,+ todo_size,+ progress,+ latest_fail_reason,+ _,+ _+ ]+ ] =+ mutation_results++ {:ok, %{rows: [[merges]]}} = run_sql("get-merges-progress")+ {:ok, %{rows: disks}} = run_sql("get-disks")++ IO.puts("\n\n#{Timex.now() |> Timex.format!("{ISO:Extended}")}")++ # List partitions that need to run+ IO.puts(+ "Progress report for partitions #{Enum.min(partitions)}-#{Enum.max(partitions)} (parts: #{length(parts)})"+ )++ IO.puts("Disks overview:")++ for [name, path, full_space, total_space, full_percentage] <- disks do+ IO.puts(+ " #{name} at #{path} is at #{full_space}/#{total_space} (#{full_percentage}% full)"+ )+ end++ IO.puts("Currently #{mutations} mutation(s) are running.")++ if mutations > 0 do+ IO.puts(" To do #{parts_to_do} parts, #{todo_size}")+ IO.puts(" Out of #{length(parts)} parts, #{total_size}")+ IO.puts(" Running for #{format_duration(running_for)}")++ if progress > 0 do+ estimated_time_left = round(running_for / progress / 100 - running_for)+ IO.puts(" Estimated #{progress}% done, #{format_duration(estimated_time_left)} left")+ end++ if latest_fail_reason do+ IO.puts(" Some mutations might be failing. ClickHouse report: #{latest_fail_reason}")+ end+ end++ IO.puts("Currently #{merges} merge(s) are running relating to mutations.")++ mutations > 0+ end++ defp dictionary_config(opts) do+ @default_dictionary_config+ |> Map.merge(Keyword.get(opts, :dictionary_config, %{}))+ end++ # See https://clickhouse.com/docs/en/sql-reference/dictionaries#clickhouse for context+ defp dictionary_connection_params() do+ Plausible.IngestRepo.config()+ |> Enum.map(fn+ {:database, database} -> "DB '#{database}'"+ {:username, username} -> "USER '#{username}'"+ {:password, password} -> "PASSWORD '#{password}'"+ _ -> nil+ end)+ |> Enum.reject(&is_nil/1)+ |> Enum.join(" ")+ end++ defp get_partitions(opts) do+ [min_partition, max_partition] = Keyword.get(opts, :partition_range, ["0", "999999"])++ {:ok, %{rows: [[partitions, parts]]}} =+ run_sql("list-partitions", min_partition: min_partition, max_partition: max_partition)++ {partitions, parts, min_partition, max_partition}+ end++ defp format_duration(seconds) do+ seconds+ |> Timex.Duration.from_seconds()+ |> Timex.format_duration(Timex.Format.Duration.Formatters.Humanized)+ end+enddiff --git a/priv/data_migrations/PopulateEventSessionColumns/sql/create-sessions-dictionary.sql.eex b/priv/data_migrations/PopulateEventSessionColumns/sql/create-sessions-dictionary.sql.eexnew file mode 100644index 0000000000..b5bdac37f2--- /dev/null+++ b/priv/data_migrations/PopulateEventSessionColumns/sql/create-sessions-dictionary.sql.eex@@ -0,0 +1,32 @@+CREATE OR REPLACE DICTIONARY sessions_dict+<%= if @cluster? do %>ON CLUSTER '{cluster}'<% end %>+(+ site_id UInt64,+ session_id UInt64,+ referrer String,+ referrer_source String,+ utm_medium String,+ utm_source String,+ utm_campaign String,+ utm_content String,+ utm_term String,+ country_code String,+ subdivision1_code String,+ subdivision2_code String,+ city_geoname_id UInt32,+ screen_size String,+ operating_system String,+ operating_system_version String,+ browser String,+ browser_version String+)+PRIMARY KEY site_id, session_id+SOURCE(CLICKHOUSE(TABLE sessions_v2 <%= @dictionary_connection_params %>))+LIFETIME(<%= @dictionary_config.lifetime %>)+LAYOUT(+ complex_key_cache(+ size_in_cells <%= @dictionary_config.size_in_cells %>+ max_threads_for_updates <%= @dictionary_config.max_threads_for_updates %>+ allow_read_expired_keys 1+ )+)diff --git a/priv/data_migrations/PopulateEventSessionColumns/sql/drop-sessions-dictionary.sql.eex b/priv/data_migrations/PopulateEventSessionColumns/sql/drop-sessions-dictionary.sql.eexnew file mode 100644index 0000000000..69826e863c--- /dev/null+++ b/priv/data_migrations/PopulateEventSessionColumns/sql/drop-sessions-dictionary.sql.eex@@ -0,0 +1,2 @@+DROP DICTIONARY IF EXISTS sessions_dict+<%= if @cluster? do %>ON CLUSTER '{cluster}'<% end %>diff --git a/priv/data_migrations/PopulateEventSessionColumns/sql/get-disks.sql.eex b/priv/data_migrations/PopulateEventSessionColumns/sql/get-disks.sql.eexnew file mode 100644index 0000000000..aa678c4777--- /dev/null+++ b/priv/data_migrations/PopulateEventSessionColumns/sql/get-disks.sql.eex@@ -0,0 +1,7 @@+SELECT+ name,+ path,+ formatReadableSize(total_space - free_space) AS full_space,+ formatReadableSize(total_space),+ round((total_space - free_space) / total_space * 100, 2)+FROM system.disksdiff --git a/priv/data_migrations/PopulateEventSessionColumns/sql/get-merges-progress.sql.eex b/priv/data_migrations/PopulateEventSessionColumns/sql/get-merges-progress.sql.eexnew file mode 100644index 0000000000..c9589f83ca--- /dev/null+++ b/priv/data_migrations/PopulateEventSessionColumns/sql/get-merges-progress.sql.eex@@ -0,0 +1,4 @@+SELECT count()+FROM system.merges+WHERE table = 'events_v2'+ AND is_mutationdiff --git a/priv/data_migrations/PopulateEventSessionColumns/sql/get-mutations-progress.sql.eex b/priv/data_migrations/PopulateEventSessionColumns/sql/get-mutations-progress.sql.eexnew file mode 100644index 0000000000..13efb4fc2c--- /dev/null+++ b/priv/data_migrations/PopulateEventSessionColumns/sql/get-mutations-progress.sql.eex@@ -0,0 +1,29 @@+SELECT+ total,+ length(parts_to_do_names),+ running_for_seconds,+ formatReadableSize(total_bytes),+ formatReadableSize(todo_bytes),+ round(100 - todo_bytes / total_bytes * 100, 2),+ latest_fail_reason,+ sum(part.bytes_on_disk) as total_bytes,+ sumIf(part.bytes_on_disk, has(parts_to_do_names, part.name)) AS todo_bytes+FROM (+ SELECT count() AS total,+ groupArrayArray(parts_to_do_names) AS parts_to_do_names,+ now() - min(create_time) AS running_for_seconds,+ anyIf(latest_fail_reason, latest_fail_reason != '') AS latest_fail_reason+ FROM system.mutations+ WHERE not is_done+ AND table = 'events_v2'+ AND command ILIKE '%sessions_dict%'+) AS mut+CROSS JOIN (+ SELECT name, bytes_on_disk+ FROM system.parts+ WHERE table = 'events_v2'+ AND active+ AND partition >= '<%= @min_partition %>'+ AND partition <= '<%= @max_partition %>'+) AS part+GROUP BY mut.total, mut.parts_to_do_names, mut.running_for_seconds, mut.latest_fail_reasondiff --git a/priv/data_migrations/PopulateEventSessionColumns/sql/kill-running-mutations.sql.eex b/priv/data_migrations/PopulateEventSessionColumns/sql/kill-running-mutations.sql.eexnew file mode 100644index 0000000000..e37c6149ce--- /dev/null+++ b/priv/data_migrations/PopulateEventSessionColumns/sql/kill-running-mutations.sql.eex@@ -0,0 +1,2 @@+KILL MUTATION <%= if @cluster? do %>ON CLUSTER '{cluster}'<% end %>+WHERE command ILIKE '%sessions_dict%'diff --git a/priv/data_migrations/PopulateEventSessionColumns/sql/list-partitions.sql.eex b/priv/data_migrations/PopulateEventSessionColumns/sql/list-partitions.sql.eexnew file mode 100644index 0000000000..0bbc9b5c64--- /dev/null+++ b/priv/data_migrations/PopulateEventSessionColumns/sql/list-partitions.sql.eex@@ -0,0 +1,7 @@+SELECT arraySort(groupUniqArray(partition)), arraySort(groupArray(name))+FROM system.parts+WHERE active+ AND database = currentDatabase()+ AND table = 'events_v2'+ AND partition >= '<%= @min_partition %>'+ AND partition <= '<%= @max_partition %>'diff --git a/priv/data_migrations/PopulateEventSessionColumns/sql/update-table.sql.eex b/priv/data_migrations/PopulateEventSessionColumns/sql/update-table.sql.eexnew file mode 100644index 0000000000..f9990c98b2--- /dev/null+++ b/priv/data_migrations/PopulateEventSessionColumns/sql/update-table.sql.eex@@ -0,0 +1,21 @@+ALTER TABLE events_v2+<%= if @cluster? do %>ON CLUSTER '{cluster}'<% end %>+UPDATE+ referrer = dictGet('sessions_dict', 'referrer', tuple(site_id, session_id)),+ referrer_source = dictGet('sessions_dict', 'referrer_source', tuple(site_id, session_id)),+ utm_medium = dictGet('sessions_dict', 'utm_medium', tuple(site_id, session_id)),+ utm_source = dictGet('sessions_dict', 'utm_source', tuple(site_id, session_id)),+ utm_campaign = dictGet('sessions_dict', 'utm_campaign', tuple(site_id, session_id)),+ utm_content = dictGet('sessions_dict', 'utm_content', tuple(site_id, session_id)),+ utm_term = dictGet('sessions_dict', 'utm_term', tuple(site_id, session_id)),+ country_code = dictGet('sessions_dict', 'country_code', tuple(site_id, session_id)),+ subdivision1_code = dictGet('sessions_dict', 'subdivision1_code', tuple(site_id, session_id)),+ subdivision2_code = dictGet('sessions_dict', 'subdivision2_code', tuple(site_id, session_id)),+ city_geoname_id = dictGet('sessions_dict', 'city_geoname_id', tuple(site_id, session_id)),+ screen_size = dictGet('sessions_dict', 'screen_size', tuple(site_id, session_id)),+ operating_system = dictGet('sessions_dict', 'operating_system', tuple(site_id, session_id)),+ operating_system_version = dictGet('sessions_dict', 'operating_system_version', tuple(site_id, session_id)),+ browser = dictGet('sessions_dict', 'browser', tuple(site_id, session_id)),+ browser_version = dictGet('sessions_dict', 'browser_version', tuple(site_id, session_id))+IN PARTITION '<%= @partition %>'+WHERE 1=1