Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.
TypeScript analyticsanalytics-dashboardclickhouseelixirgoogle-analyticsmarketingopen-source-analyticsphoenixplausible-analyticspostgresqlprivacyprivacy-friendlyself-hostedsimple-analyticsstatisticstailwindcssweb-analyticswebsite-analyticswebsite-statswebsite-tracking
Make self-hosted data migration easier (#2865)
ruslandogaadcce156325a
diff --git a/.github/workflows/elixir.yml b/.github/workflows/elixir.ymlindex 003689b0c0..540b8a47c0 100644--- a/.github/workflows/elixir.yml+++ b/.github/workflows/elixir.yml@@ -70,12 +70,8 @@ jobs:run: npm run deploy --prefix ./tracker- name: Check Credo Warningsrun: mix credo diff --from-git-merge-base origin/master- - name: Run tests (against v1 schema)+ - name: Run testsrun: mix test --include slow --max-failures 1 --warnings-as-errors- - name: Run tests (against v2 schema)- run: mix test --include slow --max-failures 1 --warnings-as-errors- env:- V2_MIGRATION_DONE: 1- name: Check Dialyzerrun: mix dialyzerenv:diff --git a/config/.env.dev b/config/.env.devindex a42c5a0f89..771e901633 100644--- a/config/.env.dev+++ b/config/.env.dev@@ -17,4 +17,3 @@ GOOGLE_CLIENT_SECRET=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PROMEX_DISABLED=falseSITE_DEFAULT_INGEST_THRESHOLD=1000000-V2_MIGRATION_DONE=1diff --git a/config/runtime.exs b/config/runtime.exsindex e9b86ac08d..82d8ceb6d5 100644--- a/config/runtime.exs+++ b/config/runtime.exs@@ -99,8 +99,6 @@ ch_db_url =|> get_var_from_path_or_env("CLICKHOUSE_MAX_BUFFER_SIZE", "10000")|> Integer.parse()-v2_migration_done = get_var_from_path_or_env(config_dir, "V2_MIGRATION_DONE")-### Mandatory params Endbuild_metadata_raw = get_var_from_path_or_env(config_dir, "BUILD_METADATA", "{}")@@ -133,7 +131,7 @@ runtime_metadata = [config :plausible, :runtime_metadata, runtime_metadata-config :plausible, :v2_migration_done, v2_migration_done+config :plausible, :v2_migration_done, truesentry_dsn = get_var_from_path_or_env(config_dir, "SENTRY_DSN")honeycomb_api_key = get_var_from_path_or_env(config_dir, "HONEYCOMB_API_KEY")diff --git a/lib/plausible/data_migration.ex b/lib/plausible/data_migration.exindex be38859286..7655f56e3d 100644--- a/lib/plausible/data_migration.ex+++ b/lib/plausible/data_migration.ex@@ -1,6 +1,6 @@defmodule Plausible.DataMigration do@moduledoc """- Base module for coordinated Clickhouse data migrations+ Base module for coordinated Clickhouse data migrationsrun via remote shell or otherwise (TBD)."""@@ -12,20 +12,43 @@ defmodule Plausible.DataMigration do@dir dir@repo repo- def run_sql_confirm(name, assigns \\ []) do- query = unwrap_with_io(name, assigns)+ def run_sql_confirm(name, options \\ []) do+ {prompt_options, assigns} =+ Keyword.split(options, [:prompt_message, :prompt_default_choice])- confirm("Execute?", fn -> do_run(name, query) end)+ query = unwrap_with_io(name, assigns)+ message = prompt_options[:prompt_message] || "Execute?"+ default_choice = Keyword.get(prompt_options, :prompt_default_choice, :yes)+ confirm(message, fn -> do_run(name, query) end, default_choice)end- def confirm(message, func) do- prompt = IO.ANSI.white() <> message <> " [Y/n]: " <> IO.ANSI.reset()+ def confirm(message, func, default_choice \\ :yes) do+ choices =+ case default_choice do+ :yes -> " [Y/n]: "+ :no -> " [y/N]: "+ end- if String.downcase(String.trim(IO.gets(prompt))) == "n" do+ prompt = IO.ANSI.white() <> message <> choices <> IO.ANSI.reset()+ answer = String.downcase(String.trim(IO.gets(prompt)))++ skip = fn ->IO.puts(" #{IO.ANSI.cyan()}Skipped.#{IO.ANSI.reset()}"){:ok, :skip}- else- func.()+ end++ case answer do+ "y" ->+ func.()++ "n" ->+ skip.()++ _ ->+ case default_choice do+ :yes -> func.()+ :no -> skip.()+ endendenddiff --git a/lib/plausible/data_migration/numeric_ids.ex b/lib/plausible/data_migration/numeric_ids.exindex 004f8ca452..f8e859f0f7 100644--- a/lib/plausible/data_migration/numeric_ids.ex+++ b/lib/plausible/data_migration/numeric_ids.ex@@ -62,10 +62,9 @@ defmodule Plausible.DataMigration.NumericIDs dostart_from = start_from || List.first(partitions)IO.puts("""- Got the following migration settings:+ Got the following migration settings:- max_threads: #{max_threads}- - dict_password: ✅- table_settings: #{table_settings}- db url: #{db_url}- cluster?: #{cluster?}@@ -90,8 +89,28 @@ defmodule Plausible.DataMigration.NumericIDs doendend- {:ok, _} = run_sql_fn.("drop-events-v2", cluster?: cluster?)- {:ok, _} = run_sql_fn.("drop-sessions-v2", cluster?: cluster?)+ drop_v2_extra_opts = fn table ->+ case @repo.query("select count(*) from {$0:Identifier}", [table]) do+ {:ok, %{rows: [[count]]}} when count > 0 ->+ [+ prompt_message: "The table contains #{count} rows. Execute?",+ prompt_default_choice: :no+ ]++ {:ok, _} ->+ [prompt_default_choice: :no]++ {:error, _} ->+ []+ end+ end++ {:ok, _} =+ run_sql_fn.("drop-events-v2", [cluster?: cluster?] ++ drop_v2_extra_opts.("events_v2"))++ {:ok, _} =+ run_sql_fn.("drop-sessions-v2", [cluster?: cluster?] ++ drop_v2_extra_opts.("sessions_v2"))+{:ok, _} = run_sql_fn.("drop-tmp-events-v2", []){:ok, _} = run_sql_fn.("drop-tmp-sessions-v2", []){:ok, _} = run_sql_fn.("drop-domains-lookup", [])diff --git a/priv/data_migrations/NumericIDs/sql/insert-into-tmp-events-v2.sql.eex b/priv/data_migrations/NumericIDs/sql/insert-into-tmp-events-v2.sql.eexindex c5fd79b4d9..26500f99fb 100644--- a/priv/data_migrations/NumericIDs/sql/insert-into-tmp-events-v2.sql.eex+++ b/priv/data_migrations/NumericIDs/sql/insert-into-tmp-events-v2.sql.eex@@ -10,7 +10,7 @@ INSERT INTO tmp_events_v2 (transferred_from)SELECT- (SELECT site_id from domains_lookup WHERE domain = domain LIMIT 1),+ site_id,timestamp,name,user_id,@@ -38,4 +38,5 @@ SELECTtransferred_fromFROMevents+INNER JOIN domains_lookup ON domains_lookup.domain = events.domainWHERE (_partition_id = '<%= @partition %>')diff --git a/priv/data_migrations/NumericIDs/sql/insert-into-tmp-sessions-v2.sql.eex b/priv/data_migrations/NumericIDs/sql/insert-into-tmp-sessions-v2.sql.eexindex c50ca8cbda..e2222b144f 100644--- a/priv/data_migrations/NumericIDs/sql/insert-into-tmp-sessions-v2.sql.eex+++ b/priv/data_migrations/NumericIDs/sql/insert-into-tmp-sessions-v2.sql.eex@@ -6,7 +6,7 @@ operating_system_version, subdivision1_code, subdivision2_code, city_geoname_id,utm_term, transferred_from, entry_meta.key, entry_meta.value)SELECT- (SELECT site_id from domains_lookup WHERE domain = domain LIMIT 1),+ site_id,session_id, sign, user_id, hostname, timestamp, start, is_bounce, entry_page,exit_page, pageviews, events, duration, referrer, referrer_source, country_code,screen_size, operating_system, browser, utm_medium, utm_source, utm_campaign, browser_version,@@ -14,4 +14,5 @@ SELECTutm_term, transferred_from, entry_meta.key, entry_meta.valueFROMsessions+INNER JOIN domains_lookup ON domains_lookup.domain = sessions.domainWHERE (_partition_id = '<%= @partition %>')