Skip to content

Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.

Make self-hosted data migration easier (#2865)

ruslandogaadcce156325a
diff --git a/.github/workflows/elixir.yml b/.github/workflows/elixir.yml
index 003689b0c0..540b8a47c0 100644
--- a/.github/workflows/elixir.yml
+++ b/.github/workflows/elixir.yml
@@ -70,12 +70,8 @@ jobs:
run: npm run deploy --prefix ./tracker
- name: Check Credo Warnings
run: mix credo diff --from-git-merge-base origin/master
- - name: Run tests (against v1 schema)
+ - name: Run tests
run: mix test --include slow --max-failures 1 --warnings-as-errors
- - name: Run tests (against v2 schema)
- run: mix test --include slow --max-failures 1 --warnings-as-errors
- env:
- V2_MIGRATION_DONE: 1
- name: Check Dialyzer
run: mix dialyzer
env:
diff --git a/config/.env.dev b/config/.env.dev
index a42c5a0f89..771e901633 100644
--- a/config/.env.dev
+++ b/config/.env.dev
@@ -17,4 +17,3 @@ GOOGLE_CLIENT_SECRET=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
PROMEX_DISABLED=false
SITE_DEFAULT_INGEST_THRESHOLD=1000000
-V2_MIGRATION_DONE=1
diff --git a/config/runtime.exs b/config/runtime.exs
index e9b86ac08d..82d8ceb6d5 100644
--- a/config/runtime.exs
+++ b/config/runtime.exs
@@ -99,8 +99,6 @@ ch_db_url =
|> get_var_from_path_or_env("CLICKHOUSE_MAX_BUFFER_SIZE", "10000")
|> Integer.parse()
-v2_migration_done = get_var_from_path_or_env(config_dir, "V2_MIGRATION_DONE")
-
### Mandatory params End
build_metadata_raw = get_var_from_path_or_env(config_dir, "BUILD_METADATA", "{}")
@@ -133,7 +131,7 @@ runtime_metadata = [
config :plausible, :runtime_metadata, runtime_metadata
-config :plausible, :v2_migration_done, v2_migration_done
+config :plausible, :v2_migration_done, true
sentry_dsn = get_var_from_path_or_env(config_dir, "SENTRY_DSN")
honeycomb_api_key = get_var_from_path_or_env(config_dir, "HONEYCOMB_API_KEY")
diff --git a/lib/plausible/data_migration.ex b/lib/plausible/data_migration.ex
index be38859286..7655f56e3d 100644
--- a/lib/plausible/data_migration.ex
+++ b/lib/plausible/data_migration.ex
@@ -1,6 +1,6 @@
defmodule Plausible.DataMigration do
@moduledoc """
- Base module for coordinated Clickhouse data migrations
+ Base module for coordinated Clickhouse data migrations
run via remote shell or otherwise (TBD).
"""
@@ -12,20 +12,43 @@ defmodule Plausible.DataMigration do
@dir dir
@repo repo
- def run_sql_confirm(name, assigns \\ []) do
- query = unwrap_with_io(name, assigns)
+ def run_sql_confirm(name, options \\ []) do
+ {prompt_options, assigns} =
+ Keyword.split(options, [:prompt_message, :prompt_default_choice])
- confirm("Execute?", fn -> do_run(name, query) end)
+ query = unwrap_with_io(name, assigns)
+ message = prompt_options[:prompt_message] || "Execute?"
+ default_choice = Keyword.get(prompt_options, :prompt_default_choice, :yes)
+ confirm(message, fn -> do_run(name, query) end, default_choice)
end
- def confirm(message, func) do
- prompt = IO.ANSI.white() <> message <> " [Y/n]: " <> IO.ANSI.reset()
+ def confirm(message, func, default_choice \\ :yes) do
+ choices =
+ case default_choice do
+ :yes -> " [Y/n]: "
+ :no -> " [y/N]: "
+ end
- if String.downcase(String.trim(IO.gets(prompt))) == "n" do
+ prompt = IO.ANSI.white() <> message <> choices <> IO.ANSI.reset()
+ answer = String.downcase(String.trim(IO.gets(prompt)))
+
+ skip = fn ->
IO.puts(" #{IO.ANSI.cyan()}Skipped.#{IO.ANSI.reset()}")
{:ok, :skip}
- else
- func.()
+ end
+
+ case answer do
+ "y" ->
+ func.()
+
+ "n" ->
+ skip.()
+
+ _ ->
+ case default_choice do
+ :yes -> func.()
+ :no -> skip.()
+ end
end
end
diff --git a/lib/plausible/data_migration/numeric_ids.ex b/lib/plausible/data_migration/numeric_ids.ex
index 004f8ca452..f8e859f0f7 100644
--- a/lib/plausible/data_migration/numeric_ids.ex
+++ b/lib/plausible/data_migration/numeric_ids.ex
@@ -62,10 +62,9 @@ defmodule Plausible.DataMigration.NumericIDs do
start_from = start_from || List.first(partitions)
IO.puts("""
- Got the following migration settings:
+ Got the following migration settings:
- max_threads: #{max_threads}
- - dict_password: ✅
- table_settings: #{table_settings}
- db url: #{db_url}
- cluster?: #{cluster?}
@@ -90,8 +89,28 @@ defmodule Plausible.DataMigration.NumericIDs do
end
end
- {:ok, _} = run_sql_fn.("drop-events-v2", cluster?: cluster?)
- {:ok, _} = run_sql_fn.("drop-sessions-v2", cluster?: cluster?)
+ drop_v2_extra_opts = fn table ->
+ case @repo.query("select count(*) from {$0:Identifier}", [table]) do
+ {:ok, %{rows: [[count]]}} when count > 0 ->
+ [
+ prompt_message: "The table contains #{count} rows. Execute?",
+ prompt_default_choice: :no
+ ]
+
+ {:ok, _} ->
+ [prompt_default_choice: :no]
+
+ {:error, _} ->
+ []
+ end
+ end
+
+ {:ok, _} =
+ run_sql_fn.("drop-events-v2", [cluster?: cluster?] ++ drop_v2_extra_opts.("events_v2"))
+
+ {:ok, _} =
+ run_sql_fn.("drop-sessions-v2", [cluster?: cluster?] ++ drop_v2_extra_opts.("sessions_v2"))
+
{:ok, _} = run_sql_fn.("drop-tmp-events-v2", [])
{:ok, _} = run_sql_fn.("drop-tmp-sessions-v2", [])
{:ok, _} = run_sql_fn.("drop-domains-lookup", [])
diff --git a/priv/data_migrations/NumericIDs/sql/insert-into-tmp-events-v2.sql.eex b/priv/data_migrations/NumericIDs/sql/insert-into-tmp-events-v2.sql.eex
index c5fd79b4d9..26500f99fb 100644
--- a/priv/data_migrations/NumericIDs/sql/insert-into-tmp-events-v2.sql.eex
+++ b/priv/data_migrations/NumericIDs/sql/insert-into-tmp-events-v2.sql.eex
@@ -10,7 +10,7 @@ INSERT INTO tmp_events_v2 (
transferred_from
)
SELECT
- (SELECT site_id from domains_lookup WHERE domain = domain LIMIT 1),
+ site_id,
timestamp,
name,
user_id,
@@ -38,4 +38,5 @@ SELECT
transferred_from
FROM
events
+INNER JOIN domains_lookup ON domains_lookup.domain = events.domain
WHERE (_partition_id = '<%= @partition %>')
diff --git a/priv/data_migrations/NumericIDs/sql/insert-into-tmp-sessions-v2.sql.eex b/priv/data_migrations/NumericIDs/sql/insert-into-tmp-sessions-v2.sql.eex
index c50ca8cbda..e2222b144f 100644
--- a/priv/data_migrations/NumericIDs/sql/insert-into-tmp-sessions-v2.sql.eex
+++ b/priv/data_migrations/NumericIDs/sql/insert-into-tmp-sessions-v2.sql.eex
@@ -6,7 +6,7 @@ operating_system_version, subdivision1_code, subdivision2_code, city_geoname_id,
utm_term, transferred_from, entry_meta.key, entry_meta.value
)
SELECT
- (SELECT site_id from domains_lookup WHERE domain = domain LIMIT 1),
+ site_id,
session_id, sign, user_id, hostname, timestamp, start, is_bounce, entry_page,
exit_page, pageviews, events, duration, referrer, referrer_source, country_code,
screen_size, operating_system, browser, utm_medium, utm_source, utm_campaign, browser_version,
@@ -14,4 +14,5 @@ SELECT
utm_term, transferred_from, entry_meta.key, entry_meta.value
FROM
sessions
+INNER JOIN domains_lookup ON domains_lookup.domain = sessions.domain
WHERE (_partition_id = '<%= @partition %>')