Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.
TypeScript analyticsanalytics-dashboardclickhouseelixirgoogle-analyticsmarketingopen-source-analyticsphoenixplausible-analyticspostgresqlprivacyprivacy-friendlyself-hostedsimple-analyticsstatisticstailwindcssweb-analyticswebsite-analyticswebsite-statswebsite-tracking
V2 migration tweaks for self hosted release (#2825)
hq1b9c211047248
diff --git a/config/.env.dev b/config/.env.devindex d7e46e3854..1f9cb25d33 100644--- a/config/.env.dev+++ b/config/.env.dev@@ -16,3 +16,4 @@ GOOGLE_CLIENT_ID=875387135161-l8tp53dpt7fdhdg9m1pc3vl42si95rh0.apps.googleusercoGOOGLE_CLIENT_SECRET=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PROMEX_DISABLED=false+V2_MIGRATION_DONE=1diff --git a/lib/plausible/data_migration/numeric_ids.ex b/lib/plausible/data_migration/numeric_ids.exindex 4774f465b6..004f8ca452 100644--- a/lib/plausible/data_migration/numeric_ids.ex+++ b/lib/plausible/data_migration/numeric_ids.ex@@ -5,7 +5,20 @@ defmodule Plausible.DataMigration.NumericIDs do"""use Plausible.DataMigration, dir: "NumericIDs"- @table_settings "SETTINGS index_granularity = 8192, storage_policy = 'tiered'"+ import Ecto.Query++ defmodule DomainsLookup do+ @moduledoc false+ use Ecto.Schema++ @primary_key false+ schema "domains_lookup" do+ field :site_id, Ch.Types.UInt64+ field :domain, :string+ end+ end++ @table_settings "SETTINGS index_granularity = 8192"def ready?() doApplication.get_env(:plausible, :v2_migration_done) || false@@ -24,13 +37,6 @@ defmodule Plausible.DataMigration.NumericIDs domax_threads ="NUMERIC_IDS_MIGRATION_MAX_THREADS" |> System.get_env("16") |> String.to_integer()- # TBD: There's most likely a bug in Clickhouse defining Postgres dictionaries,- # we'll use a static URL for now- dict_url = Keyword.get(opts, :dict_url) || System.get_env("DOMAINS_DICT_URL") || ""-- dict_password =- Keyword.get(opts, :dict_password) || System.get_env("DOMAINS_DICT_PASSWORD") || ""-table_settings =Keyword.get(opts, :table_settings) || System.get_env("NUMERIC_IDS_TABLE_SETTINGS") ||@table_settings@@ -38,12 +44,7 @@ defmodule Plausible.DataMigration.NumericIDs dostart_from =Keyword.get(opts, :start_from) || System.get_env("NUMERIC_IDS_PARTITION_START_FROM")- stop_at =- Keyword.get(opts, :stop_at) || System.get_env("NUMERIC_IDS_PARTITION_STOP_AT") ||- previous_part()-- (byte_size(dict_url) > 0 and byte_size(dict_password) > 0) ||- raise "Set DOMAINS_DICT_URL and DOMAINS_DICT_PASSWORD"+ stop_at = Keyword.get(opts, :stop_at) || System.get_env("NUMERIC_IDS_PARTITION_STOP_AT")@repo.start(db_url, max_threads)@@ -64,7 +65,6 @@ defmodule Plausible.DataMigration.NumericIDs doGot the following migration settings:- max_threads: #{max_threads}- - dict_url: #{dict_url}- dict_password: ✅- table_settings: #{table_settings}- db url: #{db_url}@@ -94,13 +94,7 @@ defmodule Plausible.DataMigration.NumericIDs do{:ok, _} = run_sql_fn.("drop-sessions-v2", cluster?: cluster?){:ok, _} = run_sql_fn.("drop-tmp-events-v2", []){:ok, _} = run_sql_fn.("drop-tmp-sessions-v2", [])- {:ok, _} = run_sql_fn.("drop-dict", [])-- {:ok, _} =- run_sql_fn.("create-dict-from-static-file",- dict_url: dict_url,- dict_password: dict_password- )+ {:ok, _} = run_sql_fn.("drop-domains-lookup", []){:ok, _} = run_sql_fn.("create-events-v2", table_settings: table_settings, cluster?: cluster?)@@ -110,6 +104,21 @@ defmodule Plausible.DataMigration.NumericIDs do{:ok, _} = run_sql_fn.("create-tmp-events-v2", table_settings: table_settings){:ok, _} = run_sql_fn.("create-tmp-sessions-v2", table_settings: table_settings)+ case run_sql_fn.("create-domains-lookup", table_settings: table_settings) do+ {:ok, _} ->+ confirm_fn.("Populate domains-lookup with postgres sites", fn ->+ mappings =+ Plausible.Site+ |> select([s], %{site_id: s.id, domain: s.domain})+ |> Plausible.Repo.all()++ @repo.insert_all(DomainsLookup, mappings)+ end)++ _ ->+ :ignore+ end+confirm_fn.("Start migration? (starting from partition: #{start_from})", fn ->IO.puts("start.. #{DateTime.utc_now()}")@@ -135,11 +144,4 @@ defmodule Plausible.DataMigration.NumericIDs doIO.puts("end.. #{DateTime.utc_now()}")end)end-- defp previous_part() do- now = NaiveDateTime.utc_now()- month = String.pad_leading("#{now.month - 1}", 2, "0")- year = "#{now.year}"- "#{year}#{month}"- endenddiff --git a/priv/data_migrations/NumericIDs/sql/create-dict-from-static-file.sql.eex b/priv/data_migrations/NumericIDs/sql/create-dict-from-static-file.sql.eexdeleted file mode 100644index be5c45a217..0000000000--- a/priv/data_migrations/NumericIDs/sql/create-dict-from-static-file.sql.eex+++ /dev/null@@ -1,9 +0,0 @@-CREATE DICTIONARY domains_lookup (-`id` UInt64,-`site_id` UInt64,-`domain` String,-`partition` String-) PRIMARY KEY domain-SOURCE(HTTP(URL '<%= @dict_url %>' CREDENTIALS(user 'user' password '<%= @dict_password %>') FORMAT 'CSVWithNames'))-LIFETIME(3600)-LAYOUT(COMPLEX_KEY_HASHED())diff --git a/priv/data_migrations/NumericIDs/sql/create-domains-lookup.sql.eex b/priv/data_migrations/NumericIDs/sql/create-domains-lookup.sql.eexnew file mode 100644index 0000000000..b18d7fb5f4--- /dev/null+++ b/priv/data_migrations/NumericIDs/sql/create-domains-lookup.sql.eex@@ -0,0 +1 @@+CREATE TABLE IF NOT EXISTS domains_lookup(site_id UInt64, domain String) ENGINE Memory;diff --git a/priv/data_migrations/NumericIDs/sql/create-events-v2.sql.eex b/priv/data_migrations/NumericIDs/sql/create-events-v2.sql.eexindex 6413354cdf..ebe5fc8619 100644--- a/priv/data_migrations/NumericIDs/sql/create-events-v2.sql.eex+++ b/priv/data_migrations/NumericIDs/sql/create-events-v2.sql.eex@@ -1,4 +1,4 @@-CREATE TABLE events_v2 <%= if @cluster? do %>ON CLUSTER '{cluster}'<% end %>+CREATE TABLE IF NOT EXISTS events_v2 <%= if @cluster? do %>ON CLUSTER '{cluster}'<% end %>(`timestamp` DateTime CODEC(Delta(4), LZ4),`name` LowCardinality(String),diff --git a/priv/data_migrations/NumericIDs/sql/create-sessions-v2.sql.eex b/priv/data_migrations/NumericIDs/sql/create-sessions-v2.sql.eexindex 38d1e5eebc..e03d910762 100644--- a/priv/data_migrations/NumericIDs/sql/create-sessions-v2.sql.eex+++ b/priv/data_migrations/NumericIDs/sql/create-sessions-v2.sql.eex@@ -1,4 +1,4 @@-CREATE TABLE sessions_v2 <%= if @cluster? do %>ON CLUSTER '{cluster}'<% end %>+CREATE TABLE IF NOT EXISTS sessions_v2 <%= if @cluster? do %>ON CLUSTER '{cluster}'<% end %>(`session_id` UInt64,`sign` Int8,diff --git a/priv/data_migrations/NumericIDs/sql/drop-dict.sql.eex b/priv/data_migrations/NumericIDs/sql/drop-dict.sql.eexdeleted file mode 100644index 3e4f226437..0000000000--- a/priv/data_migrations/NumericIDs/sql/drop-dict.sql.eex+++ /dev/null@@ -1 +0,0 @@-DROP DICTIONARY IF EXISTS domains_lookup SYNCdiff --git a/priv/data_migrations/NumericIDs/sql/drop-domains-lookup.sql.eex b/priv/data_migrations/NumericIDs/sql/drop-domains-lookup.sql.eexnew file mode 100644index 0000000000..6ca833f126--- /dev/null+++ b/priv/data_migrations/NumericIDs/sql/drop-domains-lookup.sql.eex@@ -0,0 +1 @@+DROP TABLE IF EXISTS domains_lookup SYNC;diff --git a/priv/data_migrations/NumericIDs/sql/insert-into-tmp-events-v2.sql.eex b/priv/data_migrations/NumericIDs/sql/insert-into-tmp-events-v2.sql.eexindex 454ac09962..c5fd79b4d9 100644--- a/priv/data_migrations/NumericIDs/sql/insert-into-tmp-events-v2.sql.eex+++ b/priv/data_migrations/NumericIDs/sql/insert-into-tmp-events-v2.sql.eex@@ -10,9 +10,7 @@ INSERT INTO tmp_events_v2 (transferred_from)SELECT- dictGet(- 'domains_lookup', 'site_id', domain- ),+ (SELECT site_id from domains_lookup WHERE domain = domain LIMIT 1),timestamp,name,user_id,@@ -40,15 +38,4 @@ SELECTtransferred_fromFROMevents-WHERE- (- domain IN (- SELECT- domain- FROM- dictionary('domains_lookup')- WHERE- partition <= '<%= @partition %>'- )- )-AND (_partition_id = '<%= @partition %>')+WHERE (_partition_id = '<%= @partition %>')diff --git a/priv/data_migrations/NumericIDs/sql/insert-into-tmp-sessions-v2.sql.eex b/priv/data_migrations/NumericIDs/sql/insert-into-tmp-sessions-v2.sql.eexindex 194e877af2..c50ca8cbda 100644--- a/priv/data_migrations/NumericIDs/sql/insert-into-tmp-sessions-v2.sql.eex+++ b/priv/data_migrations/NumericIDs/sql/insert-into-tmp-sessions-v2.sql.eex@@ -6,9 +6,7 @@ operating_system_version, subdivision1_code, subdivision2_code, city_geoname_id,utm_term, transferred_from, entry_meta.key, entry_meta.value)SELECT- dictGet(- 'domains_lookup', 'site_id', domain- ),+ (SELECT site_id from domains_lookup WHERE domain = domain LIMIT 1),session_id, sign, user_id, hostname, timestamp, start, is_bounce, entry_page,exit_page, pageviews, events, duration, referrer, referrer_source, country_code,screen_size, operating_system, browser, utm_medium, utm_source, utm_campaign, browser_version,@@ -16,15 +14,4 @@ SELECTutm_term, transferred_from, entry_meta.key, entry_meta.valueFROMsessions-WHERE- (- domain IN (- SELECT- domain- FROM- dictionary('domains_lookup')- WHERE- partition <= '<%= @partition %>'- )- )-AND (_partition_id = '<%= @partition %>')+WHERE (_partition_id = '<%= @partition %>')diff --git a/priv/ingest_repo/migrations/20230320094327_create_v2_schemas.exs b/priv/ingest_repo/migrations/20230320094327_create_v2_schemas.exsindex a77591f64d..b881f5eae6 100644--- a/priv/ingest_repo/migrations/20230320094327_create_v2_schemas.exs+++ b/priv/ingest_repo/migrations/20230320094327_create_v2_schemas.exs@@ -1,13 +1,11 @@defmodule Plausible.IngestRepo.Migrations.CreateV2Schemas do@moduledoc """Normally, for live environments the migration will be done via- `DataMigration.NumericIDs` module (TBD). In which case PASS_V2_SCHEMA_MIGRATION- environment variable needs to be set, to only make the standard migrate- command write an entry into schema_migrations.+ `DataMigration.NumericIDs` module.For tests, and entirely new small, self-hosted instances however,we want to keep the ability of preparing the database without enforcing- any data migration.+ any _data_ migration."""use Ecto.Migration@@ -18,12 +16,8 @@ defmodule Plausible.IngestRepo.Migrations.CreateV2Schemas do@settings "SETTINGS index_granularity = 8192"def up do- if System.get_env("PASS_V2_SCHEMA_MIGRATION") do- :ok- else- execute unwrap("create-events-v2", table_settings: @settings, cluster?: @cluster?)- execute unwrap("create-sessions-v2", table_settings: @settings, cluster?: @cluster?)- end+ execute unwrap("create-events-v2", table_settings: @settings, cluster?: @cluster?)+ execute unwrap("create-sessions-v2", table_settings: @settings, cluster?: @cluster?)enddef down do