Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.
TypeScript analyticsanalytics-dashboardclickhouseelixirgoogle-analyticsmarketingopen-source-analyticsphoenixplausible-analyticspostgresqlprivacyprivacy-friendlyself-hostedsimple-analyticsstatisticstailwindcssweb-analyticswebsite-analyticswebsite-statswebsite-tracking
Implement Numeric IDs migration (#2762)
Adam6637751a5e83
diff --git a/lib/plausible/event/clickhouse_schema.ex b/lib/plausible/clickhouse_event.exsimilarity index 100%rename from lib/plausible/event/clickhouse_schema.exrename to lib/plausible/clickhouse_event.exdiff --git a/lib/plausible/clickhouse_event_v2.ex b/lib/plausible/clickhouse_event_v2.exnew file mode 100644index 0000000000..a404d5e437--- /dev/null+++ b/lib/plausible/clickhouse_event_v2.ex@@ -0,0 +1,76 @@+defmodule Plausible.ClickhouseEventV2 do+ @moduledoc """+ Event schema for when NumericIDs migration is complete+ """+ use Ecto.Schema+ import Ecto.Changeset++ @primary_key false+ schema "events_v2" do+ field :name, :string+ field :site_id, Ch.Types.UInt64+ field :hostname, :string+ field :pathname, :string+ field :user_id, Ch.Types.UInt64+ field :session_id, Ch.Types.UInt64+ field :timestamp, :naive_datetime++ field :referrer, :string+ field :referrer_source, :string+ field :utm_medium, :string+ field :utm_source, :string+ field :utm_campaign, :string+ field :utm_content, :string+ field :utm_term, :string++ field :country_code, Ch.Types.FixedString, size: 2+ field :subdivision1_code, :string+ field :subdivision2_code, :string+ field :city_geoname_id, Ch.Types.UInt32++ field :screen_size, :string+ field :operating_system, :string+ field :operating_system_version, :string+ field :browser, :string+ field :browser_version, :string++ field :"meta.key", {:array, :string}+ field :"meta.value", {:array, :string}+ field :transferred_from, :string+ end++ def new(attrs) do+ %__MODULE__{}+ |> cast(+ attrs,+ [+ :name,+ :site_id,+ :hostname,+ :pathname,+ :user_id,+ :timestamp,+ :operating_system,+ :operating_system_version,+ :browser,+ :browser_version,+ :referrer,+ :referrer_source,+ :utm_medium,+ :utm_source,+ :utm_campaign,+ :utm_content,+ :utm_term,+ :country_code,+ :subdivision1_code,+ :subdivision2_code,+ :city_geoname_id,+ :screen_size,+ :"meta.key",+ :"meta.value"+ ],+ empty_values: [nil, ""]+ )+ |> validate_required([:name, :site_id, :hostname, :pathname, :user_id, :timestamp])+ end+enddiff --git a/lib/plausible/session/clickhouse_schema.ex b/lib/plausible/clickhouse_session.exsimilarity index 66%rename from lib/plausible/session/clickhouse_schema.exrename to lib/plausible/clickhouse_session.exindex 256fac3fe9..5f5a3c02b6 100644--- a/lib/plausible/session/clickhouse_schema.ex+++ b/lib/plausible/clickhouse_session.ex@@ -1,6 +1,5 @@defmodule Plausible.ClickhouseSession douse Ecto.Schema- import Ecto.Changeset@primary_key falseschema "sessions" do@@ -47,35 +46,4 @@ defmodule Plausible.ClickhouseSession dodef random_uint64() do:crypto.strong_rand_bytes(8) |> :binary.decode_unsigned()end-- def changeset(session, attrs) do- session- |> cast(attrs, [- :hostname,- :domain,- :entry_page,- :exit_page,- :fingerprint,- :start,- :length,- :is_bounce,- :operating_system,- :operating_system_version,- :browser_version,- :referrer,- :referrer_source,- :utm_medium,- :utm_source,- :utm_campaign,- :utm_content,- :utm_term,- :country_code,- :country_geoname_id,- :subdivision1_code,- :subdivision2_code,- :city_geoname_id,- :screen_size- ])- |> validate_required([:hostname, :domain, :fingerprint, :is_bounce, :start])- endenddiff --git a/lib/plausible/clickhouse_session_v2.ex b/lib/plausible/clickhouse_session_v2.exnew file mode 100644index 0000000000..9646527df7--- /dev/null+++ b/lib/plausible/clickhouse_session_v2.ex@@ -0,0 +1,52 @@+defmodule Plausible.ClickhouseSessionV2 do+ @moduledoc """+ Session schema for when NumericIDs migration is complete+ """+ use Ecto.Schema++ @primary_key false+ schema "sessions_v2" do+ field :hostname, :string+ field :site_id, Ch.Types.UInt64+ field :user_id, Ch.Types.UInt64+ field :session_id, Ch.Types.UInt64++ field :start, :naive_datetime+ field :duration, Ch.Types.UInt32+ field :is_bounce, :boolean+ field :entry_page, :string+ field :exit_page, :string+ field :pageviews, Ch.Types.Int32+ field :events, Ch.Types.Int32+ field :sign, Ch.Types.Int8++ field :"entry_meta.key", {:array, :string}+ field :"entry_meta.value", {:array, :string}++ field :utm_medium, :string+ field :utm_source, :string+ field :utm_campaign, :string+ field :utm_content, :string+ field :utm_term, :string+ field :referrer, :string+ field :referrer_source, :string++ field :country_code, Ch.Types.FixedString, size: 2+ field :subdivision1_code, :string+ field :subdivision2_code, :string+ field :city_geoname_id, Ch.Types.UInt32++ field :screen_size, :string+ field :operating_system, :string+ field :operating_system_version, :string+ field :browser, :string+ field :browser_version, :string+ field :timestamp, :naive_datetime++ field :transferred_from, :string+ end++ def random_uint64() do+ :crypto.strong_rand_bytes(8) |> :binary.decode_unsigned()+ end+enddiff --git a/lib/plausible/data_migration.ex b/lib/plausible/data_migration.exnew file mode 100644index 0000000000..611826cef2--- /dev/null+++ b/lib/plausible/data_migration.ex@@ -0,0 +1,64 @@+defmodule Plausible.DataMigration do+ @moduledoc """+ Base module for coordinated Clickhouse data migrations+ run via remote shell or otherwise (TBD).+ """++ defmacro __using__(opts) do+ dir = Keyword.fetch!(opts, :dir)+ repo = Keyword.get(opts, :repo, Plausible.DataMigration.Repo)++ quote bind_quoted: [dir: dir, repo: repo] do+ @dir dir+ @repo repo++ def run_sql_confirm(name, assigns \\ []) do+ query = unwrap(name, assigns)++ confirm("Execute?", fn -> do_run(name, query) end)+ end++ def confirm(message, func) do+ prompt = IO.ANSI.white() <> message <> " [Y/n]: " <> IO.ANSI.reset()++ if String.downcase(String.trim(IO.gets(prompt))) == "n" do+ IO.puts(" #{IO.ANSI.cyan()}Skipped.#{IO.ANSI.reset()}")+ {:ok, :skip}+ else+ func.()+ end+ end++ defp unwrap(name, assigns) do+ "priv/data_migrations"+ |> Path.join(@dir)+ |> Path.join("sql")+ |> Path.join(name <> ".sql.eex")+ |> EEx.eval_file(assigns: assigns)+ end++ def run_sql(name, assigns \\ []) do+ query = unwrap(name, assigns)+ do_run(name, query)+ end++ defp do_run(name, query) do+ {:ok, res} = @repo.query(query, [], timeout: :infinity)+ IO.puts(" #{IO.ANSI.yellow()}#{name} #{IO.ANSI.green()}Done!#{IO.ANSI.reset()}\n")+ IO.puts(String.duplicate("-", 78))+ {:ok, res}+ end++ defp unwrap_with_io(name, assigns) do+ IO.puts("#{IO.ANSI.yellow()}Running #{name}#{IO.ANSI.reset()}")+ query = unwrap(name, assigns)++ IO.puts("""+ -> Query: #{IO.ANSI.blue()}#{String.trim(query)}#{IO.ANSI.reset()}+ """)++ query+ end+ end+ end+enddiff --git a/lib/plausible/data_migration/numeric_ids.ex b/lib/plausible/data_migration/numeric_ids.exnew file mode 100644index 0000000000..8782df7a2c--- /dev/null+++ b/lib/plausible/data_migration/numeric_ids.ex@@ -0,0 +1,141 @@+defmodule Plausible.DataMigration.NumericIDs do+ @moduledoc """+ Numeric IDs migration, SQL files available at:+ priv/data_migrations/NumericIDs/sql+ """+ use Plausible.DataMigration, dir: "NumericIDs"++ @table_settings "SETTINGS index_granularity = 8192, storage_policy = 'tiered'"++ # credo:disable-for-next-line Credo.Check.Refactor.CyclomaticComplexity+ def run(opts \\ []) do+ interactive? = Keyword.get(opts, :interactive?, true)++ db_url =+ System.get_env(+ "NUMERIC_IDS_MIGRATION_DB_URL",+ Application.get_env(:plausible, Plausible.IngestRepo)[:url]+ )++ max_threads =+ "NUMERIC_IDS_MIGRATION_MAX_THREADS" |> System.get_env("16") |> String.to_integer()++ # TBD: There's most likely a bug in Clickhouse defining Postgres dictionaries,+ # we'll use a static URL for now+ dict_url = Keyword.get(opts, :dict_url) || System.get_env("DOMAINS_DICT_URL") || ""++ dict_password =+ Keyword.get(opts, :dict_password) || System.get_env("DOMAINS_DICT_PASSWORD") || ""++ table_settings =+ Keyword.get(opts, :table_settings) || System.get_env("NUMERIC_IDS_TABLE_SETTINGS") ||+ @table_settings++ start_from =+ Keyword.get(opts, :start_from) || System.get_env("NUMERIC_IDS_PARTITION_START_FROM")++ stop_at =+ Keyword.get(opts, :stop_at) || System.get_env("NUMERIC_IDS_PARTITION_STOP_AT") ||+ previous_part()++ (byte_size(dict_url) > 0 and byte_size(dict_password) > 0) ||+ raise "Set DOMAINS_DICT_URL and DOMAINS_DICT_PASSWORD"++ @repo.start(db_url, max_threads)++ cluster? =+ case run_sql("check-replicas") do+ {:ok, %{num_rows: 0}} -> false+ {:ok, %{num_rows: 1}} -> true+ end++ {:ok, %{rows: partitions}} =+ run_sql("list-partitions", start_from: start_from, stop_at: stop_at)++ partitions = Enum.map(partitions, fn [part] -> part end)++ start_from = start_from || List.first(partitions)++ IO.puts("""+ Got the following migration settings:++ - max_threads: #{max_threads}+ - dict_url: #{dict_url}+ - dict_password: ✅+ - table_settings: #{table_settings}+ - db url: #{db_url}+ - cluster?: #{cluster?}+ - partitions to do: #{inspect(partitions, pretty: true, width: 80)}+ - start from: #{start_from}+ - stop at: #{stop_at}+ """)++ run_sql_fn =+ if interactive? do+ &run_sql_confirm/2+ else+ &run_sql/2+ end++ confirm_fn =+ if interactive? do+ &confirm/2+ else+ fn _, run_fn ->+ run_fn.()+ end+ end++ {:ok, _} = run_sql_fn.("drop-events-v2", cluster?: cluster?)+ {:ok, _} = run_sql_fn.("drop-sessions-v2", cluster?: cluster?)+ {:ok, _} = run_sql_fn.("drop-tmp-events-v2", [])+ {:ok, _} = run_sql_fn.("drop-tmp-sessions-v2", [])+ {:ok, _} = run_sql_fn.("drop-dict", [])++ {:ok, _} =+ run_sql_fn.("create-dict-from-static-file",+ dict_url: dict_url,+ dict_password: dict_password+ )++ {:ok, _} = run_sql_fn.("create-events-v2", table_settings: table_settings, cluster?: cluster?)++ {:ok, _} =+ run_sql_fn.("create-sessions-v2", table_settings: table_settings, cluster?: cluster?)++ {:ok, _} = run_sql_fn.("create-tmp-events-v2", table_settings: table_settings)+ {:ok, _} = run_sql_fn.("create-tmp-sessions-v2", table_settings: table_settings)++ confirm_fn.("Start migration? (starting from partition: #{start_from})", fn ->+ IO.puts("start.. #{DateTime.utc_now()}")++ for part <- partitions do+ part_start = System.monotonic_time()++ confirm_fn.("Run partition: #{part}?", fn ->+ {:ok, _} = run_sql("insert-into-tmp-events-v2", partition: part)+ {:ok, _} = run_sql("attach-tmp-events-v2", partition: part)+ {:ok, _} = run_sql("truncate-tmp-events-v2", [])+ {:ok, _} = run_sql("insert-into-tmp-sessions-v2", partition: part)+ {:ok, _} = run_sql("attach-tmp-sessions-v2", partition: part)+ {:ok, _} = run_sql("truncate-tmp-sessions-v2", [])+ end)++ part_end = System.monotonic_time()++ IO.puts(+ "#{part} took #{System.convert_time_unit(part_end - part_start, :native, :second)} seconds"+ )+ end++ IO.puts("end.. #{DateTime.utc_now()}")+ end)+ end++ defp previous_part() do+ now = NaiveDateTime.utc_now()+ month = String.pad_leading("#{now.month - 1}", 2, "0")+ year = "#{now.year}"+ "#{year}#{month}"+ end+enddiff --git a/lib/plausible/data_migration/repo.ex b/lib/plausible/data_migration/repo.exnew file mode 100644index 0000000000..3ceac17b21--- /dev/null+++ b/lib/plausible/data_migration/repo.ex@@ -0,0 +1,22 @@+defmodule Plausible.DataMigration.Repo do+ @moduledoc """+ Ecto.Repo for Clickhouse data migrations, to be started manually,+ outside of the main application supervision tree.+ """+ use Ecto.Repo,+ otp_app: :plausible,+ adapter: Ecto.Adapters.ClickHouse++ def start(url, max_threads) when is_binary(url) and is_integer(max_threads) do+ start_link(+ url: url,+ queue_target: 500,+ queue_interval: 2000,+ pool_size: 1,+ settings: [+ max_insert_threads: max_threads,+ send_progress_in_http_headers: 1+ ]+ )+ end+enddiff --git a/priv/data_migrations/NumericIDs/sql/attach-tmp-events-v2.sql.eex b/priv/data_migrations/NumericIDs/sql/attach-tmp-events-v2.sql.eexnew file mode 100644index 0000000000..ca274978ff--- /dev/null+++ b/priv/data_migrations/NumericIDs/sql/attach-tmp-events-v2.sql.eex@@ -0,0 +1 @@+ALTER TABLE events_v2 ATTACH PARTITION ID '<%= @partition %>' FROM tmp_events_v2diff --git a/priv/data_migrations/NumericIDs/sql/attach-tmp-sessions-v2.sql.eex b/priv/data_migrations/NumericIDs/sql/attach-tmp-sessions-v2.sql.eexnew file mode 100644index 0000000000..1b64580f92--- /dev/null+++ b/priv/data_migrations/NumericIDs/sql/attach-tmp-sessions-v2.sql.eex@@ -0,0 +1 @@+ALTER TABLE sessions_v2 ATTACH PARTITION ID '<%= @partition %>' FROM tmp_sessions_v2diff --git a/priv/data_migrations/NumericIDs/sql/check-replicas.sql.eex b/priv/data_migrations/NumericIDs/sql/check-replicas.sql.eexnew file mode 100644index 0000000000..6d639edcff--- /dev/null+++ b/priv/data_migrations/NumericIDs/sql/check-replicas.sql.eex@@ -0,0 +1 @@+SELECT 1 FROM system.replicas WHERE table = 'events'diff --git a/priv/data_migrations/NumericIDs/sql/create-dict-from-static-file.sql.eex b/priv/data_migrations/NumericIDs/sql/create-dict-from-static-file.sql.eexnew file mode 100644index 0000000000..be5c45a217--- /dev/null+++ b/priv/data_migrations/NumericIDs/sql/create-dict-from-static-file.sql.eex@@ -0,0 +1,9 @@+CREATE DICTIONARY domains_lookup (+`id` UInt64,+`site_id` UInt64,+`domain` String,+`partition` String+) PRIMARY KEY domain+SOURCE(HTTP(URL '<%= @dict_url %>' CREDENTIALS(user 'user' password '<%= @dict_password %>') FORMAT 'CSVWithNames'))+LIFETIME(3600)+LAYOUT(COMPLEX_KEY_HASHED())diff --git a/priv/data_migrations/NumericIDs/sql/create-events-v2.sql.eex b/priv/data_migrations/NumericIDs/sql/create-events-v2.sql.eexnew file mode 100644index 0000000000..6413354cdf--- /dev/null+++ b/priv/data_migrations/NumericIDs/sql/create-events-v2.sql.eex@@ -0,0 +1,39 @@+CREATE TABLE events_v2 <%= if @cluster? do %>ON CLUSTER '{cluster}'<% end %>+(+ `timestamp` DateTime CODEC(Delta(4), LZ4),+ `name` LowCardinality(String),+ `site_id` UInt64,+ `user_id` UInt64,+ `session_id` UInt64,+ `hostname` String,+ `pathname` String CODEC(ZSTD(3)),+ `referrer` String,+ `referrer_source` String,+ `country_code` FixedString(2),+ `screen_size` LowCardinality(String),+ `operating_system` LowCardinality(String),+ `browser` LowCardinality(String),+ `utm_medium` String,+ `utm_source` String,+ `utm_campaign` String,+ `meta.key` Array(String),+ `meta.value` Array(String),+ `browser_version` LowCardinality(String),+ `operating_system_version` LowCardinality(String),+ `subdivision1_code` LowCardinality(String),+ `subdivision2_code` LowCardinality(String),+ `city_geoname_id` UInt32,+ `utm_content` String,+ `utm_term` String,+ `transferred_from` String+)+<%= if @cluster? do %>+ENGINE = ReplicatedMergeTree('/clickhouse/{cluster}/tables/{shard}/{database}/events_v2', '{replica}')+<% else %>+ENGINE = MergeTree()+<% end %>+PARTITION BY toYYYYMM(timestamp)+PRIMARY KEY (site_id, toDate(timestamp), name, user_id)+ORDER BY (site_id, toDate(timestamp), name, user_id, timestamp)+SAMPLE BY user_id+<%= @table_settings %>diff --git a/priv/data_migrations/NumericIDs/sql/create-sessions-v2.sql.eex b/priv/data_migrations/NumericIDs/sql/create-sessions-v2.sql.eexnew file mode 100644index 0000000000..38d1e5eebc--- /dev/null+++ b/priv/data_migrations/NumericIDs/sql/create-sessions-v2.sql.eex@@ -0,0 +1,45 @@+CREATE TABLE sessions_v2 <%= if @cluster? do %>ON CLUSTER '{cluster}'<% end %>+(+ `session_id` UInt64,+ `sign` Int8,+ `site_id` UInt64,+ `user_id` UInt64,+ `hostname` String,+ `timestamp` DateTime CODEC(DoubleDelta, LZ4),+ `start` DateTime CODEC(DoubleDelta, LZ4),+ `is_bounce` UInt8,+ `entry_page` String,+ `exit_page` String,+ `pageviews` Int32,+ `events` Int32,+ `duration` UInt32,+ `referrer` String,+ `referrer_source` String,+ `country_code` LowCardinality(FixedString(2)),+ `screen_size` LowCardinality(String),+ `operating_system` LowCardinality(String),+ `browser` LowCardinality(String),+ `utm_medium` String,+ `utm_source` String,+ `utm_campaign` String,+ `browser_version` LowCardinality(String),+ `operating_system_version` LowCardinality(String),+ `subdivision1_code` LowCardinality(String),+ `subdivision2_code` LowCardinality(String),+ `city_geoname_id` UInt32,+ `utm_content` String,+ `utm_term` String,+ `transferred_from` String,+ `entry_meta.key` Array(String),+ `entry_meta.value` Array(String)+)+<%= if @cluster? do %>+ENGINE = ReplicatedCollapsingMergeTree('/clickhouse/{cluster}/tables/{shard}/plausible_prod/sessions_v2', '{replica}', sign)+<% else %>+ENGINE = CollapsingMergeTree(sign)+<% end %>+PARTITION BY toYYYYMM(start)+PRIMARY KEY (site_id, toDate(start), user_id, session_id)+ORDER BY (site_id, toDate(start), user_id, session_id)+SAMPLE BY user_id+<%= @table_settings %>diff --git a/priv/data_migrations/NumericIDs/sql/create-tmp-events-v2.sql.eex b/priv/data_migrations/NumericIDs/sql/create-tmp-events-v2.sql.eexnew file mode 100644index 0000000000..be70d6c445--- /dev/null+++ b/priv/data_migrations/NumericIDs/sql/create-tmp-events-v2.sql.eex@@ -0,0 +1,6 @@+CREATE TABLE IF NOT EXISTS+ tmp_events_v2 AS events_v2 ENGINE = MergeTree()+ ORDER BY (site_id, toDate(timestamp), name, user_id, timestamp)+ PRIMARY KEY (site_id, toDate(timestamp), name, user_id)+ PARTITION BY toYYYYMM(timestamp)+ <%= @table_settings %>diff --git a/priv/data_migrations/NumericIDs/sql/create-tmp-sessions-v2.sql.eex b/priv/data_migrations/NumericIDs/sql/create-tmp-sessions-v2.sql.eexnew file mode 100644index 0000000000..bc93a129b5--- /dev/null+++ b/priv/data_migrations/NumericIDs/sql/create-tmp-sessions-v2.sql.eex@@ -0,0 +1,6 @@+CREATE TABLE IF NOT EXISTS+ tmp_sessions_v2 AS sessions_v2 ENGINE = CollapsingMergeTree(sign)+ PARTITION BY toYYYYMM(start)+ PRIMARY KEY (site_id, toDate(start), user_id, session_id)+ ORDER BY (site_id, toDate(start), user_id, session_id)+ <%= @table_settings %>diff --git a/priv/data_migrations/NumericIDs/sql/drop-dict.sql.eex b/priv/data_migrations/NumericIDs/sql/drop-dict.sql.eexnew file mode 100644index 0000000000..3e4f226437--- /dev/null+++ b/priv/data_migrations/NumericIDs/sql/drop-dict.sql.eex@@ -0,0 +1 @@+DROP DICTIONARY IF EXISTS domains_lookup SYNCdiff --git a/priv/data_migrations/NumericIDs/sql/drop-events-v2.sql.eex b/priv/data_migrations/NumericIDs/sql/drop-events-v2.sql.eexnew file mode 100644index 0000000000..ae3927e49d--- /dev/null+++ b/priv/data_migrations/NumericIDs/sql/drop-events-v2.sql.eex@@ -0,0 +1 @@+DROP TABLE IF EXISTS events_v2 <%= if @cluster? do %>ON CLUSTER '{cluster}'<% end %> SYNCdiff --git a/priv/data_migrations/NumericIDs/sql/drop-sessions-v2.sql.eex b/priv/data_migrations/NumericIDs/sql/drop-sessions-v2.sql.eexnew file mode 100644index 0000000000..eee7668420--- /dev/null+++ b/priv/data_migrations/NumericIDs/sql/drop-sessions-v2.sql.eex@@ -0,0 +1 @@+DROP TABLE IF EXISTS sessions_v2 <%= if @cluster? do %>ON CLUSTER '{cluster}'<% end %> SYNCdiff --git a/priv/data_migrations/NumericIDs/sql/drop-tmp-events-v2.sql.eex b/priv/data_migrations/NumericIDs/sql/drop-tmp-events-v2.sql.eexnew file mode 100644index 0000000000..9ce47fbae7--- /dev/null+++ b/priv/data_migrations/NumericIDs/sql/drop-tmp-events-v2.sql.eex@@ -0,0 +1 @@+DROP TABLE IF EXISTS tmp_events_v2 SYNCdiff --git a/priv/data_migrations/NumericIDs/sql/drop-tmp-sessions-v2.sql.eex b/priv/data_migrations/NumericIDs/sql/drop-tmp-sessions-v2.sql.eexnew file mode 100644index 0000000000..ea3666c511--- /dev/null+++ b/priv/data_migrations/NumericIDs/sql/drop-tmp-sessions-v2.sql.eex@@ -0,0 +1 @@+DROP TABLE IF EXISTS tmp_sessions_v2 SYNCdiff --git a/priv/data_migrations/NumericIDs/sql/insert-into-tmp-events-v2.sql.eex b/priv/data_migrations/NumericIDs/sql/insert-into-tmp-events-v2.sql.eexnew file mode 100644index 0000000000..454ac09962--- /dev/null+++ b/priv/data_migrations/NumericIDs/sql/insert-into-tmp-events-v2.sql.eex@@ -0,0 +1,54 @@+INSERT INTO tmp_events_v2 (+ site_id, timestamp, name, user_id,+ session_id, hostname, pathname, referrer,+ referrer_source, country_code, screen_size,+ operating_system, browser, utm_medium,+ utm_source, utm_campaign, meta.key,+ meta.value, browser_version, operating_system_version,+ subdivision1_code, subdivision2_code,+ city_geoname_id, utm_content, utm_term,+ transferred_from+)+SELECT+ dictGet(+ 'domains_lookup', 'site_id', domain+ ),+ timestamp,+ name,+ user_id,+ session_id,+ hostname,+ pathname,+ referrer,+ referrer_source,+ country_code,+ screen_size,+ operating_system,+ browser,+ utm_medium,+ utm_source,+ utm_campaign,+ meta.key,+ meta.value,+ browser_version,+ operating_system_version,+ subdivision1_code,+ subdivision2_code,+ city_geoname_id,+ utm_content,+ utm_term,+ transferred_from+FROM+ events+WHERE+ (+ domain IN (+ SELECT+ domain+ FROM+ dictionary('domains_lookup')+ WHERE+ partition <= '<%= @partition %>'+ )+ )+AND (_partition_id = '<%= @partition %>')diff --git a/priv/data_migrations/NumericIDs/sql/insert-into-tmp-sessions-v2.sql.eex b/priv/data_migrations/NumericIDs/sql/insert-into-tmp-sessions-v2.sql.eexnew file mode 100644index 0000000000..194e877af2--- /dev/null+++ b/priv/data_migrations/NumericIDs/sql/insert-into-tmp-sessions-v2.sql.eex@@ -0,0 +1,30 @@+INSERT INTO tmp_sessions_v2 (+site_id, session_id, sign, user_id, hostname, timestamp, start, is_bounce, entry_page,+exit_page, pageviews, events, duration, referrer, referrer_source, country_code,+screen_size, operating_system, browser, utm_medium, utm_source, utm_campaign, browser_version,+operating_system_version, subdivision1_code, subdivision2_code, city_geoname_id, utm_content,+utm_term, transferred_from, entry_meta.key, entry_meta.value+)+SELECT+ dictGet(+ 'domains_lookup', 'site_id', domain+ ),+ session_id, sign, user_id, hostname, timestamp, start, is_bounce, entry_page,+ exit_page, pageviews, events, duration, referrer, referrer_source, country_code,+ screen_size, operating_system, browser, utm_medium, utm_source, utm_campaign, browser_version,+ operating_system_version, subdivision1_code, subdivision2_code, city_geoname_id, utm_content,+ utm_term, transferred_from, entry_meta.key, entry_meta.value+FROM+ sessions+WHERE+ (+ domain IN (+ SELECT+ domain+ FROM+ dictionary('domains_lookup')+ WHERE+ partition <= '<%= @partition %>'+ )+ )+AND (_partition_id = '<%= @partition %>')diff --git a/priv/data_migrations/NumericIDs/sql/list-partitions.sql.eex b/priv/data_migrations/NumericIDs/sql/list-partitions.sql.eexnew file mode 100644index 0000000000..c945e73764--- /dev/null+++ b/priv/data_migrations/NumericIDs/sql/list-partitions.sql.eex@@ -0,0 +1,6 @@+SELECT partition+FROM system.parts+WHERE table = 'events'+<%= if @start_from do %>AND partition >= '<%= @start_from %>'<% end %>+<%= if @stop_at do %>AND partition <= '<%= @stop_at %>'<% end %>+ORDER BY partitiondiff --git a/priv/data_migrations/NumericIDs/sql/truncate-tmp-events-v2.sql.eex b/priv/data_migrations/NumericIDs/sql/truncate-tmp-events-v2.sql.eexnew file mode 100644index 0000000000..18e6fef98c--- /dev/null+++ b/priv/data_migrations/NumericIDs/sql/truncate-tmp-events-v2.sql.eex@@ -0,0 +1 @@+TRUNCATE TABLE tmp_events_v2diff --git a/priv/data_migrations/NumericIDs/sql/truncate-tmp-sessions-v2.sql.eex b/priv/data_migrations/NumericIDs/sql/truncate-tmp-sessions-v2.sql.eexnew file mode 100644index 0000000000..22b8bbddf9--- /dev/null+++ b/priv/data_migrations/NumericIDs/sql/truncate-tmp-sessions-v2.sql.eex@@ -0,0 +1 @@+TRUNCATE TABLE tmp_sessions_v2diff --git a/priv/ingest_repo/migrations/20230320094327_create_v2_schemas.exs b/priv/ingest_repo/migrations/20230320094327_create_v2_schemas.exsnew file mode 100644index 0000000000..a77591f64d--- /dev/null+++ b/priv/ingest_repo/migrations/20230320094327_create_v2_schemas.exs@@ -0,0 +1,33 @@+defmodule Plausible.IngestRepo.Migrations.CreateV2Schemas do+ @moduledoc """+ Normally, for live environments the migration will be done via+ `DataMigration.NumericIDs` module (TBD). In which case PASS_V2_SCHEMA_MIGRATION+ environment variable needs to be set, to only make the standard migrate+ command write an entry into schema_migrations.++ For tests, and entirely new small, self-hosted instances however,+ we want to keep the ability of preparing the database without enforcing+ any data migration.+ """++ use Ecto.Migration++ use Plausible.DataMigration, dir: "NumericIDs"++ @cluster? false+ @settings "SETTINGS index_granularity = 8192"++ def up do+ if System.get_env("PASS_V2_SCHEMA_MIGRATION") do+ :ok+ else+ execute unwrap("create-events-v2", table_settings: @settings, cluster?: @cluster?)+ execute unwrap("create-sessions-v2", table_settings: @settings, cluster?: @cluster?)+ end+ end++ def down do+ execute unwrap("drop-events-v2", cluster?: @cluster?)+ execute unwrap("drop-sessions-v2", cluster?: @cluster?)+ end+end