Skip to content

Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.

Implement Numeric IDs migration (#2762)

Adam6637751a5e83
diff --git a/lib/plausible/event/clickhouse_schema.ex b/lib/plausible/clickhouse_event.ex
similarity index 100%
rename from lib/plausible/event/clickhouse_schema.ex
rename to lib/plausible/clickhouse_event.ex
diff --git a/lib/plausible/clickhouse_event_v2.ex b/lib/plausible/clickhouse_event_v2.ex
new file mode 100644
index 0000000000..a404d5e437
--- /dev/null
+++ b/lib/plausible/clickhouse_event_v2.ex
@@ -0,0 +1,76 @@
+defmodule Plausible.ClickhouseEventV2 do
+ @moduledoc """
+ Event schema for when NumericIDs migration is complete
+ """
+ use Ecto.Schema
+ import Ecto.Changeset
+
+ @primary_key false
+ schema "events_v2" do
+ field :name, :string
+ field :site_id, Ch.Types.UInt64
+ field :hostname, :string
+ field :pathname, :string
+ field :user_id, Ch.Types.UInt64
+ field :session_id, Ch.Types.UInt64
+ field :timestamp, :naive_datetime
+
+ field :referrer, :string
+ field :referrer_source, :string
+ field :utm_medium, :string
+ field :utm_source, :string
+ field :utm_campaign, :string
+ field :utm_content, :string
+ field :utm_term, :string
+
+ field :country_code, Ch.Types.FixedString, size: 2
+ field :subdivision1_code, :string
+ field :subdivision2_code, :string
+ field :city_geoname_id, Ch.Types.UInt32
+
+ field :screen_size, :string
+ field :operating_system, :string
+ field :operating_system_version, :string
+ field :browser, :string
+ field :browser_version, :string
+
+ field :"meta.key", {:array, :string}
+ field :"meta.value", {:array, :string}
+ field :transferred_from, :string
+ end
+
+ def new(attrs) do
+ %__MODULE__{}
+ |> cast(
+ attrs,
+ [
+ :name,
+ :site_id,
+ :hostname,
+ :pathname,
+ :user_id,
+ :timestamp,
+ :operating_system,
+ :operating_system_version,
+ :browser,
+ :browser_version,
+ :referrer,
+ :referrer_source,
+ :utm_medium,
+ :utm_source,
+ :utm_campaign,
+ :utm_content,
+ :utm_term,
+ :country_code,
+ :subdivision1_code,
+ :subdivision2_code,
+ :city_geoname_id,
+ :screen_size,
+ :"meta.key",
+ :"meta.value"
+ ],
+ empty_values: [nil, ""]
+ )
+ |> validate_required([:name, :site_id, :hostname, :pathname, :user_id, :timestamp])
+ end
+end
diff --git a/lib/plausible/session/clickhouse_schema.ex b/lib/plausible/clickhouse_session.ex
similarity index 66%
rename from lib/plausible/session/clickhouse_schema.ex
rename to lib/plausible/clickhouse_session.ex
index 256fac3fe9..5f5a3c02b6 100644
--- a/lib/plausible/session/clickhouse_schema.ex
+++ b/lib/plausible/clickhouse_session.ex
@@ -1,6 +1,5 @@
defmodule Plausible.ClickhouseSession do
use Ecto.Schema
- import Ecto.Changeset
@primary_key false
schema "sessions" do
@@ -47,35 +46,4 @@ defmodule Plausible.ClickhouseSession do
def random_uint64() do
:crypto.strong_rand_bytes(8) |> :binary.decode_unsigned()
end
-
- def changeset(session, attrs) do
- session
- |> cast(attrs, [
- :hostname,
- :domain,
- :entry_page,
- :exit_page,
- :fingerprint,
- :start,
- :length,
- :is_bounce,
- :operating_system,
- :operating_system_version,
- :browser_version,
- :referrer,
- :referrer_source,
- :utm_medium,
- :utm_source,
- :utm_campaign,
- :utm_content,
- :utm_term,
- :country_code,
- :country_geoname_id,
- :subdivision1_code,
- :subdivision2_code,
- :city_geoname_id,
- :screen_size
- ])
- |> validate_required([:hostname, :domain, :fingerprint, :is_bounce, :start])
- end
end
diff --git a/lib/plausible/clickhouse_session_v2.ex b/lib/plausible/clickhouse_session_v2.ex
new file mode 100644
index 0000000000..9646527df7
--- /dev/null
+++ b/lib/plausible/clickhouse_session_v2.ex
@@ -0,0 +1,52 @@
+defmodule Plausible.ClickhouseSessionV2 do
+ @moduledoc """
+ Session schema for when NumericIDs migration is complete
+ """
+ use Ecto.Schema
+
+ @primary_key false
+ schema "sessions_v2" do
+ field :hostname, :string
+ field :site_id, Ch.Types.UInt64
+ field :user_id, Ch.Types.UInt64
+ field :session_id, Ch.Types.UInt64
+
+ field :start, :naive_datetime
+ field :duration, Ch.Types.UInt32
+ field :is_bounce, :boolean
+ field :entry_page, :string
+ field :exit_page, :string
+ field :pageviews, Ch.Types.Int32
+ field :events, Ch.Types.Int32
+ field :sign, Ch.Types.Int8
+
+ field :"entry_meta.key", {:array, :string}
+ field :"entry_meta.value", {:array, :string}
+
+ field :utm_medium, :string
+ field :utm_source, :string
+ field :utm_campaign, :string
+ field :utm_content, :string
+ field :utm_term, :string
+ field :referrer, :string
+ field :referrer_source, :string
+
+ field :country_code, Ch.Types.FixedString, size: 2
+ field :subdivision1_code, :string
+ field :subdivision2_code, :string
+ field :city_geoname_id, Ch.Types.UInt32
+
+ field :screen_size, :string
+ field :operating_system, :string
+ field :operating_system_version, :string
+ field :browser, :string
+ field :browser_version, :string
+ field :timestamp, :naive_datetime
+
+ field :transferred_from, :string
+ end
+
+ def random_uint64() do
+ :crypto.strong_rand_bytes(8) |> :binary.decode_unsigned()
+ end
+end
diff --git a/lib/plausible/data_migration.ex b/lib/plausible/data_migration.ex
new file mode 100644
index 0000000000..611826cef2
--- /dev/null
+++ b/lib/plausible/data_migration.ex
@@ -0,0 +1,64 @@
+defmodule Plausible.DataMigration do
+ @moduledoc """
+ Base module for coordinated Clickhouse data migrations
+ run via remote shell or otherwise (TBD).
+ """
+
+ defmacro __using__(opts) do
+ dir = Keyword.fetch!(opts, :dir)
+ repo = Keyword.get(opts, :repo, Plausible.DataMigration.Repo)
+
+ quote bind_quoted: [dir: dir, repo: repo] do
+ @dir dir
+ @repo repo
+
+ def run_sql_confirm(name, assigns \\ []) do
+ query = unwrap(name, assigns)
+
+ confirm("Execute?", fn -> do_run(name, query) end)
+ end
+
+ def confirm(message, func) do
+ prompt = IO.ANSI.white() <> message <> " [Y/n]: " <> IO.ANSI.reset()
+
+ if String.downcase(String.trim(IO.gets(prompt))) == "n" do
+ IO.puts(" #{IO.ANSI.cyan()}Skipped.#{IO.ANSI.reset()}")
+ {:ok, :skip}
+ else
+ func.()
+ end
+ end
+
+ defp unwrap(name, assigns) do
+ "priv/data_migrations"
+ |> Path.join(@dir)
+ |> Path.join("sql")
+ |> Path.join(name <> ".sql.eex")
+ |> EEx.eval_file(assigns: assigns)
+ end
+
+ def run_sql(name, assigns \\ []) do
+ query = unwrap(name, assigns)
+ do_run(name, query)
+ end
+
+ defp do_run(name, query) do
+ {:ok, res} = @repo.query(query, [], timeout: :infinity)
+ IO.puts(" #{IO.ANSI.yellow()}#{name} #{IO.ANSI.green()}Done!#{IO.ANSI.reset()}\n")
+ IO.puts(String.duplicate("-", 78))
+ {:ok, res}
+ end
+
+ defp unwrap_with_io(name, assigns) do
+ IO.puts("#{IO.ANSI.yellow()}Running #{name}#{IO.ANSI.reset()}")
+ query = unwrap(name, assigns)
+
+ IO.puts("""
+ -> Query: #{IO.ANSI.blue()}#{String.trim(query)}#{IO.ANSI.reset()}
+ """)
+
+ query
+ end
+ end
+ end
+end
diff --git a/lib/plausible/data_migration/numeric_ids.ex b/lib/plausible/data_migration/numeric_ids.ex
new file mode 100644
index 0000000000..8782df7a2c
--- /dev/null
+++ b/lib/plausible/data_migration/numeric_ids.ex
@@ -0,0 +1,141 @@
+defmodule Plausible.DataMigration.NumericIDs do
+ @moduledoc """
+ Numeric IDs migration, SQL files available at:
+ priv/data_migrations/NumericIDs/sql
+ """
+ use Plausible.DataMigration, dir: "NumericIDs"
+
+ @table_settings "SETTINGS index_granularity = 8192, storage_policy = 'tiered'"
+
+ # credo:disable-for-next-line Credo.Check.Refactor.CyclomaticComplexity
+ def run(opts \\ []) do
+ interactive? = Keyword.get(opts, :interactive?, true)
+
+ db_url =
+ System.get_env(
+ "NUMERIC_IDS_MIGRATION_DB_URL",
+ Application.get_env(:plausible, Plausible.IngestRepo)[:url]
+ )
+
+ max_threads =
+ "NUMERIC_IDS_MIGRATION_MAX_THREADS" |> System.get_env("16") |> String.to_integer()
+
+ # TBD: There's most likely a bug in Clickhouse defining Postgres dictionaries,
+ # we'll use a static URL for now
+ dict_url = Keyword.get(opts, :dict_url) || System.get_env("DOMAINS_DICT_URL") || ""
+
+ dict_password =
+ Keyword.get(opts, :dict_password) || System.get_env("DOMAINS_DICT_PASSWORD") || ""
+
+ table_settings =
+ Keyword.get(opts, :table_settings) || System.get_env("NUMERIC_IDS_TABLE_SETTINGS") ||
+ @table_settings
+
+ start_from =
+ Keyword.get(opts, :start_from) || System.get_env("NUMERIC_IDS_PARTITION_START_FROM")
+
+ stop_at =
+ Keyword.get(opts, :stop_at) || System.get_env("NUMERIC_IDS_PARTITION_STOP_AT") ||
+ previous_part()
+
+ (byte_size(dict_url) > 0 and byte_size(dict_password) > 0) ||
+ raise "Set DOMAINS_DICT_URL and DOMAINS_DICT_PASSWORD"
+
+ @repo.start(db_url, max_threads)
+
+ cluster? =
+ case run_sql("check-replicas") do
+ {:ok, %{num_rows: 0}} -> false
+ {:ok, %{num_rows: 1}} -> true
+ end
+
+ {:ok, %{rows: partitions}} =
+ run_sql("list-partitions", start_from: start_from, stop_at: stop_at)
+
+ partitions = Enum.map(partitions, fn [part] -> part end)
+
+ start_from = start_from || List.first(partitions)
+
+ IO.puts("""
+ Got the following migration settings:
+
+ - max_threads: #{max_threads}
+ - dict_url: #{dict_url}
+ - dict_password: ✅
+ - table_settings: #{table_settings}
+ - db url: #{db_url}
+ - cluster?: #{cluster?}
+ - partitions to do: #{inspect(partitions, pretty: true, width: 80)}
+ - start from: #{start_from}
+ - stop at: #{stop_at}
+ """)
+
+ run_sql_fn =
+ if interactive? do
+ &run_sql_confirm/2
+ else
+ &run_sql/2
+ end
+
+ confirm_fn =
+ if interactive? do
+ &confirm/2
+ else
+ fn _, run_fn ->
+ run_fn.()
+ end
+ end
+
+ {:ok, _} = run_sql_fn.("drop-events-v2", cluster?: cluster?)
+ {:ok, _} = run_sql_fn.("drop-sessions-v2", cluster?: cluster?)
+ {:ok, _} = run_sql_fn.("drop-tmp-events-v2", [])
+ {:ok, _} = run_sql_fn.("drop-tmp-sessions-v2", [])
+ {:ok, _} = run_sql_fn.("drop-dict", [])
+
+ {:ok, _} =
+ run_sql_fn.("create-dict-from-static-file",
+ dict_url: dict_url,
+ dict_password: dict_password
+ )
+
+ {:ok, _} = run_sql_fn.("create-events-v2", table_settings: table_settings, cluster?: cluster?)
+
+ {:ok, _} =
+ run_sql_fn.("create-sessions-v2", table_settings: table_settings, cluster?: cluster?)
+
+ {:ok, _} = run_sql_fn.("create-tmp-events-v2", table_settings: table_settings)
+ {:ok, _} = run_sql_fn.("create-tmp-sessions-v2", table_settings: table_settings)
+
+ confirm_fn.("Start migration? (starting from partition: #{start_from})", fn ->
+ IO.puts("start.. #{DateTime.utc_now()}")
+
+ for part <- partitions do
+ part_start = System.monotonic_time()
+
+ confirm_fn.("Run partition: #{part}?", fn ->
+ {:ok, _} = run_sql("insert-into-tmp-events-v2", partition: part)
+ {:ok, _} = run_sql("attach-tmp-events-v2", partition: part)
+ {:ok, _} = run_sql("truncate-tmp-events-v2", [])
+ {:ok, _} = run_sql("insert-into-tmp-sessions-v2", partition: part)
+ {:ok, _} = run_sql("attach-tmp-sessions-v2", partition: part)
+ {:ok, _} = run_sql("truncate-tmp-sessions-v2", [])
+ end)
+
+ part_end = System.monotonic_time()
+
+ IO.puts(
+ "#{part} took #{System.convert_time_unit(part_end - part_start, :native, :second)} seconds"
+ )
+ end
+
+ IO.puts("end.. #{DateTime.utc_now()}")
+ end)
+ end
+
+ defp previous_part() do
+ now = NaiveDateTime.utc_now()
+ month = String.pad_leading("#{now.month - 1}", 2, "0")
+ year = "#{now.year}"
+ "#{year}#{month}"
+ end
+end
diff --git a/lib/plausible/data_migration/repo.ex b/lib/plausible/data_migration/repo.ex
new file mode 100644
index 0000000000..3ceac17b21
--- /dev/null
+++ b/lib/plausible/data_migration/repo.ex
@@ -0,0 +1,22 @@
+defmodule Plausible.DataMigration.Repo do
+ @moduledoc """
+ Ecto.Repo for Clickhouse data migrations, to be started manually,
+ outside of the main application supervision tree.
+ """
+ use Ecto.Repo,
+ otp_app: :plausible,
+ adapter: Ecto.Adapters.ClickHouse
+
+ def start(url, max_threads) when is_binary(url) and is_integer(max_threads) do
+ start_link(
+ url: url,
+ queue_target: 500,
+ queue_interval: 2000,
+ pool_size: 1,
+ settings: [
+ max_insert_threads: max_threads,
+ send_progress_in_http_headers: 1
+ ]
+ )
+ end
+end
diff --git a/priv/data_migrations/NumericIDs/sql/attach-tmp-events-v2.sql.eex b/priv/data_migrations/NumericIDs/sql/attach-tmp-events-v2.sql.eex
new file mode 100644
index 0000000000..ca274978ff
--- /dev/null
+++ b/priv/data_migrations/NumericIDs/sql/attach-tmp-events-v2.sql.eex
@@ -0,0 +1 @@
+ALTER TABLE events_v2 ATTACH PARTITION ID '<%= @partition %>' FROM tmp_events_v2
diff --git a/priv/data_migrations/NumericIDs/sql/attach-tmp-sessions-v2.sql.eex b/priv/data_migrations/NumericIDs/sql/attach-tmp-sessions-v2.sql.eex
new file mode 100644
index 0000000000..1b64580f92
--- /dev/null
+++ b/priv/data_migrations/NumericIDs/sql/attach-tmp-sessions-v2.sql.eex
@@ -0,0 +1 @@
+ALTER TABLE sessions_v2 ATTACH PARTITION ID '<%= @partition %>' FROM tmp_sessions_v2
diff --git a/priv/data_migrations/NumericIDs/sql/check-replicas.sql.eex b/priv/data_migrations/NumericIDs/sql/check-replicas.sql.eex
new file mode 100644
index 0000000000..6d639edcff
--- /dev/null
+++ b/priv/data_migrations/NumericIDs/sql/check-replicas.sql.eex
@@ -0,0 +1 @@
+SELECT 1 FROM system.replicas WHERE table = 'events'
diff --git a/priv/data_migrations/NumericIDs/sql/create-dict-from-static-file.sql.eex b/priv/data_migrations/NumericIDs/sql/create-dict-from-static-file.sql.eex
new file mode 100644
index 0000000000..be5c45a217
--- /dev/null
+++ b/priv/data_migrations/NumericIDs/sql/create-dict-from-static-file.sql.eex
@@ -0,0 +1,9 @@
+CREATE DICTIONARY domains_lookup (
+`id` UInt64,
+`site_id` UInt64,
+`domain` String,
+`partition` String
+) PRIMARY KEY domain
+SOURCE(HTTP(URL '<%= @dict_url %>' CREDENTIALS(user 'user' password '<%= @dict_password %>') FORMAT 'CSVWithNames'))
+LIFETIME(3600)
+LAYOUT(COMPLEX_KEY_HASHED())
diff --git a/priv/data_migrations/NumericIDs/sql/create-events-v2.sql.eex b/priv/data_migrations/NumericIDs/sql/create-events-v2.sql.eex
new file mode 100644
index 0000000000..6413354cdf
--- /dev/null
+++ b/priv/data_migrations/NumericIDs/sql/create-events-v2.sql.eex
@@ -0,0 +1,39 @@
+CREATE TABLE events_v2 <%= if @cluster? do %>ON CLUSTER '{cluster}'<% end %>
+(
+ `timestamp` DateTime CODEC(Delta(4), LZ4),
+ `name` LowCardinality(String),
+ `site_id` UInt64,
+ `user_id` UInt64,
+ `session_id` UInt64,
+ `hostname` String,
+ `pathname` String CODEC(ZSTD(3)),
+ `referrer` String,
+ `referrer_source` String,
+ `country_code` FixedString(2),
+ `screen_size` LowCardinality(String),
+ `operating_system` LowCardinality(String),
+ `browser` LowCardinality(String),
+ `utm_medium` String,
+ `utm_source` String,
+ `utm_campaign` String,
+ `meta.key` Array(String),
+ `meta.value` Array(String),
+ `browser_version` LowCardinality(String),
+ `operating_system_version` LowCardinality(String),
+ `subdivision1_code` LowCardinality(String),
+ `subdivision2_code` LowCardinality(String),
+ `city_geoname_id` UInt32,
+ `utm_content` String,
+ `utm_term` String,
+ `transferred_from` String
+)
+<%= if @cluster? do %>
+ENGINE = ReplicatedMergeTree('/clickhouse/{cluster}/tables/{shard}/{database}/events_v2', '{replica}')
+<% else %>
+ENGINE = MergeTree()
+<% end %>
+PARTITION BY toYYYYMM(timestamp)
+PRIMARY KEY (site_id, toDate(timestamp), name, user_id)
+ORDER BY (site_id, toDate(timestamp), name, user_id, timestamp)
+SAMPLE BY user_id
+<%= @table_settings %>
diff --git a/priv/data_migrations/NumericIDs/sql/create-sessions-v2.sql.eex b/priv/data_migrations/NumericIDs/sql/create-sessions-v2.sql.eex
new file mode 100644
index 0000000000..38d1e5eebc
--- /dev/null
+++ b/priv/data_migrations/NumericIDs/sql/create-sessions-v2.sql.eex
@@ -0,0 +1,45 @@
+CREATE TABLE sessions_v2 <%= if @cluster? do %>ON CLUSTER '{cluster}'<% end %>
+(
+ `session_id` UInt64,
+ `sign` Int8,
+ `site_id` UInt64,
+ `user_id` UInt64,
+ `hostname` String,
+ `timestamp` DateTime CODEC(DoubleDelta, LZ4),
+ `start` DateTime CODEC(DoubleDelta, LZ4),
+ `is_bounce` UInt8,
+ `entry_page` String,
+ `exit_page` String,
+ `pageviews` Int32,
+ `events` Int32,
+ `duration` UInt32,
+ `referrer` String,
+ `referrer_source` String,
+ `country_code` LowCardinality(FixedString(2)),
+ `screen_size` LowCardinality(String),
+ `operating_system` LowCardinality(String),
+ `browser` LowCardinality(String),
+ `utm_medium` String,
+ `utm_source` String,
+ `utm_campaign` String,
+ `browser_version` LowCardinality(String),
+ `operating_system_version` LowCardinality(String),
+ `subdivision1_code` LowCardinality(String),
+ `subdivision2_code` LowCardinality(String),
+ `city_geoname_id` UInt32,
+ `utm_content` String,
+ `utm_term` String,
+ `transferred_from` String,
+ `entry_meta.key` Array(String),
+ `entry_meta.value` Array(String)
+)
+<%= if @cluster? do %>
+ENGINE = ReplicatedCollapsingMergeTree('/clickhouse/{cluster}/tables/{shard}/plausible_prod/sessions_v2', '{replica}', sign)
+<% else %>
+ENGINE = CollapsingMergeTree(sign)
+<% end %>
+PARTITION BY toYYYYMM(start)
+PRIMARY KEY (site_id, toDate(start), user_id, session_id)
+ORDER BY (site_id, toDate(start), user_id, session_id)
+SAMPLE BY user_id
+<%= @table_settings %>
diff --git a/priv/data_migrations/NumericIDs/sql/create-tmp-events-v2.sql.eex b/priv/data_migrations/NumericIDs/sql/create-tmp-events-v2.sql.eex
new file mode 100644
index 0000000000..be70d6c445
--- /dev/null
+++ b/priv/data_migrations/NumericIDs/sql/create-tmp-events-v2.sql.eex
@@ -0,0 +1,6 @@
+CREATE TABLE IF NOT EXISTS
+ tmp_events_v2 AS events_v2 ENGINE = MergeTree()
+ ORDER BY (site_id, toDate(timestamp), name, user_id, timestamp)
+ PRIMARY KEY (site_id, toDate(timestamp), name, user_id)
+ PARTITION BY toYYYYMM(timestamp)
+ <%= @table_settings %>
diff --git a/priv/data_migrations/NumericIDs/sql/create-tmp-sessions-v2.sql.eex b/priv/data_migrations/NumericIDs/sql/create-tmp-sessions-v2.sql.eex
new file mode 100644
index 0000000000..bc93a129b5
--- /dev/null
+++ b/priv/data_migrations/NumericIDs/sql/create-tmp-sessions-v2.sql.eex
@@ -0,0 +1,6 @@
+CREATE TABLE IF NOT EXISTS
+ tmp_sessions_v2 AS sessions_v2 ENGINE = CollapsingMergeTree(sign)
+ PARTITION BY toYYYYMM(start)
+ PRIMARY KEY (site_id, toDate(start), user_id, session_id)
+ ORDER BY (site_id, toDate(start), user_id, session_id)
+ <%= @table_settings %>
diff --git a/priv/data_migrations/NumericIDs/sql/drop-dict.sql.eex b/priv/data_migrations/NumericIDs/sql/drop-dict.sql.eex
new file mode 100644
index 0000000000..3e4f226437
--- /dev/null
+++ b/priv/data_migrations/NumericIDs/sql/drop-dict.sql.eex
@@ -0,0 +1 @@
+DROP DICTIONARY IF EXISTS domains_lookup SYNC
diff --git a/priv/data_migrations/NumericIDs/sql/drop-events-v2.sql.eex b/priv/data_migrations/NumericIDs/sql/drop-events-v2.sql.eex
new file mode 100644
index 0000000000..ae3927e49d
--- /dev/null
+++ b/priv/data_migrations/NumericIDs/sql/drop-events-v2.sql.eex
@@ -0,0 +1 @@
+DROP TABLE IF EXISTS events_v2 <%= if @cluster? do %>ON CLUSTER '{cluster}'<% end %> SYNC
diff --git a/priv/data_migrations/NumericIDs/sql/drop-sessions-v2.sql.eex b/priv/data_migrations/NumericIDs/sql/drop-sessions-v2.sql.eex
new file mode 100644
index 0000000000..eee7668420
--- /dev/null
+++ b/priv/data_migrations/NumericIDs/sql/drop-sessions-v2.sql.eex
@@ -0,0 +1 @@
+DROP TABLE IF EXISTS sessions_v2 <%= if @cluster? do %>ON CLUSTER '{cluster}'<% end %> SYNC
diff --git a/priv/data_migrations/NumericIDs/sql/drop-tmp-events-v2.sql.eex b/priv/data_migrations/NumericIDs/sql/drop-tmp-events-v2.sql.eex
new file mode 100644
index 0000000000..9ce47fbae7
--- /dev/null
+++ b/priv/data_migrations/NumericIDs/sql/drop-tmp-events-v2.sql.eex
@@ -0,0 +1 @@
+DROP TABLE IF EXISTS tmp_events_v2 SYNC
diff --git a/priv/data_migrations/NumericIDs/sql/drop-tmp-sessions-v2.sql.eex b/priv/data_migrations/NumericIDs/sql/drop-tmp-sessions-v2.sql.eex
new file mode 100644
index 0000000000..ea3666c511
--- /dev/null
+++ b/priv/data_migrations/NumericIDs/sql/drop-tmp-sessions-v2.sql.eex
@@ -0,0 +1 @@
+DROP TABLE IF EXISTS tmp_sessions_v2 SYNC
diff --git a/priv/data_migrations/NumericIDs/sql/insert-into-tmp-events-v2.sql.eex b/priv/data_migrations/NumericIDs/sql/insert-into-tmp-events-v2.sql.eex
new file mode 100644
index 0000000000..454ac09962
--- /dev/null
+++ b/priv/data_migrations/NumericIDs/sql/insert-into-tmp-events-v2.sql.eex
@@ -0,0 +1,54 @@
+INSERT INTO tmp_events_v2 (
+ site_id, timestamp, name, user_id,
+ session_id, hostname, pathname, referrer,
+ referrer_source, country_code, screen_size,
+ operating_system, browser, utm_medium,
+ utm_source, utm_campaign, meta.key,
+ meta.value, browser_version, operating_system_version,
+ subdivision1_code, subdivision2_code,
+ city_geoname_id, utm_content, utm_term,
+ transferred_from
+)
+SELECT
+ dictGet(
+ 'domains_lookup', 'site_id', domain
+ ),
+ timestamp,
+ name,
+ user_id,
+ session_id,
+ hostname,
+ pathname,
+ referrer,
+ referrer_source,
+ country_code,
+ screen_size,
+ operating_system,
+ browser,
+ utm_medium,
+ utm_source,
+ utm_campaign,
+ meta.key,
+ meta.value,
+ browser_version,
+ operating_system_version,
+ subdivision1_code,
+ subdivision2_code,
+ city_geoname_id,
+ utm_content,
+ utm_term,
+ transferred_from
+FROM
+ events
+WHERE
+ (
+ domain IN (
+ SELECT
+ domain
+ FROM
+ dictionary('domains_lookup')
+ WHERE
+ partition <= '<%= @partition %>'
+ )
+ )
+AND (_partition_id = '<%= @partition %>')
diff --git a/priv/data_migrations/NumericIDs/sql/insert-into-tmp-sessions-v2.sql.eex b/priv/data_migrations/NumericIDs/sql/insert-into-tmp-sessions-v2.sql.eex
new file mode 100644
index 0000000000..194e877af2
--- /dev/null
+++ b/priv/data_migrations/NumericIDs/sql/insert-into-tmp-sessions-v2.sql.eex
@@ -0,0 +1,30 @@
+INSERT INTO tmp_sessions_v2 (
+site_id, session_id, sign, user_id, hostname, timestamp, start, is_bounce, entry_page,
+exit_page, pageviews, events, duration, referrer, referrer_source, country_code,
+screen_size, operating_system, browser, utm_medium, utm_source, utm_campaign, browser_version,
+operating_system_version, subdivision1_code, subdivision2_code, city_geoname_id, utm_content,
+utm_term, transferred_from, entry_meta.key, entry_meta.value
+)
+SELECT
+ dictGet(
+ 'domains_lookup', 'site_id', domain
+ ),
+ session_id, sign, user_id, hostname, timestamp, start, is_bounce, entry_page,
+ exit_page, pageviews, events, duration, referrer, referrer_source, country_code,
+ screen_size, operating_system, browser, utm_medium, utm_source, utm_campaign, browser_version,
+ operating_system_version, subdivision1_code, subdivision2_code, city_geoname_id, utm_content,
+ utm_term, transferred_from, entry_meta.key, entry_meta.value
+FROM
+ sessions
+WHERE
+ (
+ domain IN (
+ SELECT
+ domain
+ FROM
+ dictionary('domains_lookup')
+ WHERE
+ partition <= '<%= @partition %>'
+ )
+ )
+AND (_partition_id = '<%= @partition %>')
diff --git a/priv/data_migrations/NumericIDs/sql/list-partitions.sql.eex b/priv/data_migrations/NumericIDs/sql/list-partitions.sql.eex
new file mode 100644
index 0000000000..c945e73764
--- /dev/null
+++ b/priv/data_migrations/NumericIDs/sql/list-partitions.sql.eex
@@ -0,0 +1,6 @@
+SELECT partition
+FROM system.parts
+WHERE table = 'events'
+<%= if @start_from do %>AND partition >= '<%= @start_from %>'<% end %>
+<%= if @stop_at do %>AND partition <= '<%= @stop_at %>'<% end %>
+ORDER BY partition
diff --git a/priv/data_migrations/NumericIDs/sql/truncate-tmp-events-v2.sql.eex b/priv/data_migrations/NumericIDs/sql/truncate-tmp-events-v2.sql.eex
new file mode 100644
index 0000000000..18e6fef98c
--- /dev/null
+++ b/priv/data_migrations/NumericIDs/sql/truncate-tmp-events-v2.sql.eex
@@ -0,0 +1 @@
+TRUNCATE TABLE tmp_events_v2
diff --git a/priv/data_migrations/NumericIDs/sql/truncate-tmp-sessions-v2.sql.eex b/priv/data_migrations/NumericIDs/sql/truncate-tmp-sessions-v2.sql.eex
new file mode 100644
index 0000000000..22b8bbddf9
--- /dev/null
+++ b/priv/data_migrations/NumericIDs/sql/truncate-tmp-sessions-v2.sql.eex
@@ -0,0 +1 @@
+TRUNCATE TABLE tmp_sessions_v2
diff --git a/priv/ingest_repo/migrations/20230320094327_create_v2_schemas.exs b/priv/ingest_repo/migrations/20230320094327_create_v2_schemas.exs
new file mode 100644
index 0000000000..a77591f64d
--- /dev/null
+++ b/priv/ingest_repo/migrations/20230320094327_create_v2_schemas.exs
@@ -0,0 +1,33 @@
+defmodule Plausible.IngestRepo.Migrations.CreateV2Schemas do
+ @moduledoc """
+ Normally, for live environments the migration will be done via
+ `DataMigration.NumericIDs` module (TBD). In which case PASS_V2_SCHEMA_MIGRATION
+ environment variable needs to be set, to only make the standard migrate
+ command write an entry into schema_migrations.
+
+ For tests, and entirely new small, self-hosted instances however,
+ we want to keep the ability of preparing the database without enforcing
+ any data migration.
+ """
+
+ use Ecto.Migration
+
+ use Plausible.DataMigration, dir: "NumericIDs"
+
+ @cluster? false
+ @settings "SETTINGS index_granularity = 8192"
+
+ def up do
+ if System.get_env("PASS_V2_SCHEMA_MIGRATION") do
+ :ok
+ else
+ execute unwrap("create-events-v2", table_settings: @settings, cluster?: @cluster?)
+ execute unwrap("create-sessions-v2", table_settings: @settings, cluster?: @cluster?)
+ end
+ end
+
+ def down do
+ execute unwrap("drop-events-v2", cluster?: @cluster?)
+ execute unwrap("drop-sessions-v2", cluster?: @cluster?)
+ end
+end