Skip to content

Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.

CLICKHOUSE_DEFAULT_STORAGE_POLICY env (#4851)

Karl-Aksel Puulmann2112feee8896
diff --git a/config/runtime.exs b/config/runtime.exs
index 7d6f5d4ce0..5069ce500e 100644
--- a/config/runtime.exs
+++ b/config/runtime.exs
@@ -560,6 +560,9 @@ config :plausible, Plausible.IngestRepo,
pool_size: ingest_pool_size,
settings: [
materialized_views_ignore_errors: 1
+ ],
+ table_settings: [
+ storage_policy: get_var_from_path_or_env(config_dir, "CLICKHOUSE_DEFAULT_STORAGE_POLICY")
]
config :plausible, Plausible.AsyncInsertRepo,
diff --git a/lib/plausible/data_migration/acquisition_channel.ex b/lib/plausible/data_migration/acquisition_channel.ex
index 34dbe2c5e2..754a8dbe54 100644
--- a/lib/plausible/data_migration/acquisition_channel.ex
+++ b/lib/plausible/data_migration/acquisition_channel.ex
@@ -25,6 +25,7 @@ defmodule Plausible.DataMigration.AcquisitionChannel do
"acquisition_channel_functions",
[
on_cluster_statement: on_cluster_statement,
+ table_settings: Plausible.MigrationUtils.table_settings_expr(),
dictionary_connection_params: Plausible.MigrationUtils.dictionary_connection_params(),
insert_quorum: insert_quorum
],
diff --git a/lib/plausible/data_migration/locations_sync.ex b/lib/plausible/data_migration/locations_sync.ex
index 7ebba6098b..b2708e16ff 100644
--- a/lib/plausible/data_migration/locations_sync.ex
+++ b/lib/plausible/data_migration/locations_sync.ex
@@ -91,7 +91,12 @@ defmodule Plausible.DataMigration.LocationsSync do
cluster? = Plausible.IngestRepo.clustered_table?("sessions_v2")
{:ok, _} = run_sql("truncate-location-data-table", cluster?: cluster?)
- {:ok, _} = run_sql("create-location-data-table", cluster?: cluster?)
+
+ {:ok, _} =
+ run_sql("create-location-data-table",
+ cluster?: cluster?,
+ table_settings: Plausible.MigrationUtils.table_settings_expr(:suffix)
+ )
countries =
Location.Country.all()
diff --git a/lib/plausible/data_migration/numeric_ids.ex b/lib/plausible/data_migration/numeric_ids.ex
index 1980709ad8..43092d2694 100644
--- a/lib/plausible/data_migration/numeric_ids.ex
+++ b/lib/plausible/data_migration/numeric_ids.ex
@@ -18,8 +18,6 @@ defmodule Plausible.DataMigration.NumericIDs do
end
end
- @table_settings "SETTINGS index_granularity = 8192"
-
def run(opts \\ []) do
interactive? = Keyword.get(opts, :interactive?, true)
@@ -34,7 +32,7 @@ defmodule Plausible.DataMigration.NumericIDs do
table_settings =
Keyword.get(opts, :table_settings) || System.get_env("NUMERIC_IDS_TABLE_SETTINGS") ||
- @table_settings
+ Plausible.MigrationUtils.table_settings_expr()
start_from =
Keyword.get(opts, :start_from) || System.get_env("NUMERIC_IDS_PARTITION_START_FROM")
diff --git a/lib/plausible/migration_utils.ex b/lib/plausible/migration_utils.ex
index e477537e76..93cbca632e 100644
--- a/lib/plausible/migration_utils.ex
+++ b/lib/plausible/migration_utils.ex
@@ -21,4 +21,23 @@ defmodule Plausible.MigrationUtils do
|> Enum.reject(&is_nil/1)
|> Enum.join(" ")
end
+
+ def table_settings() do
+ IngestRepo.config()
+ |> Keyword.get(:table_settings)
+ |> Enum.reject(fn {_, v} -> is_nil(v) end)
+ end
+
+ def table_settings_expr(type \\ :prefix) do
+ expr = Enum.map_join(table_settings(), ", ", fn {k, v} -> "#{k} = #{encode(v)}" end)
+
+ case {table_settings(), type} do
+ {[], _} -> ""
+ {_, :prefix} -> "SETTINGS #{expr}"
+ {_, :suffix} -> ", #{expr}"
+ end
+ end
+
+ defp encode(value) when is_number(value), do: value
+ defp encode(value) when is_binary(value), do: "'#{value}'"
end
diff --git a/priv/data_migrations/AcquisitionChannel/sql/acquisition_channel_functions.sql.eex b/priv/data_migrations/AcquisitionChannel/sql/acquisition_channel_functions.sql.eex
index 742edca9fe..7287f7f57f 100644
--- a/priv/data_migrations/AcquisitionChannel/sql/acquisition_channel_functions.sql.eex
+++ b/priv/data_migrations/AcquisitionChannel/sql/acquisition_channel_functions.sql.eex
@@ -9,7 +9,8 @@ ENGINE = ReplicatedMergeTree('/clickhouse/{cluster}/tables/{shard}/plausible_pro
<% else %>
ENGINE = MergeTree()
<% end %>
-ORDER BY referrer_source;
+ORDER BY referrer_source
+<%= @table_settings %>;
TRUNCATE TABLE acquisition_channel_source_category SETTINGS alter_sync=2;
@@ -41,7 +42,8 @@ ENGINE = ReplicatedMergeTree('/clickhouse/{cluster}/tables/{shard}/plausible_pro
<% else %>
ENGINE = MergeTree()
<% end %>
-ORDER BY referrer_source;
+ORDER BY referrer_source
+<%= @table_settings %>;
TRUNCATE TABLE acquisition_channel_paid_sources SETTINGS alter_sync=2;
diff --git a/priv/data_migrations/LocationsSync/sql/create-location-data-table.sql.eex b/priv/data_migrations/LocationsSync/sql/create-location-data-table.sql.eex
index 0487c5c7e9..f5e5593f75 100644
--- a/priv/data_migrations/LocationsSync/sql/create-location-data-table.sql.eex
+++ b/priv/data_migrations/LocationsSync/sql/create-location-data-table.sql.eex
@@ -10,4 +10,4 @@ ENGINE = ReplicatedMergeTree('/clickhouse/{cluster}/tables/{shard}/plausible_pro
ENGINE = MergeTree()
<% end %>
ORDER BY (type, id)
-SETTINGS index_granularity = 128
+SETTINGS index_granularity = 128 <%= @table_settings %>
diff --git a/priv/ingest_repo/migrations/20200915070607_create_events_and_sessions.exs b/priv/ingest_repo/migrations/20200915070607_create_events_and_sessions.exs
index 6f329cdfb9..527612827a 100644
--- a/priv/ingest_repo/migrations/20200915070607_create_events_and_sessions.exs
+++ b/priv/ingest_repo/migrations/20200915070607_create_events_and_sessions.exs
@@ -10,8 +10,11 @@ defmodule Plausible.ClickhouseRepo.Migrations.CreateEventsAndSessions do
create_if_not_exists table(:events,
primary_key: false,
engine: "MergeTree",
- options:
- "PARTITION BY toYYYYMM(timestamp) ORDER BY (domain, toDate(timestamp), user_id) SETTINGS index_granularity = 8192"
+ options: """
+ PARTITION BY toYYYYMM(timestamp)
+ ORDER BY (domain, toDate(timestamp), user_id)
+ #{Plausible.MigrationUtils.table_settings_expr()}
+ """
) do
add(:name, :string)
add(:domain, :string)
@@ -34,8 +37,11 @@ defmodule Plausible.ClickhouseRepo.Migrations.CreateEventsAndSessions do
create_if_not_exists table(:sessions,
primary_key: false,
engine: "CollapsingMergeTree(sign)",
- options:
- "PARTITION BY toYYYYMM(start) ORDER BY (domain, toDate(start), user_id, session_id) SETTINGS index_granularity = 8192"
+ options: """
+ PARTITION BY toYYYYMM(start)
+ ORDER BY (domain, toDate(start), user_id, session_id)
+ #{Plausible.MigrationUtils.table_settings_expr()}
+ """
) do
add(:session_id, :UInt64)
add(:sign, :Int8)
diff --git a/priv/ingest_repo/migrations/20211112130238_create_imported_tables.exs b/priv/ingest_repo/migrations/20211112130238_create_imported_tables.exs
index 5c7b72a875..cd1d827db5 100644
--- a/priv/ingest_repo/migrations/20211112130238_create_imported_tables.exs
+++ b/priv/ingest_repo/migrations/20211112130238_create_imported_tables.exs
@@ -5,7 +5,10 @@ defmodule Plausible.ClickhouseRepo.Migrations.CreateImportedVisitors do
create_if_not_exists table(:imported_visitors,
primary_key: false,
engine: "MergeTree",
- options: "ORDER BY (site_id, date)"
+ options: """
+ ORDER BY (site_id, date)
+ #{Plausible.MigrationUtils.table_settings_expr()}
+ """
) do
add(:site_id, :UInt64)
add(:date, :date)
@@ -19,7 +22,10 @@ defmodule Plausible.ClickhouseRepo.Migrations.CreateImportedVisitors do
create_if_not_exists table(:imported_sources,
primary_key: false,
engine: "MergeTree",
- options: "ORDER BY (site_id, date, source)"
+ options: """
+ ORDER BY (site_id, date, source)
+ #{Plausible.MigrationUtils.table_settings_expr()}
+ """
) do
add(:site_id, :UInt64)
add(:date, :date)
@@ -37,7 +43,10 @@ defmodule Plausible.ClickhouseRepo.Migrations.CreateImportedVisitors do
create_if_not_exists table(:imported_pages,
primary_key: false,
engine: "MergeTree",
- options: "ORDER BY (site_id, date, hostname, page)"
+ options: """
+ ORDER BY (site_id, date, hostname, page)
+ #{Plausible.MigrationUtils.table_settings_expr()}
+ """
) do
add(:site_id, :UInt64)
add(:date, :date)
@@ -52,7 +61,10 @@ defmodule Plausible.ClickhouseRepo.Migrations.CreateImportedVisitors do
create_if_not_exists table(:imported_entry_pages,
primary_key: false,
engine: "MergeTree",
- options: "ORDER BY (site_id, date, entry_page)"
+ options: """
+ ORDER BY (site_id, date, entry_page)
+ #{Plausible.MigrationUtils.table_settings_expr()}
+ """
) do
add(:site_id, :UInt64)
add(:date, :date)
@@ -66,7 +78,10 @@ defmodule Plausible.ClickhouseRepo.Migrations.CreateImportedVisitors do
create_if_not_exists table(:imported_exit_pages,
primary_key: false,
engine: "MergeTree",
- options: "ORDER BY (site_id, date, exit_page)"
+ options: """
+ ORDER BY (site_id, date, exit_page)
+ #{Plausible.MigrationUtils.table_settings_expr()}
+ """
) do
add(:site_id, :UInt64)
add(:date, :date)
@@ -78,7 +93,10 @@ defmodule Plausible.ClickhouseRepo.Migrations.CreateImportedVisitors do
create_if_not_exists table(:imported_locations,
primary_key: false,
engine: "MergeTree",
- options: "ORDER BY (site_id, date, country, region, city)"
+ options: """
+ ORDER BY (site_id, date, country, region, city)
+ #{Plausible.MigrationUtils.table_settings_expr()}
+ """
) do
add(:site_id, :UInt64)
add(:date, :date)
@@ -94,7 +112,10 @@ defmodule Plausible.ClickhouseRepo.Migrations.CreateImportedVisitors do
create_if_not_exists table(:imported_devices,
primary_key: false,
engine: "MergeTree",
- options: "ORDER BY (site_id, date, device)"
+ options: """
+ ORDER BY (site_id, date, device)
+ #{Plausible.MigrationUtils.table_settings_expr()}
+ """
) do
add(:site_id, :UInt64)
add(:date, :date)
@@ -108,7 +129,10 @@ defmodule Plausible.ClickhouseRepo.Migrations.CreateImportedVisitors do
create_if_not_exists table(:imported_browsers,
primary_key: false,
engine: "MergeTree",
- options: "ORDER BY (site_id, date, browser)"
+ options: """
+ ORDER BY (site_id, date, browser)
+ #{Plausible.MigrationUtils.table_settings_expr()}
+ """
) do
add(:site_id, :UInt64)
add(:date, :date)
@@ -122,7 +146,10 @@ defmodule Plausible.ClickhouseRepo.Migrations.CreateImportedVisitors do
create_if_not_exists table(:imported_operating_systems,
primary_key: false,
engine: "MergeTree",
- options: "ORDER BY (site_id, date, operating_system)"
+ options: """
+ ORDER BY (site_id, date, operating_system)
+ #{Plausible.MigrationUtils.table_settings_expr()}
+ """
) do
add(:site_id, :UInt64)
add(:date, :date)
diff --git a/priv/ingest_repo/migrations/20230214114402_create_ingest_counters_table.exs b/priv/ingest_repo/migrations/20230214114402_create_ingest_counters_table.exs
index 75596571c0..d1e2b4a9f1 100644
--- a/priv/ingest_repo/migrations/20230214114402_create_ingest_counters_table.exs
+++ b/priv/ingest_repo/migrations/20230214114402_create_ingest_counters_table.exs
@@ -5,8 +5,10 @@ defmodule Plausible.IngestRepo.Migrations.CreateIngestCountersTable do
create_if_not_exists table(:ingest_counters,
primary_key: false,
engine: "SummingMergeTree(value)",
- options:
- "ORDER BY (domain, toDate(event_timebucket), metric, toStartOfMinute(event_timebucket))"
+ options: """
+ ORDER BY (domain, toDate(event_timebucket), metric, toStartOfMinute(event_timebucket))
+ #{Plausible.MigrationUtils.table_settings_expr()}
+ """
) do
add(:event_timebucket, :utc_datetime)
add(:domain, :"LowCardinality(String)")
diff --git a/priv/ingest_repo/migrations/20230320094327_create_v2_schemas.exs b/priv/ingest_repo/migrations/20230320094327_create_v2_schemas.exs
index b881f5eae6..00de3b0c3d 100644
--- a/priv/ingest_repo/migrations/20230320094327_create_v2_schemas.exs
+++ b/priv/ingest_repo/migrations/20230320094327_create_v2_schemas.exs
@@ -1,10 +1,10 @@
defmodule Plausible.IngestRepo.Migrations.CreateV2Schemas do
@moduledoc """
Normally, for live environments the migration will be done via
- `DataMigration.NumericIDs` module.
+ `DataMigration.NumericIDs` module.
- For tests, and entirely new small, self-hosted instances however,
- we want to keep the ability of preparing the database without enforcing
+ For tests, and entirely new small, self-hosted instances however,
+ we want to keep the ability of preparing the database without enforcing
any _data_ migration.
"""
@@ -13,11 +13,12 @@ defmodule Plausible.IngestRepo.Migrations.CreateV2Schemas do
use Plausible.DataMigration, dir: "NumericIDs"
@cluster? false
- @settings "SETTINGS index_granularity = 8192"
def up do
- execute unwrap("create-events-v2", table_settings: @settings, cluster?: @cluster?)
- execute unwrap("create-sessions-v2", table_settings: @settings, cluster?: @cluster?)
+ table_settings = Plausible.MigrationUtils.table_settings_expr()
+
+ execute unwrap("create-events-v2", table_settings: table_settings, cluster?: @cluster?)
+ execute unwrap("create-sessions-v2", table_settings: table_settings, cluster?: @cluster?)
end
def down do
diff --git a/priv/ingest_repo/migrations/20240423094014_add_imported_custom_events.exs b/priv/ingest_repo/migrations/20240423094014_add_imported_custom_events.exs
index 0cc2f76b7f..549f86de4f 100644
--- a/priv/ingest_repo/migrations/20240423094014_add_imported_custom_events.exs
+++ b/priv/ingest_repo/migrations/20240423094014_add_imported_custom_events.exs
@@ -4,19 +4,20 @@ defmodule Plausible.IngestRepo.Migrations.AddImportedCustomEvents do
def change do
# NOTE: Using another table for determining cluster presence
on_cluster = Plausible.MigrationUtils.on_cluster_statement("imported_pages")
+ table_settings = Plausible.MigrationUtils.table_settings_expr(:suffix)
settings =
if Plausible.IngestRepo.clustered_table?("imported_pages") do
"""
ENGINE = ReplicatedMergeTree('/clickhouse/{cluster}/tables/{shard}/{database}/imported_custom_events', '{replica}')
ORDER BY (site_id, import_id, date, name)
- SETTINGS replicated_deduplication_window = 0, storage_policy = 'tiered'
+ SETTINGS replicated_deduplication_window = 0 #{table_settings}
"""
else
"""
ENGINE = MergeTree()
ORDER BY (site_id, import_id, date, name)
- SETTINGS replicated_deduplication_window = 0
+ SETTINGS replicated_deduplication_window = 0 #{table_settings}
"""
end