Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.
TypeScript analyticsanalytics-dashboardclickhouseelixirgoogle-analyticsmarketingopen-source-analyticsphoenixplausible-analyticspostgresqlprivacyprivacy-friendlyself-hostedsimple-analyticsstatisticstailwindcssweb-analyticswebsite-analyticswebsite-statswebsite-tracking
CLICKHOUSE_DEFAULT_STORAGE_POLICY env (#4851)
Karl-Aksel Puulmann2112feee8896
diff --git a/config/runtime.exs b/config/runtime.exsindex 7d6f5d4ce0..5069ce500e 100644--- a/config/runtime.exs+++ b/config/runtime.exs@@ -560,6 +560,9 @@ config :plausible, Plausible.IngestRepo,pool_size: ingest_pool_size,settings: [materialized_views_ignore_errors: 1+ ],+ table_settings: [+ storage_policy: get_var_from_path_or_env(config_dir, "CLICKHOUSE_DEFAULT_STORAGE_POLICY")]config :plausible, Plausible.AsyncInsertRepo,diff --git a/lib/plausible/data_migration/acquisition_channel.ex b/lib/plausible/data_migration/acquisition_channel.exindex 34dbe2c5e2..754a8dbe54 100644--- a/lib/plausible/data_migration/acquisition_channel.ex+++ b/lib/plausible/data_migration/acquisition_channel.ex@@ -25,6 +25,7 @@ defmodule Plausible.DataMigration.AcquisitionChannel do"acquisition_channel_functions",[on_cluster_statement: on_cluster_statement,+ table_settings: Plausible.MigrationUtils.table_settings_expr(),dictionary_connection_params: Plausible.MigrationUtils.dictionary_connection_params(),insert_quorum: insert_quorum],diff --git a/lib/plausible/data_migration/locations_sync.ex b/lib/plausible/data_migration/locations_sync.exindex 7ebba6098b..b2708e16ff 100644--- a/lib/plausible/data_migration/locations_sync.ex+++ b/lib/plausible/data_migration/locations_sync.ex@@ -91,7 +91,12 @@ defmodule Plausible.DataMigration.LocationsSync docluster? = Plausible.IngestRepo.clustered_table?("sessions_v2"){:ok, _} = run_sql("truncate-location-data-table", cluster?: cluster?)- {:ok, _} = run_sql("create-location-data-table", cluster?: cluster?)++ {:ok, _} =+ run_sql("create-location-data-table",+ cluster?: cluster?,+ table_settings: Plausible.MigrationUtils.table_settings_expr(:suffix)+ )countries =Location.Country.all()diff --git a/lib/plausible/data_migration/numeric_ids.ex b/lib/plausible/data_migration/numeric_ids.exindex 1980709ad8..43092d2694 100644--- a/lib/plausible/data_migration/numeric_ids.ex+++ b/lib/plausible/data_migration/numeric_ids.ex@@ -18,8 +18,6 @@ defmodule Plausible.DataMigration.NumericIDs doendend- @table_settings "SETTINGS index_granularity = 8192"-def run(opts \\ []) dointeractive? = Keyword.get(opts, :interactive?, true)@@ -34,7 +32,7 @@ defmodule Plausible.DataMigration.NumericIDs dotable_settings =Keyword.get(opts, :table_settings) || System.get_env("NUMERIC_IDS_TABLE_SETTINGS") ||- @table_settings+ Plausible.MigrationUtils.table_settings_expr()start_from =Keyword.get(opts, :start_from) || System.get_env("NUMERIC_IDS_PARTITION_START_FROM")diff --git a/lib/plausible/migration_utils.ex b/lib/plausible/migration_utils.exindex e477537e76..93cbca632e 100644--- a/lib/plausible/migration_utils.ex+++ b/lib/plausible/migration_utils.ex@@ -21,4 +21,23 @@ defmodule Plausible.MigrationUtils do|> Enum.reject(&is_nil/1)|> Enum.join(" ")end++ def table_settings() do+ IngestRepo.config()+ |> Keyword.get(:table_settings)+ |> Enum.reject(fn {_, v} -> is_nil(v) end)+ end++ def table_settings_expr(type \\ :prefix) do+ expr = Enum.map_join(table_settings(), ", ", fn {k, v} -> "#{k} = #{encode(v)}" end)++ case {table_settings(), type} do+ {[], _} -> ""+ {_, :prefix} -> "SETTINGS #{expr}"+ {_, :suffix} -> ", #{expr}"+ end+ end++ defp encode(value) when is_number(value), do: value+ defp encode(value) when is_binary(value), do: "'#{value}'"enddiff --git a/priv/data_migrations/AcquisitionChannel/sql/acquisition_channel_functions.sql.eex b/priv/data_migrations/AcquisitionChannel/sql/acquisition_channel_functions.sql.eexindex 742edca9fe..7287f7f57f 100644--- a/priv/data_migrations/AcquisitionChannel/sql/acquisition_channel_functions.sql.eex+++ b/priv/data_migrations/AcquisitionChannel/sql/acquisition_channel_functions.sql.eex@@ -9,7 +9,8 @@ ENGINE = ReplicatedMergeTree('/clickhouse/{cluster}/tables/{shard}/plausible_pro<% else %>ENGINE = MergeTree()<% end %>-ORDER BY referrer_source;+ORDER BY referrer_source+<%= @table_settings %>;TRUNCATE TABLE acquisition_channel_source_category SETTINGS alter_sync=2;@@ -41,7 +42,8 @@ ENGINE = ReplicatedMergeTree('/clickhouse/{cluster}/tables/{shard}/plausible_pro<% else %>ENGINE = MergeTree()<% end %>-ORDER BY referrer_source;+ORDER BY referrer_source+<%= @table_settings %>;TRUNCATE TABLE acquisition_channel_paid_sources SETTINGS alter_sync=2;diff --git a/priv/data_migrations/LocationsSync/sql/create-location-data-table.sql.eex b/priv/data_migrations/LocationsSync/sql/create-location-data-table.sql.eexindex 0487c5c7e9..f5e5593f75 100644--- a/priv/data_migrations/LocationsSync/sql/create-location-data-table.sql.eex+++ b/priv/data_migrations/LocationsSync/sql/create-location-data-table.sql.eex@@ -10,4 +10,4 @@ ENGINE = ReplicatedMergeTree('/clickhouse/{cluster}/tables/{shard}/plausible_proENGINE = MergeTree()<% end %>ORDER BY (type, id)-SETTINGS index_granularity = 128+SETTINGS index_granularity = 128 <%= @table_settings %>diff --git a/priv/ingest_repo/migrations/20200915070607_create_events_and_sessions.exs b/priv/ingest_repo/migrations/20200915070607_create_events_and_sessions.exsindex 6f329cdfb9..527612827a 100644--- a/priv/ingest_repo/migrations/20200915070607_create_events_and_sessions.exs+++ b/priv/ingest_repo/migrations/20200915070607_create_events_and_sessions.exs@@ -10,8 +10,11 @@ defmodule Plausible.ClickhouseRepo.Migrations.CreateEventsAndSessions docreate_if_not_exists table(:events,primary_key: false,engine: "MergeTree",- options:- "PARTITION BY toYYYYMM(timestamp) ORDER BY (domain, toDate(timestamp), user_id) SETTINGS index_granularity = 8192"+ options: """+ PARTITION BY toYYYYMM(timestamp)+ ORDER BY (domain, toDate(timestamp), user_id)+ #{Plausible.MigrationUtils.table_settings_expr()}+ """) doadd(:name, :string)add(:domain, :string)@@ -34,8 +37,11 @@ defmodule Plausible.ClickhouseRepo.Migrations.CreateEventsAndSessions docreate_if_not_exists table(:sessions,primary_key: false,engine: "CollapsingMergeTree(sign)",- options:- "PARTITION BY toYYYYMM(start) ORDER BY (domain, toDate(start), user_id, session_id) SETTINGS index_granularity = 8192"+ options: """+ PARTITION BY toYYYYMM(start)+ ORDER BY (domain, toDate(start), user_id, session_id)+ #{Plausible.MigrationUtils.table_settings_expr()}+ """) doadd(:session_id, :UInt64)add(:sign, :Int8)diff --git a/priv/ingest_repo/migrations/20211112130238_create_imported_tables.exs b/priv/ingest_repo/migrations/20211112130238_create_imported_tables.exsindex 5c7b72a875..cd1d827db5 100644--- a/priv/ingest_repo/migrations/20211112130238_create_imported_tables.exs+++ b/priv/ingest_repo/migrations/20211112130238_create_imported_tables.exs@@ -5,7 +5,10 @@ defmodule Plausible.ClickhouseRepo.Migrations.CreateImportedVisitors docreate_if_not_exists table(:imported_visitors,primary_key: false,engine: "MergeTree",- options: "ORDER BY (site_id, date)"+ options: """+ ORDER BY (site_id, date)+ #{Plausible.MigrationUtils.table_settings_expr()}+ """) doadd(:site_id, :UInt64)add(:date, :date)@@ -19,7 +22,10 @@ defmodule Plausible.ClickhouseRepo.Migrations.CreateImportedVisitors docreate_if_not_exists table(:imported_sources,primary_key: false,engine: "MergeTree",- options: "ORDER BY (site_id, date, source)"+ options: """+ ORDER BY (site_id, date, source)+ #{Plausible.MigrationUtils.table_settings_expr()}+ """) doadd(:site_id, :UInt64)add(:date, :date)@@ -37,7 +43,10 @@ defmodule Plausible.ClickhouseRepo.Migrations.CreateImportedVisitors docreate_if_not_exists table(:imported_pages,primary_key: false,engine: "MergeTree",- options: "ORDER BY (site_id, date, hostname, page)"+ options: """+ ORDER BY (site_id, date, hostname, page)+ #{Plausible.MigrationUtils.table_settings_expr()}+ """) doadd(:site_id, :UInt64)add(:date, :date)@@ -52,7 +61,10 @@ defmodule Plausible.ClickhouseRepo.Migrations.CreateImportedVisitors docreate_if_not_exists table(:imported_entry_pages,primary_key: false,engine: "MergeTree",- options: "ORDER BY (site_id, date, entry_page)"+ options: """+ ORDER BY (site_id, date, entry_page)+ #{Plausible.MigrationUtils.table_settings_expr()}+ """) doadd(:site_id, :UInt64)add(:date, :date)@@ -66,7 +78,10 @@ defmodule Plausible.ClickhouseRepo.Migrations.CreateImportedVisitors docreate_if_not_exists table(:imported_exit_pages,primary_key: false,engine: "MergeTree",- options: "ORDER BY (site_id, date, exit_page)"+ options: """+ ORDER BY (site_id, date, exit_page)+ #{Plausible.MigrationUtils.table_settings_expr()}+ """) doadd(:site_id, :UInt64)add(:date, :date)@@ -78,7 +93,10 @@ defmodule Plausible.ClickhouseRepo.Migrations.CreateImportedVisitors docreate_if_not_exists table(:imported_locations,primary_key: false,engine: "MergeTree",- options: "ORDER BY (site_id, date, country, region, city)"+ options: """+ ORDER BY (site_id, date, country, region, city)+ #{Plausible.MigrationUtils.table_settings_expr()}+ """) doadd(:site_id, :UInt64)add(:date, :date)@@ -94,7 +112,10 @@ defmodule Plausible.ClickhouseRepo.Migrations.CreateImportedVisitors docreate_if_not_exists table(:imported_devices,primary_key: false,engine: "MergeTree",- options: "ORDER BY (site_id, date, device)"+ options: """+ ORDER BY (site_id, date, device)+ #{Plausible.MigrationUtils.table_settings_expr()}+ """) doadd(:site_id, :UInt64)add(:date, :date)@@ -108,7 +129,10 @@ defmodule Plausible.ClickhouseRepo.Migrations.CreateImportedVisitors docreate_if_not_exists table(:imported_browsers,primary_key: false,engine: "MergeTree",- options: "ORDER BY (site_id, date, browser)"+ options: """+ ORDER BY (site_id, date, browser)+ #{Plausible.MigrationUtils.table_settings_expr()}+ """) doadd(:site_id, :UInt64)add(:date, :date)@@ -122,7 +146,10 @@ defmodule Plausible.ClickhouseRepo.Migrations.CreateImportedVisitors docreate_if_not_exists table(:imported_operating_systems,primary_key: false,engine: "MergeTree",- options: "ORDER BY (site_id, date, operating_system)"+ options: """+ ORDER BY (site_id, date, operating_system)+ #{Plausible.MigrationUtils.table_settings_expr()}+ """) doadd(:site_id, :UInt64)add(:date, :date)diff --git a/priv/ingest_repo/migrations/20230214114402_create_ingest_counters_table.exs b/priv/ingest_repo/migrations/20230214114402_create_ingest_counters_table.exsindex 75596571c0..d1e2b4a9f1 100644--- a/priv/ingest_repo/migrations/20230214114402_create_ingest_counters_table.exs+++ b/priv/ingest_repo/migrations/20230214114402_create_ingest_counters_table.exs@@ -5,8 +5,10 @@ defmodule Plausible.IngestRepo.Migrations.CreateIngestCountersTable docreate_if_not_exists table(:ingest_counters,primary_key: false,engine: "SummingMergeTree(value)",- options:- "ORDER BY (domain, toDate(event_timebucket), metric, toStartOfMinute(event_timebucket))"+ options: """+ ORDER BY (domain, toDate(event_timebucket), metric, toStartOfMinute(event_timebucket))+ #{Plausible.MigrationUtils.table_settings_expr()}+ """) doadd(:event_timebucket, :utc_datetime)add(:domain, :"LowCardinality(String)")diff --git a/priv/ingest_repo/migrations/20230320094327_create_v2_schemas.exs b/priv/ingest_repo/migrations/20230320094327_create_v2_schemas.exsindex b881f5eae6..00de3b0c3d 100644--- a/priv/ingest_repo/migrations/20230320094327_create_v2_schemas.exs+++ b/priv/ingest_repo/migrations/20230320094327_create_v2_schemas.exs@@ -1,10 +1,10 @@defmodule Plausible.IngestRepo.Migrations.CreateV2Schemas do@moduledoc """Normally, for live environments the migration will be done via- `DataMigration.NumericIDs` module.+ `DataMigration.NumericIDs` module.- For tests, and entirely new small, self-hosted instances however,- we want to keep the ability of preparing the database without enforcing+ For tests, and entirely new small, self-hosted instances however,+ we want to keep the ability of preparing the database without enforcingany _data_ migration."""@@ -13,11 +13,12 @@ defmodule Plausible.IngestRepo.Migrations.CreateV2Schemas douse Plausible.DataMigration, dir: "NumericIDs"@cluster? false- @settings "SETTINGS index_granularity = 8192"def up do- execute unwrap("create-events-v2", table_settings: @settings, cluster?: @cluster?)- execute unwrap("create-sessions-v2", table_settings: @settings, cluster?: @cluster?)+ table_settings = Plausible.MigrationUtils.table_settings_expr()++ execute unwrap("create-events-v2", table_settings: table_settings, cluster?: @cluster?)+ execute unwrap("create-sessions-v2", table_settings: table_settings, cluster?: @cluster?)enddef down dodiff --git a/priv/ingest_repo/migrations/20240423094014_add_imported_custom_events.exs b/priv/ingest_repo/migrations/20240423094014_add_imported_custom_events.exsindex 0cc2f76b7f..549f86de4f 100644--- a/priv/ingest_repo/migrations/20240423094014_add_imported_custom_events.exs+++ b/priv/ingest_repo/migrations/20240423094014_add_imported_custom_events.exs@@ -4,19 +4,20 @@ defmodule Plausible.IngestRepo.Migrations.AddImportedCustomEvents dodef change do# NOTE: Using another table for determining cluster presenceon_cluster = Plausible.MigrationUtils.on_cluster_statement("imported_pages")+ table_settings = Plausible.MigrationUtils.table_settings_expr(:suffix)settings =if Plausible.IngestRepo.clustered_table?("imported_pages") do"""ENGINE = ReplicatedMergeTree('/clickhouse/{cluster}/tables/{shard}/{database}/imported_custom_events', '{replica}')ORDER BY (site_id, import_id, date, name)- SETTINGS replicated_deduplication_window = 0, storage_policy = 'tiered'+ SETTINGS replicated_deduplication_window = 0 #{table_settings}"""else"""ENGINE = MergeTree()ORDER BY (site_id, import_id, date, name)- SETTINGS replicated_deduplication_window = 0+ SETTINGS replicated_deduplication_window = 0 #{table_settings}"""end