Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.
TypeScript analyticsanalytics-dashboardclickhouseelixirgoogle-analyticsmarketingopen-source-analyticsphoenixplausible-analyticspostgresqlprivacyprivacy-friendlyself-hostedsimple-analyticsstatisticstailwindcssweb-analyticswebsite-analyticswebsite-statswebsite-tracking
Make ClickhouseRepo query timeout and max_execution_time configurable (#6691)
Krzysztof Jabłoński918fa6a49300
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex a6c2a58a15..4989d898e7 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -6,6 +6,7 @@ All notable changes to this project will be documented in this file.### Added+- Add `CLICKHOUSE_QUERY_TIMEOUT_MS` and `CLICKHOUSE_MAX_EXECUTION_TIME_SEC` env vars to make the stats query timeout (default `15000`) and ClickHouse `max_execution_time` (default: timeout in seconds + 5) configurable plausible/community-edition#277- Annotations feature- Pages report can now be broken down by URL (hostname + path) in addition to path only- New "AI Assistants" acquisition channel + improved recognized sources databasediff --git a/config/runtime.exs b/config/runtime.exsindex d16b986089..eee544da06 100644--- a/config/runtime.exs+++ b/config/runtime.exs@@ -145,6 +145,18 @@ end|> get_var_from_path_or_env("CLICKHOUSE_MAX_BUFFER_SIZE_BYTES", "100000")|> Integer.parse()+ch_query_timeout_ms = get_int_from_path_or_env(config_dir, "CLICKHOUSE_QUERY_TIMEOUT_MS", 15_000)++# Server-side ClickHouse limit for stats queries. Kept slightly above the+# client-side timeout by default (15s/20s), so that ClickHouse stops queries+# the client has already given up on.+ch_max_execution_time_sec =+ get_int_from_path_or_env(+ config_dir,+ "CLICKHOUSE_MAX_EXECUTION_TIME_SEC",+ div(ch_query_timeout_ms, 1000) + 5+ )+persistor_backend =case get_var_from_path_or_env(config_dir, "PERSISTOR_BACKEND", "embedded") do"embedded" -> Plausible.Ingestion.Persistor.Embedded@@ -667,7 +679,7 @@ ch_query_condition_cache_settings =config :plausible, Plausible.ClickhouseRepo,queue_target: 500,queue_interval: 2000,- timeout: 15_000,+ timeout: ch_query_timeout_ms,url: ch_db_url,transport_opts: ch_transport_opts,settings:@@ -676,11 +688,10 @@ config :plausible, Plausible.ClickhouseRepo,join_algorithm: "direct,parallel_hash,hash",# stops queries when ClickhouseRepo connection :timeout value reachedcancel_http_readonly_queries_on_client_close: 1,- # stops queries when they will likely take over 20s- # NB! when :timeout is overridden to be over 20s,- # for it to have meaningful effect,- # this must be overridden as well- max_execution_time: 20+ # stops queries when they will likely take over+ # CLICKHOUSE_MAX_EXECUTION_TIME_SEC seconds, which defaults to+ # CLICKHOUSE_QUERY_TIMEOUT_MS / 1000 + 5 (20s)+ max_execution_time: ch_max_execution_time_sec] ++ ch_query_condition_cache_settingsconfig :plausible, Plausible.IngestRepo,diff --git a/test/plausible/config_test.exs b/test/plausible/config_test.exsindex 89c9523da6..63948915d2 100644--- a/test/plausible/config_test.exs+++ b/test/plausible/config_test.exs@@ -709,6 +709,55 @@ defmodule Plausible.ConfigTest doendend+ describe "clickhouse" do+ test "default query timeout and max_execution_time" do+ env = [+ {"CLICKHOUSE_QUERY_TIMEOUT_MS", nil},+ {"CLICKHOUSE_MAX_EXECUTION_TIME_SEC", nil}+ ]++ config = runtime_config(env)+ repo_config = get_in(config, [:plausible, Plausible.ClickhouseRepo])++ assert repo_config[:timeout] == 15_000+ assert repo_config[:settings][:max_execution_time] == 20+ end++ test "CLICKHOUSE_QUERY_TIMEOUT_MS overrides timeout and derives max_execution_time" do+ env = [+ {"CLICKHOUSE_QUERY_TIMEOUT_MS", "60000"},+ {"CLICKHOUSE_MAX_EXECUTION_TIME_SEC", nil}+ ]++ config = runtime_config(env)+ repo_config = get_in(config, [:plausible, Plausible.ClickhouseRepo])++ assert repo_config[:timeout] == 60_000+ assert repo_config[:settings][:max_execution_time] == 65+ end++ test "CLICKHOUSE_MAX_EXECUTION_TIME_SEC overrides derived max_execution_time" do+ env = [+ {"CLICKHOUSE_QUERY_TIMEOUT_MS", "60000"},+ {"CLICKHOUSE_MAX_EXECUTION_TIME_SEC", "120"}+ ]++ config = runtime_config(env)+ repo_config = get_in(config, [:plausible, Plausible.ClickhouseRepo])++ assert repo_config[:timeout] == 60_000+ assert repo_config[:settings][:max_execution_time] == 120+ end++ test "CLICKHOUSE_QUERY_TIMEOUT_MS must be an integer" do+ env = [{"CLICKHOUSE_QUERY_TIMEOUT_MS", "15s"}]++ assert_raise RuntimeError, ~r/CLICKHOUSE_QUERY_TIMEOUT_MS must be an integer/, fn ->+ runtime_config(env)+ end+ end+ end+defp runtime_config(env) doput_system_env_undo(env)Config.Reader.read!("config/runtime.exs", env: :prod)