Skip to content

Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.

Make ClickhouseRepo query timeout and max_execution_time configurable (#6691)

Krzysztof Jabłoński918fa6a49300
diff --git a/CHANGELOG.md b/CHANGELOG.md
index a6c2a58a15..4989d898e7 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -6,6 +6,7 @@ All notable changes to this project will be documented in this file.
### Added
+- Add `CLICKHOUSE_QUERY_TIMEOUT_MS` and `CLICKHOUSE_MAX_EXECUTION_TIME_SEC` env vars to make the stats query timeout (default `15000`) and ClickHouse `max_execution_time` (default: timeout in seconds + 5) configurable plausible/community-edition#277
- Annotations feature
- Pages report can now be broken down by URL (hostname + path) in addition to path only
- New "AI Assistants" acquisition channel + improved recognized sources database
diff --git a/config/runtime.exs b/config/runtime.exs
index d16b986089..eee544da06 100644
--- a/config/runtime.exs
+++ b/config/runtime.exs
@@ -145,6 +145,18 @@ end
|> get_var_from_path_or_env("CLICKHOUSE_MAX_BUFFER_SIZE_BYTES", "100000")
|> Integer.parse()
+ch_query_timeout_ms = get_int_from_path_or_env(config_dir, "CLICKHOUSE_QUERY_TIMEOUT_MS", 15_000)
+
+# Server-side ClickHouse limit for stats queries. Kept slightly above the
+# client-side timeout by default (15s/20s), so that ClickHouse stops queries
+# the client has already given up on.
+ch_max_execution_time_sec =
+ get_int_from_path_or_env(
+ config_dir,
+ "CLICKHOUSE_MAX_EXECUTION_TIME_SEC",
+ div(ch_query_timeout_ms, 1000) + 5
+ )
+
persistor_backend =
case get_var_from_path_or_env(config_dir, "PERSISTOR_BACKEND", "embedded") do
"embedded" -> Plausible.Ingestion.Persistor.Embedded
@@ -667,7 +679,7 @@ ch_query_condition_cache_settings =
config :plausible, Plausible.ClickhouseRepo,
queue_target: 500,
queue_interval: 2000,
- timeout: 15_000,
+ timeout: ch_query_timeout_ms,
url: ch_db_url,
transport_opts: ch_transport_opts,
settings:
@@ -676,11 +688,10 @@ config :plausible, Plausible.ClickhouseRepo,
join_algorithm: "direct,parallel_hash,hash",
# stops queries when ClickhouseRepo connection :timeout value reached
cancel_http_readonly_queries_on_client_close: 1,
- # stops queries when they will likely take over 20s
- # NB! when :timeout is overridden to be over 20s,
- # for it to have meaningful effect,
- # this must be overridden as well
- max_execution_time: 20
+ # stops queries when they will likely take over
+ # CLICKHOUSE_MAX_EXECUTION_TIME_SEC seconds, which defaults to
+ # CLICKHOUSE_QUERY_TIMEOUT_MS / 1000 + 5 (20s)
+ max_execution_time: ch_max_execution_time_sec
] ++ ch_query_condition_cache_settings
config :plausible, Plausible.IngestRepo,
diff --git a/test/plausible/config_test.exs b/test/plausible/config_test.exs
index 89c9523da6..63948915d2 100644
--- a/test/plausible/config_test.exs
+++ b/test/plausible/config_test.exs
@@ -709,6 +709,55 @@ defmodule Plausible.ConfigTest do
end
end
+ describe "clickhouse" do
+ test "default query timeout and max_execution_time" do
+ env = [
+ {"CLICKHOUSE_QUERY_TIMEOUT_MS", nil},
+ {"CLICKHOUSE_MAX_EXECUTION_TIME_SEC", nil}
+ ]
+
+ config = runtime_config(env)
+ repo_config = get_in(config, [:plausible, Plausible.ClickhouseRepo])
+
+ assert repo_config[:timeout] == 15_000
+ assert repo_config[:settings][:max_execution_time] == 20
+ end
+
+ test "CLICKHOUSE_QUERY_TIMEOUT_MS overrides timeout and derives max_execution_time" do
+ env = [
+ {"CLICKHOUSE_QUERY_TIMEOUT_MS", "60000"},
+ {"CLICKHOUSE_MAX_EXECUTION_TIME_SEC", nil}
+ ]
+
+ config = runtime_config(env)
+ repo_config = get_in(config, [:plausible, Plausible.ClickhouseRepo])
+
+ assert repo_config[:timeout] == 60_000
+ assert repo_config[:settings][:max_execution_time] == 65
+ end
+
+ test "CLICKHOUSE_MAX_EXECUTION_TIME_SEC overrides derived max_execution_time" do
+ env = [
+ {"CLICKHOUSE_QUERY_TIMEOUT_MS", "60000"},
+ {"CLICKHOUSE_MAX_EXECUTION_TIME_SEC", "120"}
+ ]
+
+ config = runtime_config(env)
+ repo_config = get_in(config, [:plausible, Plausible.ClickhouseRepo])
+
+ assert repo_config[:timeout] == 60_000
+ assert repo_config[:settings][:max_execution_time] == 120
+ end
+
+ test "CLICKHOUSE_QUERY_TIMEOUT_MS must be an integer" do
+ env = [{"CLICKHOUSE_QUERY_TIMEOUT_MS", "15s"}]
+
+ assert_raise RuntimeError, ~r/CLICKHOUSE_QUERY_TIMEOUT_MS must be an integer/, fn ->
+ runtime_config(env)
+ end
+ end
+ end
+
defp runtime_config(env) do
put_system_env_undo(env)
Config.Reader.read!("config/runtime.exs", env: :prod)