Skip to content

Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.

Limit ClickhouseRepo queries max_execution_time (#6018)

Artur Pata59e3caba5e76
diff --git a/config/runtime.exs b/config/runtime.exs
index 37baec0937..019a2e5229 100644
--- a/config/runtime.exs
+++ b/config/runtime.exs
@@ -624,12 +624,19 @@ ch_transport_opts =
config :plausible, Plausible.ClickhouseRepo,
queue_target: 500,
queue_interval: 2000,
+ timeout: 15_000,
url: ch_db_url,
transport_opts: ch_transport_opts,
settings: [
readonly: 1,
join_algorithm: "direct,parallel_hash,hash",
- cancel_http_readonly_queries_on_client_close: 1
+ # stops queries when :timeout ClickhouseRepo connection :timeout value reached
+ cancel_http_readonly_queries_on_client_close: 1,
+ # stops queries when they will likely take over 20s
+ # NB! when :timeout is overridden to be over 20s,
+ # for it to have meaningful effect,
+ # this must be overridden as well
+ max_execution_time: 20
]
config :plausible, Plausible.IngestRepo,
diff --git a/lib/plausible/clickhouse_repo.ex b/lib/plausible/clickhouse_repo.ex
index 56cb0f299e..cf5c0dedf7 100644
--- a/lib/plausible/clickhouse_repo.ex
+++ b/lib/plausible/clickhouse_repo.ex
@@ -65,4 +65,15 @@ defmodule Plausible.ClickhouseRepo do
{query, opts}
end
+
+ def get_config_without_ch_query_execution_timeout() do
+ {settings, config} = Plausible.ClickhouseRepo.config() |> Keyword.pop!(:settings)
+
+ config
+ |> Keyword.replace!(:pool_size, 1)
+ |> Keyword.put(
+ :settings,
+ settings |> Keyword.put(:max_execution_time, 0)
+ )
+ end
end
diff --git a/lib/workers/clickhouse_clean_sites.ex b/lib/workers/clickhouse_clean_sites.ex
index 43279dbce7..3d14ec0c7a 100644
--- a/lib/workers/clickhouse_clean_sites.ex
+++ b/lib/workers/clickhouse_clean_sites.ex
@@ -59,10 +59,22 @@ defmodule Plausible.Workers.ClickhouseCleanSites do
|> Plausible.Repo.all()
|> MapSet.new()
- ch_sites =
- from(e in "events_v2", group_by: e.site_id, select: e.site_id)
- |> Plausible.ClickhouseRepo.all(timeout: :infinity)
- |> MapSet.new()
+ {:ok, ch} =
+ Plausible.ClickhouseRepo.get_config_without_ch_query_execution_timeout()
+ |> Ch.start_link()
+
+ %Ch.Result{columns: ["site_id"], rows: rows} =
+ DBConnection.run(
+ ch,
+ fn conn ->
+ Ch.query!(conn, "FROM events_v2 SELECT site_id GROUP BY site_id", [],
+ timeout: :infinity
+ )
+ end,
+ timeout: :infinity
+ )
+
+ ch_sites = rows |> MapSet.new(fn [site_id] -> site_id end)
MapSet.difference(ch_sites, pg_sites) |> MapSet.to_list()
end
diff --git a/lib/workers/export_analytics.ex b/lib/workers/export_analytics.ex
index 755aea162b..bccc9f3a8d 100644
--- a/lib/workers/export_analytics.ex
+++ b/lib/workers/export_analytics.ex
@@ -44,8 +44,7 @@ defmodule Plausible.Workers.ExportAnalytics do
# it's ok to use start_link to keep connection lifecycle
# bound to that of the worker
{:ok, ch} =
- Plausible.ClickhouseRepo.config()
- |> Keyword.replace!(:pool_size, 1)
+ Plausible.ClickhouseRepo.get_config_without_ch_query_execution_timeout()
|> Ch.start_link()
try do
diff --git a/test/plausible/exports_test.exs b/test/plausible/exports_test.exs
index 896d09cfd3..9925eb1917 100644
--- a/test/plausible/exports_test.exs
+++ b/test/plausible/exports_test.exs
@@ -71,7 +71,7 @@ defmodule Plausible.ExportsTest do
@describetag :tmp_dir
setup do
- config = Keyword.replace!(Plausible.ClickhouseRepo.config(), :pool_size, 1)
+ config = Plausible.ClickhouseRepo.get_config_without_ch_query_execution_timeout()
{:ok, ch: start_supervised!({Ch, config})}
end