Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.
TypeScript analyticsanalytics-dashboardclickhouseelixirgoogle-analyticsmarketingopen-source-analyticsphoenixplausible-analyticspostgresqlprivacyprivacy-friendlyself-hostedsimple-analyticsstatisticstailwindcssweb-analyticswebsite-analyticswebsite-statswebsite-tracking
Limit ClickhouseRepo queries max_execution_time (#6018)
Artur Pata59e3caba5e76
diff --git a/config/runtime.exs b/config/runtime.exsindex 37baec0937..019a2e5229 100644--- a/config/runtime.exs+++ b/config/runtime.exs@@ -624,12 +624,19 @@ ch_transport_opts =config :plausible, Plausible.ClickhouseRepo,queue_target: 500,queue_interval: 2000,+ timeout: 15_000,url: ch_db_url,transport_opts: ch_transport_opts,settings: [readonly: 1,join_algorithm: "direct,parallel_hash,hash",- cancel_http_readonly_queries_on_client_close: 1+ # stops queries when :timeout ClickhouseRepo connection :timeout value reached+ cancel_http_readonly_queries_on_client_close: 1,+ # stops queries when they will likely take over 20s+ # NB! when :timeout is overridden to be over 20s,+ # for it to have meaningful effect,+ # this must be overridden as well+ max_execution_time: 20]config :plausible, Plausible.IngestRepo,diff --git a/lib/plausible/clickhouse_repo.ex b/lib/plausible/clickhouse_repo.exindex 56cb0f299e..cf5c0dedf7 100644--- a/lib/plausible/clickhouse_repo.ex+++ b/lib/plausible/clickhouse_repo.ex@@ -65,4 +65,15 @@ defmodule Plausible.ClickhouseRepo do{query, opts}end++ def get_config_without_ch_query_execution_timeout() do+ {settings, config} = Plausible.ClickhouseRepo.config() |> Keyword.pop!(:settings)++ config+ |> Keyword.replace!(:pool_size, 1)+ |> Keyword.put(+ :settings,+ settings |> Keyword.put(:max_execution_time, 0)+ )+ endenddiff --git a/lib/workers/clickhouse_clean_sites.ex b/lib/workers/clickhouse_clean_sites.exindex 43279dbce7..3d14ec0c7a 100644--- a/lib/workers/clickhouse_clean_sites.ex+++ b/lib/workers/clickhouse_clean_sites.ex@@ -59,10 +59,22 @@ defmodule Plausible.Workers.ClickhouseCleanSites do|> Plausible.Repo.all()|> MapSet.new()- ch_sites =- from(e in "events_v2", group_by: e.site_id, select: e.site_id)- |> Plausible.ClickhouseRepo.all(timeout: :infinity)- |> MapSet.new()+ {:ok, ch} =+ Plausible.ClickhouseRepo.get_config_without_ch_query_execution_timeout()+ |> Ch.start_link()++ %Ch.Result{columns: ["site_id"], rows: rows} =+ DBConnection.run(+ ch,+ fn conn ->+ Ch.query!(conn, "FROM events_v2 SELECT site_id GROUP BY site_id", [],+ timeout: :infinity+ )+ end,+ timeout: :infinity+ )++ ch_sites = rows |> MapSet.new(fn [site_id] -> site_id end)MapSet.difference(ch_sites, pg_sites) |> MapSet.to_list()enddiff --git a/lib/workers/export_analytics.ex b/lib/workers/export_analytics.exindex 755aea162b..bccc9f3a8d 100644--- a/lib/workers/export_analytics.ex+++ b/lib/workers/export_analytics.ex@@ -44,8 +44,7 @@ defmodule Plausible.Workers.ExportAnalytics do# it's ok to use start_link to keep connection lifecycle# bound to that of the worker{:ok, ch} =- Plausible.ClickhouseRepo.config()- |> Keyword.replace!(:pool_size, 1)+ Plausible.ClickhouseRepo.get_config_without_ch_query_execution_timeout()|> Ch.start_link()try dodiff --git a/test/plausible/exports_test.exs b/test/plausible/exports_test.exsindex 896d09cfd3..9925eb1917 100644--- a/test/plausible/exports_test.exs+++ b/test/plausible/exports_test.exs@@ -71,7 +71,7 @@ defmodule Plausible.ExportsTest do@describetag :tmp_dirsetup do- config = Keyword.replace!(Plausible.ClickhouseRepo.config(), :pool_size, 1)+ config = Plausible.ClickhouseRepo.get_config_without_ch_query_execution_timeout(){:ok, ch: start_supervised!({Ch, config})}end