Skip to content

Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.

Speed up calculating monthly usage for users with many many sites (#4101)

Karl-Aksel Puulmann32a16a72faeb
diff --git a/lib/plausible/clickhouse_repo.ex b/lib/plausible/clickhouse_repo.ex
index f6f8631d97..37d5961cb9 100644
--- a/lib/plausible/clickhouse_repo.ex
+++ b/lib/plausible/clickhouse_repo.ex
@@ -13,7 +13,7 @@ defmodule Plausible.ClickhouseRepo do
end
@task_timeout 60_000
- def parallel_tasks(queries) do
+ def parallel_tasks(queries, opts \\ []) do
ctx = OpenTelemetry.Ctx.get_current()
execute_with_tracing = fn fun ->
@@ -21,7 +21,12 @@ defmodule Plausible.ClickhouseRepo do
fun.()
end
- Task.async_stream(queries, execute_with_tracing, max_concurrency: 3, timeout: @task_timeout)
+ max_concurrency = Keyword.get(opts, :max_concurrency, 3)
+
+ Task.async_stream(queries, execute_with_tracing,
+ max_concurrency: max_concurrency,
+ timeout: @task_timeout
+ )
|> Enum.to_list()
|> Keyword.values()
end
diff --git a/lib/plausible/stats/clickhouse.ex b/lib/plausible/stats/clickhouse.ex
index f217d64f36..ed112b9a93 100644
--- a/lib/plausible/stats/clickhouse.ex
+++ b/lib/plausible/stats/clickhouse.ex
@@ -50,30 +50,10 @@ defmodule Plausible.Stats.Clickhouse do
|> Map.new()
end
- def usage_breakdown([d | _] = domains, date_range) when is_binary(d) do
- Enum.chunk_every(domains, 300)
- |> Enum.reduce({0, 0}, fn domains, {pageviews_total, custom_events_total} ->
- {chunk_pageviews, chunk_custom_events} =
- ClickhouseRepo.one(
- from(e in "events",
- where: e.domain in ^domains,
- where: fragment("toDate(?)", e.timestamp) >= ^date_range.first,
- where: fragment("toDate(?)", e.timestamp) <= ^date_range.last,
- select: {
- fragment("countIf(? = 'pageview')", e.name),
- fragment("countIf(? != 'pageview')", e.name)
- }
- )
- )
-
- {pageviews_total + chunk_pageviews, custom_events_total + chunk_custom_events}
- end)
- end
-
def usage_breakdown([sid | _] = site_ids, date_range) when is_integer(sid) do
- Enum.chunk_every(site_ids, 300)
- |> Enum.reduce({0, 0}, fn site_ids, {pageviews_total, custom_events_total} ->
- {chunk_pageviews, chunk_custom_events} =
+ Enum.chunk_every(site_ids, 1000)
+ |> Enum.map(fn site_ids ->
+ fn ->
ClickhouseRepo.one(
from(e in "events_v2",
where: e.site_id in ^site_ids,
@@ -85,8 +65,11 @@ defmodule Plausible.Stats.Clickhouse do
}
)
)
-
- {pageviews_total + chunk_pageviews, custom_events_total + chunk_custom_events}
+ end
+ end)
+ |> ClickhouseRepo.parallel_tasks(max_concurrency: 10)
+ |> Enum.reduce(fn {pageviews, custom_events}, {pageviews_total, custom_events_total} ->
+ {pageviews_total + pageviews, custom_events_total + custom_events}
end)
end