Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.
TypeScript analyticsanalytics-dashboardclickhouseelixirgoogle-analyticsmarketingopen-source-analyticsphoenixplausible-analyticspostgresqlprivacyprivacy-friendlyself-hostedsimple-analyticsstatisticstailwindcssweb-analyticswebsite-analyticswebsite-statswebsite-tracking
Speed up calculating monthly usage for users with many many sites (#4101)
Karl-Aksel Puulmann32a16a72faeb
diff --git a/lib/plausible/clickhouse_repo.ex b/lib/plausible/clickhouse_repo.exindex f6f8631d97..37d5961cb9 100644--- a/lib/plausible/clickhouse_repo.ex+++ b/lib/plausible/clickhouse_repo.ex@@ -13,7 +13,7 @@ defmodule Plausible.ClickhouseRepo doend@task_timeout 60_000- def parallel_tasks(queries) do+ def parallel_tasks(queries, opts \\ []) doctx = OpenTelemetry.Ctx.get_current()execute_with_tracing = fn fun ->@@ -21,7 +21,12 @@ defmodule Plausible.ClickhouseRepo dofun.()end- Task.async_stream(queries, execute_with_tracing, max_concurrency: 3, timeout: @task_timeout)+ max_concurrency = Keyword.get(opts, :max_concurrency, 3)++ Task.async_stream(queries, execute_with_tracing,+ max_concurrency: max_concurrency,+ timeout: @task_timeout+ )|> Enum.to_list()|> Keyword.values()enddiff --git a/lib/plausible/stats/clickhouse.ex b/lib/plausible/stats/clickhouse.exindex f217d64f36..ed112b9a93 100644--- a/lib/plausible/stats/clickhouse.ex+++ b/lib/plausible/stats/clickhouse.ex@@ -50,30 +50,10 @@ defmodule Plausible.Stats.Clickhouse do|> Map.new()end- def usage_breakdown([d | _] = domains, date_range) when is_binary(d) do- Enum.chunk_every(domains, 300)- |> Enum.reduce({0, 0}, fn domains, {pageviews_total, custom_events_total} ->- {chunk_pageviews, chunk_custom_events} =- ClickhouseRepo.one(- from(e in "events",- where: e.domain in ^domains,- where: fragment("toDate(?)", e.timestamp) >= ^date_range.first,- where: fragment("toDate(?)", e.timestamp) <= ^date_range.last,- select: {- fragment("countIf(? = 'pageview')", e.name),- fragment("countIf(? != 'pageview')", e.name)- }- )- )-- {pageviews_total + chunk_pageviews, custom_events_total + chunk_custom_events}- end)- end-def usage_breakdown([sid | _] = site_ids, date_range) when is_integer(sid) do- Enum.chunk_every(site_ids, 300)- |> Enum.reduce({0, 0}, fn site_ids, {pageviews_total, custom_events_total} ->- {chunk_pageviews, chunk_custom_events} =+ Enum.chunk_every(site_ids, 1000)+ |> Enum.map(fn site_ids ->+ fn ->ClickhouseRepo.one(from(e in "events_v2",where: e.site_id in ^site_ids,@@ -85,8 +65,11 @@ defmodule Plausible.Stats.Clickhouse do}))-- {pageviews_total + chunk_pageviews, custom_events_total + chunk_custom_events}+ end+ end)+ |> ClickhouseRepo.parallel_tasks(max_concurrency: 10)+ |> Enum.reduce(fn {pageviews, custom_events}, {pageviews_total, custom_events_total} ->+ {pageviews_total + pageviews, custom_events_total + custom_events}end)end