Skip to content

Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.

Refine parallel application-side joins (#2564)

Uku Taht51fc9c0945b2
diff --git a/lib/plausible/clickhouse_repo.ex b/lib/plausible/clickhouse_repo.ex
index 6acb6136d7..67cfa61bf2 100644
--- a/lib/plausible/clickhouse_repo.ex
+++ b/lib/plausible/clickhouse_repo.ex
@@ -10,4 +10,18 @@ defmodule Plausible.ClickhouseRepo do
import Ecto.Query, only: [from: 1, from: 2]
end
end
+
+ @task_timeout 60_000
+ def parallel_tasks(queries) do
+ ctx = OpenTelemetry.Ctx.get_current()
+
+ execute_with_tracing = fn fun ->
+ OpenTelemetry.Ctx.attach(ctx)
+ fun.()
+ end
+
+ Task.async_stream(queries, execute_with_tracing, max_concurrency: 3, timeout: @task_timeout)
+ |> Enum.to_list()
+ |> Keyword.values()
+ end
end
diff --git a/lib/plausible/stats/aggregate.ex b/lib/plausible/stats/aggregate.ex
index dba5023d0c..d0c3299df5 100644
--- a/lib/plausible/stats/aggregate.ex
+++ b/lib/plausible/stats/aggregate.ex
@@ -8,20 +8,19 @@ defmodule Plausible.Stats.Aggregate do
def aggregate(site, query, metrics) do
event_metrics = Enum.filter(metrics, &(&1 in @event_metrics))
- event_task = Task.async(fn -> aggregate_events(site, query, event_metrics) end)
+ event_task = fn -> aggregate_events(site, query, event_metrics) end
session_metrics = Enum.filter(metrics, &(&1 in @session_metrics))
- session_task = Task.async(fn -> aggregate_sessions(site, query, session_metrics) end)
+ session_task = fn -> aggregate_sessions(site, query, session_metrics) end
time_on_page_task =
if :time_on_page in metrics do
- Task.async(fn -> aggregate_time_on_page(site, query) end)
+ fn -> aggregate_time_on_page(site, query) end
else
- Task.async(fn -> %{} end)
+ fn -> %{} end
end
- Task.await(session_task, 10_000)
- |> Map.merge(Task.await(event_task, 10_000))
- |> Map.merge(Task.await(time_on_page_task, 10_000))
+ Plausible.ClickhouseRepo.parallel_tasks([session_task, event_task, time_on_page_task])
+ |> Enum.reduce(%{}, fn aggregate, task_result -> Map.merge(aggregate, task_result) end)
|> Enum.map(fn {metric, value} ->
{metric, %{value: round(value || 0)}}
end)
diff --git a/lib/plausible/stats/timeseries.ex b/lib/plausible/stats/timeseries.ex
index dfd94d7058..37ca49aa59 100644
--- a/lib/plausible/stats/timeseries.ex
+++ b/lib/plausible/stats/timeseries.ex
@@ -13,13 +13,10 @@ defmodule Plausible.Stats.Timeseries do
session_metrics = Enum.filter(metrics, &(&1 in @session_metrics))
[event_result, session_result] =
- Task.await_many(
- [
- Task.async(fn -> events_timeseries(site, query, event_metrics) end),
- Task.async(fn -> sessions_timeseries(site, query, session_metrics) end)
- ],
- 10_000
- )
+ Plausible.ClickhouseRepo.parallel_tasks([
+ fn -> events_timeseries(site, query, event_metrics) end,
+ fn -> sessions_timeseries(site, query, session_metrics) end
+ ])
Enum.map(steps, fn step ->
empty_row(step, metrics)
diff --git a/lib/plausible_web/controllers/api/external_stats_controller.ex b/lib/plausible_web/controllers/api/external_stats_controller.ex
index cf6dca78c6..0b1ffc83a9 100644
--- a/lib/plausible_web/controllers/api/external_stats_controller.ex
+++ b/lib/plausible_web/controllers/api/external_stats_controller.ex
@@ -23,13 +23,10 @@ defmodule PlausibleWeb.Api.ExternalStatsController do
prev_query = Query.shift_back(query, site)
[prev_result, curr_result] =
- Task.await_many(
- [
- Task.async(fn -> Plausible.Stats.aggregate(site, prev_query, metrics) end),
- Task.async(fn -> Plausible.Stats.aggregate(site, query, metrics) end)
- ],
- 10_000
- )
+ Plausible.ClickhouseRepo.parallel_tasks([
+ fn -> Plausible.Stats.aggregate(site, prev_query, metrics) end,
+ fn -> Plausible.Stats.aggregate(site, query, metrics) end
+ ])
Enum.map(curr_result, fn {metric, %{value: current_val}} ->
%{value: prev_val} = prev_result[metric]
diff --git a/lib/plausible_web/controllers/stats_controller.ex b/lib/plausible_web/controllers/stats_controller.ex
index fc99b47729..2e719c72a3 100644
--- a/lib/plausible_web/controllers/stats_controller.ex
+++ b/lib/plausible_web/controllers/stats_controller.ex
@@ -111,30 +111,33 @@ defmodule PlausibleWeb.StatsController do
params = Map.merge(params, %{"limit" => "300", "csv" => "True", "detailed" => "True"})
limited_params = Map.merge(params, %{"limit" => "100"})
- csvs = [
- {'sources.csv', fn -> Api.StatsController.sources(conn, params) end},
- {'utm_mediums.csv', fn -> Api.StatsController.utm_mediums(conn, params) end},
- {'utm_sources.csv', fn -> Api.StatsController.utm_sources(conn, params) end},
- {'utm_campaigns.csv', fn -> Api.StatsController.utm_campaigns(conn, params) end},
- {'utm_contents.csv', fn -> Api.StatsController.utm_contents(conn, params) end},
- {'utm_terms.csv', fn -> Api.StatsController.utm_terms(conn, params) end},
- {'pages.csv', fn -> Api.StatsController.pages(conn, limited_params) end},
- {'entry_pages.csv', fn -> Api.StatsController.entry_pages(conn, params) end},
- {'exit_pages.csv', fn -> Api.StatsController.exit_pages(conn, limited_params) end},
- {'countries.csv', fn -> Api.StatsController.countries(conn, params) end},
- {'regions.csv', fn -> Api.StatsController.regions(conn, params) end},
- {'cities.csv', fn -> Api.StatsController.cities(conn, params) end},
- {'browsers.csv', fn -> Api.StatsController.browsers(conn, params) end},
- {'operating_systems.csv', fn -> Api.StatsController.operating_systems(conn, params) end},
- {'devices.csv', fn -> Api.StatsController.screen_sizes(conn, params) end},
- {'conversions.csv', fn -> Api.StatsController.conversions(conn, params) end},
- {'prop_breakdown.csv', fn -> Api.StatsController.all_props_breakdown(conn, params) end}
- ]
+ csvs = %{
+ 'sources.csv' => fn -> Api.StatsController.sources(conn, params) end,
+ 'utm_mediums.csv' => fn -> Api.StatsController.utm_mediums(conn, params) end,
+ 'utm_sources.csv' => fn -> Api.StatsController.utm_sources(conn, params) end,
+ 'utm_campaigns.csv' => fn -> Api.StatsController.utm_campaigns(conn, params) end,
+ 'utm_contents.csv' => fn -> Api.StatsController.utm_contents(conn, params) end,
+ 'utm_terms.csv' => fn -> Api.StatsController.utm_terms(conn, params) end,
+ 'pages.csv' => fn -> Api.StatsController.pages(conn, limited_params) end,
+ 'entry_pages.csv' => fn -> Api.StatsController.entry_pages(conn, params) end,
+ 'exit_pages.csv' => fn -> Api.StatsController.exit_pages(conn, limited_params) end,
+ 'countries.csv' => fn -> Api.StatsController.countries(conn, params) end,
+ 'regions.csv' => fn -> Api.StatsController.regions(conn, params) end,
+ 'cities.csv' => fn -> Api.StatsController.cities(conn, params) end,
+ 'browsers.csv' => fn -> Api.StatsController.browsers(conn, params) end,
+ 'operating_systems.csv' => fn -> Api.StatsController.operating_systems(conn, params) end,
+ 'devices.csv' => fn -> Api.StatsController.screen_sizes(conn, params) end,
+ 'conversions.csv' => fn -> Api.StatsController.conversions(conn, params) end,
+ 'prop_breakdown.csv' => fn -> Api.StatsController.all_props_breakdown(conn, params) end
+ }
+
+ csv_values =
+ Map.values(csvs)
+ |> Plausible.ClickhouseRepo.parallel_tasks()
csvs =
- csvs
- |> Enum.map(fn {file, task} -> {file, Task.async(task)} end)
- |> Enum.map(fn {file, task} -> {file, Task.await(task)} end)
+ Map.keys(csvs)
+ |> Enum.zip(csv_values)
csvs = [{'visitors.csv', visitors} | csvs]