Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.
TypeScript analyticsanalytics-dashboardclickhouseelixirgoogle-analyticsmarketingopen-source-analyticsphoenixplausible-analyticspostgresqlprivacyprivacy-friendlyself-hostedsimple-analyticsstatisticstailwindcssweb-analyticswebsite-analyticswebsite-statswebsite-tracking
Refine parallel application-side joins (#2564)
Uku Taht51fc9c0945b2
diff --git a/lib/plausible/clickhouse_repo.ex b/lib/plausible/clickhouse_repo.exindex 6acb6136d7..67cfa61bf2 100644--- a/lib/plausible/clickhouse_repo.ex+++ b/lib/plausible/clickhouse_repo.ex@@ -10,4 +10,18 @@ defmodule Plausible.ClickhouseRepo doimport Ecto.Query, only: [from: 1, from: 2]endend++ @task_timeout 60_000+ def parallel_tasks(queries) do+ ctx = OpenTelemetry.Ctx.get_current()++ execute_with_tracing = fn fun ->+ OpenTelemetry.Ctx.attach(ctx)+ fun.()+ end++ Task.async_stream(queries, execute_with_tracing, max_concurrency: 3, timeout: @task_timeout)+ |> Enum.to_list()+ |> Keyword.values()+ endenddiff --git a/lib/plausible/stats/aggregate.ex b/lib/plausible/stats/aggregate.exindex dba5023d0c..d0c3299df5 100644--- a/lib/plausible/stats/aggregate.ex+++ b/lib/plausible/stats/aggregate.ex@@ -8,20 +8,19 @@ defmodule Plausible.Stats.Aggregate dodef aggregate(site, query, metrics) doevent_metrics = Enum.filter(metrics, &(&1 in @event_metrics))- event_task = Task.async(fn -> aggregate_events(site, query, event_metrics) end)+ event_task = fn -> aggregate_events(site, query, event_metrics) endsession_metrics = Enum.filter(metrics, &(&1 in @session_metrics))- session_task = Task.async(fn -> aggregate_sessions(site, query, session_metrics) end)+ session_task = fn -> aggregate_sessions(site, query, session_metrics) endtime_on_page_task =if :time_on_page in metrics do- Task.async(fn -> aggregate_time_on_page(site, query) end)+ fn -> aggregate_time_on_page(site, query) endelse- Task.async(fn -> %{} end)+ fn -> %{} endend- Task.await(session_task, 10_000)- |> Map.merge(Task.await(event_task, 10_000))- |> Map.merge(Task.await(time_on_page_task, 10_000))+ Plausible.ClickhouseRepo.parallel_tasks([session_task, event_task, time_on_page_task])+ |> Enum.reduce(%{}, fn aggregate, task_result -> Map.merge(aggregate, task_result) end)|> Enum.map(fn {metric, value} ->{metric, %{value: round(value || 0)}}end)diff --git a/lib/plausible/stats/timeseries.ex b/lib/plausible/stats/timeseries.exindex dfd94d7058..37ca49aa59 100644--- a/lib/plausible/stats/timeseries.ex+++ b/lib/plausible/stats/timeseries.ex@@ -13,13 +13,10 @@ defmodule Plausible.Stats.Timeseries dosession_metrics = Enum.filter(metrics, &(&1 in @session_metrics))[event_result, session_result] =- Task.await_many(- [- Task.async(fn -> events_timeseries(site, query, event_metrics) end),- Task.async(fn -> sessions_timeseries(site, query, session_metrics) end)- ],- 10_000- )+ Plausible.ClickhouseRepo.parallel_tasks([+ fn -> events_timeseries(site, query, event_metrics) end,+ fn -> sessions_timeseries(site, query, session_metrics) end+ ])Enum.map(steps, fn step ->empty_row(step, metrics)diff --git a/lib/plausible_web/controllers/api/external_stats_controller.ex b/lib/plausible_web/controllers/api/external_stats_controller.exindex cf6dca78c6..0b1ffc83a9 100644--- a/lib/plausible_web/controllers/api/external_stats_controller.ex+++ b/lib/plausible_web/controllers/api/external_stats_controller.ex@@ -23,13 +23,10 @@ defmodule PlausibleWeb.Api.ExternalStatsController doprev_query = Query.shift_back(query, site)[prev_result, curr_result] =- Task.await_many(- [- Task.async(fn -> Plausible.Stats.aggregate(site, prev_query, metrics) end),- Task.async(fn -> Plausible.Stats.aggregate(site, query, metrics) end)- ],- 10_000- )+ Plausible.ClickhouseRepo.parallel_tasks([+ fn -> Plausible.Stats.aggregate(site, prev_query, metrics) end,+ fn -> Plausible.Stats.aggregate(site, query, metrics) end+ ])Enum.map(curr_result, fn {metric, %{value: current_val}} ->%{value: prev_val} = prev_result[metric]diff --git a/lib/plausible_web/controllers/stats_controller.ex b/lib/plausible_web/controllers/stats_controller.exindex fc99b47729..2e719c72a3 100644--- a/lib/plausible_web/controllers/stats_controller.ex+++ b/lib/plausible_web/controllers/stats_controller.ex@@ -111,30 +111,33 @@ defmodule PlausibleWeb.StatsController doparams = Map.merge(params, %{"limit" => "300", "csv" => "True", "detailed" => "True"})limited_params = Map.merge(params, %{"limit" => "100"})- csvs = [- {'sources.csv', fn -> Api.StatsController.sources(conn, params) end},- {'utm_mediums.csv', fn -> Api.StatsController.utm_mediums(conn, params) end},- {'utm_sources.csv', fn -> Api.StatsController.utm_sources(conn, params) end},- {'utm_campaigns.csv', fn -> Api.StatsController.utm_campaigns(conn, params) end},- {'utm_contents.csv', fn -> Api.StatsController.utm_contents(conn, params) end},- {'utm_terms.csv', fn -> Api.StatsController.utm_terms(conn, params) end},- {'pages.csv', fn -> Api.StatsController.pages(conn, limited_params) end},- {'entry_pages.csv', fn -> Api.StatsController.entry_pages(conn, params) end},- {'exit_pages.csv', fn -> Api.StatsController.exit_pages(conn, limited_params) end},- {'countries.csv', fn -> Api.StatsController.countries(conn, params) end},- {'regions.csv', fn -> Api.StatsController.regions(conn, params) end},- {'cities.csv', fn -> Api.StatsController.cities(conn, params) end},- {'browsers.csv', fn -> Api.StatsController.browsers(conn, params) end},- {'operating_systems.csv', fn -> Api.StatsController.operating_systems(conn, params) end},- {'devices.csv', fn -> Api.StatsController.screen_sizes(conn, params) end},- {'conversions.csv', fn -> Api.StatsController.conversions(conn, params) end},- {'prop_breakdown.csv', fn -> Api.StatsController.all_props_breakdown(conn, params) end}- ]+ csvs = %{+ 'sources.csv' => fn -> Api.StatsController.sources(conn, params) end,+ 'utm_mediums.csv' => fn -> Api.StatsController.utm_mediums(conn, params) end,+ 'utm_sources.csv' => fn -> Api.StatsController.utm_sources(conn, params) end,+ 'utm_campaigns.csv' => fn -> Api.StatsController.utm_campaigns(conn, params) end,+ 'utm_contents.csv' => fn -> Api.StatsController.utm_contents(conn, params) end,+ 'utm_terms.csv' => fn -> Api.StatsController.utm_terms(conn, params) end,+ 'pages.csv' => fn -> Api.StatsController.pages(conn, limited_params) end,+ 'entry_pages.csv' => fn -> Api.StatsController.entry_pages(conn, params) end,+ 'exit_pages.csv' => fn -> Api.StatsController.exit_pages(conn, limited_params) end,+ 'countries.csv' => fn -> Api.StatsController.countries(conn, params) end,+ 'regions.csv' => fn -> Api.StatsController.regions(conn, params) end,+ 'cities.csv' => fn -> Api.StatsController.cities(conn, params) end,+ 'browsers.csv' => fn -> Api.StatsController.browsers(conn, params) end,+ 'operating_systems.csv' => fn -> Api.StatsController.operating_systems(conn, params) end,+ 'devices.csv' => fn -> Api.StatsController.screen_sizes(conn, params) end,+ 'conversions.csv' => fn -> Api.StatsController.conversions(conn, params) end,+ 'prop_breakdown.csv' => fn -> Api.StatsController.all_props_breakdown(conn, params) end+ }++ csv_values =+ Map.values(csvs)+ |> Plausible.ClickhouseRepo.parallel_tasks()csvs =- csvs- |> Enum.map(fn {file, task} -> {file, Task.async(task)} end)- |> Enum.map(fn {file, task} -> {file, Task.await(task)} end)+ Map.keys(csvs)+ |> Enum.zip(csv_values)csvs = [{'visitors.csv', visitors} | csvs]