Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.
TypeScript analyticsanalytics-dashboardclickhouseelixirgoogle-analyticsmarketingopen-source-analyticsphoenixplausible-analyticspostgresqlprivacyprivacy-friendlyself-hostedsimple-analyticsstatisticstailwindcssweb-analyticswebsite-analyticswebsite-statswebsite-tracking
Revert "Move conversion_rate logic from elixir to clickhouse (#3887)" (#3923)
Karl-Aksel Puulmann253fb5d67d30
diff --git a/extra/lib/plausible/stats/goal/revenue.ex b/extra/lib/plausible/stats/goal/revenue.exindex 7b6e3368f5..46c4049b9f 100644--- a/extra/lib/plausible/stats/goal/revenue.ex+++ b/extra/lib/plausible/stats/goal/revenue.ex@@ -10,17 +10,21 @@ defmodule Plausible.Stats.Goal.Revenue do@revenue_metricsend- def total_revenue_query() do- dynamic(- [e],- fragment("toDecimal64(sum(?) * any(_sample_factor), 3)", e.revenue_reporting_amount)+ def total_revenue_query(query) do+ from(e in query,+ select_merge: %{+ total_revenue:+ fragment("toDecimal64(sum(?) * any(_sample_factor), 3)", e.revenue_reporting_amount)+ })end- def average_revenue_query() do- dynamic(- [e],- fragment("toDecimal64(avg(?) * any(_sample_factor), 3)", e.revenue_reporting_amount)+ def average_revenue_query(query) do+ from(e in query,+ select_merge: %{+ average_revenue:+ fragment("toDecimal64(avg(?) * any(_sample_factor), 3)", e.revenue_reporting_amount)+ })enddiff --git a/lib/plausible/stats/aggregate.ex b/lib/plausible/stats/aggregate.exindex 68b96183a2..56ded33f86 100644--- a/lib/plausible/stats/aggregate.ex+++ b/lib/plausible/stats/aggregate.ex@@ -11,9 +11,7 @@ defmodule Plausible.Stats.Aggregate do:visitors,:pageviews,:events,- :sample_percent,- :conversion_rate,- :total_visitors+ :sample_percent] ++ @revenue_metrics@session_metrics [:visits, :bounce_rate, :visit_duration, :views_per_visit, :sample_percent]@@ -47,6 +45,7 @@ defmodule Plausible.Stats.Aggregate doPlausible.ClickhouseRepo.parallel_tasks([session_task, event_task, time_on_page_task])|> Enum.reduce(%{}, fn aggregate, task_result -> Map.merge(aggregate, task_result) end)+ |> maybe_put_cr(site, query, metrics)|> Util.keep_requested_metrics(metrics)|> cast_revenue_metrics_to_money(currency)|> Enum.map(&maybe_round_value/1)@@ -54,20 +53,45 @@ defmodule Plausible.Stats.Aggregate do|> Enum.into(%{})end+ defp maybe_put_cr(aggregate_result, site, query, metrics) do+ if :conversion_rate in metrics do+ all =+ query+ |> Query.remove_event_filters([:goal, :props])+ |> then(fn query -> aggregate_events(site, query, [:visitors]) end)+ |> Map.fetch!(:visitors)++ converted = aggregate_result.visitors++ cr = Util.calculate_cr(all, converted)++ aggregate_result = Map.put(aggregate_result, :conversion_rate, cr)++ if :total_visitors in metrics do+ Map.put(aggregate_result, :total_visitors, all)+ else+ aggregate_result+ end+ else+ aggregate_result+ end+ end+defp aggregate_events(_, _, []), do: %{}defp aggregate_events(site, query, metrics) do- from(e in base_event_query(site, query), select: ^select_event_metrics(metrics))+ from(e in base_event_query(site, query), select: %{})+ |> select_event_metrics(metrics)|> merge_imported(site, query, :aggregate, metrics)- |> maybe_add_conversion_rate(site, query, metrics, include_imported: query.include_imported)|> ClickhouseRepo.one()enddefp aggregate_sessions(_, _, []), do: %{}defp aggregate_sessions(site, query, metrics) do- from(e in query_sessions(site, query), select: ^select_session_metrics(metrics, query))+ from(e in query_sessions(site, query), select: %{})|> filter_converted_sessions(site, query)+ |> select_session_metrics(metrics, query)|> merge_imported(site, query, :aggregate, metrics)|> ClickhouseRepo.one()|> Util.keep_requested_metrics(metrics)diff --git a/lib/plausible/stats/base.ex b/lib/plausible/stats/base.exindex 463461bf8b..83b4ad6d28 100644--- a/lib/plausible/stats/base.ex+++ b/lib/plausible/stats/base.ex@@ -209,152 +209,163 @@ defmodule Plausible.Stats.Base dodef apply_entry_prop_filter(sessions_q, _, _), do: sessions_q- def select_event_metrics(metrics) do- metrics- |> Enum.map(&select_event_metric/1)- |> Enum.reduce(%{}, &Map.merge/2)- end+ def select_event_metrics(q, []), do: q- defp select_event_metric(:pageviews) do- %{- pageviews:- dynamic(- [e],+ def select_event_metrics(q, [:pageviews | rest]) do+ from(e in q,+ select_merge: %{+ pageviews:fragment("toUInt64(round(countIf(? = 'pageview') * any(_sample_factor)))", e.name)- )- }+ }+ )+ |> select_event_metrics(rest)end- defp select_event_metric(:events) do- %{- events: dynamic([], fragment("toUInt64(round(count(*) * any(_sample_factor)))"))- }+ def select_event_metrics(q, [:events | rest]) do+ from(e in q,+ select_merge: %{events: fragment("toUInt64(round(count(*) * any(_sample_factor)))")}+ )+ |> select_event_metrics(rest)end- defp select_event_metric(:visitors) do- %{- visitors: dynamic([e], selected_as(fragment(@uniq_users_expression, e.user_id), :visitors))- }+ def select_event_metrics(q, [:visitors | rest]) do+ from(e in q,+ select_merge: %{+ visitors: selected_as(fragment(@uniq_users_expression, e.user_id), :visitors)+ }+ )+ |> select_event_metrics(rest)endon_full_build do- defp select_event_metric(:total_revenue) do- %{total_revenue: Plausible.Stats.Goal.Revenue.total_revenue_query()}+ def select_event_metrics(q, [:total_revenue | rest]) do+ q+ |> Plausible.Stats.Goal.Revenue.total_revenue_query()+ |> select_event_metrics(rest)end- defp select_event_metric(:average_revenue) do- %{average_revenue: Plausible.Stats.Goal.Revenue.average_revenue_query()}+ def select_event_metrics(q, [:average_revenue | rest]) do+ q+ |> Plausible.Stats.Goal.Revenue.average_revenue_query()+ |> select_event_metrics(rest)endend- defp select_event_metric(:sample_percent) do- %{- sample_percent:- dynamic(- [],+ def select_event_metrics(q, [:sample_percent | rest]) do+ from(e in q,+ select_merge: %{+ sample_percent:fragment("if(any(_sample_factor) > 1, round(100 / any(_sample_factor)), 100)")- )- }+ }+ )+ |> select_event_metrics(rest)end- defp select_event_metric(:percentage), do: %{}- defp select_event_metric(:conversion_rate), do: %{}- defp select_event_metric(:total_visitors), do: %{}+ def select_event_metrics(q, [:percentage | rest]) do+ q |> select_event_metrics(rest)+ end- defp select_event_metric(unknown), do: raise("Unknown metric: #{unknown}")+ def select_event_metrics(_, [unknown | _]), do: raise("Unknown metric: #{unknown}")- def select_session_metrics(metrics, query) do- metrics- |> Enum.map(&select_session_metric(&1, query))- |> Enum.reduce(%{}, &Map.merge/2)- end+ def select_session_metrics(q, [], _query), do: q- defp select_session_metric(:bounce_rate, query) do+ def select_session_metrics(q, [:bounce_rate | rest], query) docondition = dynamic_filter_condition(query, "event:page", :entry_page)- %{- bounce_rate:- dynamic(- [],- fragment(- "toUInt32(ifNotFinite(round(sumIf(is_bounce * sign, ?) / sumIf(sign, ?) * 100), 0))",- ^condition,- ^condition- )- ),- __internal_visits: dynamic([], fragment("toUInt32(sum(sign))"))- }+ from(s in q,+ select_merge:+ ^%{+ bounce_rate:+ dynamic(+ [],+ fragment(+ "toUInt32(ifNotFinite(round(sumIf(is_bounce * sign, ?) / sumIf(sign, ?) * 100), 0))",+ ^condition,+ ^condition+ )+ ),+ __internal_visits: dynamic([], fragment("toUInt32(sum(sign))"))+ }+ )+ |> select_session_metrics(rest, query)end- defp select_session_metric(:visits, _query) do- %{- visits: dynamic([s], fragment("toUInt64(round(sum(?) * any(_sample_factor)))", s.sign))- }+ def select_session_metrics(q, [:visits | rest], query) do+ from(s in q,+ select_merge: %{+ visits: fragment("toUInt64(round(sum(?) * any(_sample_factor)))", s.sign)+ }+ )+ |> select_session_metrics(rest, query)end- defp select_session_metric(:pageviews, _query) do- %{- pageviews:- dynamic(- [s],+ def select_session_metrics(q, [:pageviews | rest], query) do+ from(s in q,+ select_merge: %{+ pageviews:fragment("toUInt64(round(sum(? * ?) * any(_sample_factor)))", s.sign, s.pageviews)- )- }+ }+ )+ |> select_session_metrics(rest, query)end- defp select_session_metric(:events, _query) do- %{- events:- dynamic(- [s],- fragment("toUInt64(round(sum(? * ?) * any(_sample_factor)))", s.sign, s.events)- )- }+ def select_session_metrics(q, [:events | rest], query) do+ from(s in q,+ select_merge: %{+ events: fragment("toUInt64(round(sum(? * ?) * any(_sample_factor)))", s.sign, s.events)+ }+ )+ |> select_session_metrics(rest, query)end- defp select_session_metric(:visitors, _query) do- %{- visitors:- dynamic(- [s],+ def select_session_metrics(q, [:visitors | rest], query) do+ from(s in q,+ select_merge: %{+ visitors:selected_as(fragment("toUInt64(round(uniq(?) * any(_sample_factor)))", s.user_id),:visitors)- )- }+ }+ )+ |> select_session_metrics(rest, query)end- defp select_session_metric(:visit_duration, _query) do- %{- visit_duration:- dynamic([], fragment("toUInt32(ifNotFinite(round(sum(duration * sign) / sum(sign)), 0))")),- __internal_visits: dynamic([], fragment("toUInt32(sum(sign))"))- }+ def select_session_metrics(q, [:visit_duration | rest], query) do+ from(s in q,+ select_merge: %{+ :visit_duration =>+ fragment("toUInt32(ifNotFinite(round(sum(duration * sign) / sum(sign)), 0))"),+ __internal_visits: fragment("toUInt32(sum(sign))")+ }+ )+ |> select_session_metrics(rest, query)end- defp select_session_metric(:views_per_visit, _query) do- %{- views_per_visit:- dynamic(- [s],- fragment("ifNotFinite(round(sum(? * ?) / sum(?), 2), 0)", s.sign, s.pageviews, s.sign)- ),- __internal_visits: dynamic([], fragment("toUInt32(sum(sign))"))- }+ def select_session_metrics(q, [:views_per_visit | rest], query) do+ from(s in q,+ select_merge: %{+ views_per_visit:+ fragment("ifNotFinite(round(sum(? * ?) / sum(?), 2), 0)", s.sign, s.pageviews, s.sign),+ __internal_visits: fragment("toUInt32(sum(sign))")+ }+ )+ |> select_session_metrics(rest, query)end- defp select_session_metric(:sample_percent, _query) do- %{- sample_percent:- dynamic(- [],+ def select_session_metrics(q, [:sample_percent | rest], query) do+ from(e in q,+ select_merge: %{+ sample_percent:fragment("if(any(_sample_factor) > 1, round(100 / any(_sample_factor)), 100)")- )- }+ }+ )+ |> select_session_metrics(rest, query)end- defp select_session_metric(:percentage, _query), do: %{}+ def select_session_metrics(q, [:percentage | rest], query) do+ q |> select_session_metrics(rest, query)+ enddef dynamic_filter_condition(query, filter_key, db_field) docase query && query.filters && query.filters[filter_key] do@@ -575,7 +586,7 @@ defmodule Plausible.Stats.Base do|> select([e], total_visitors: fragment(@uniq_users_expression, e.user_id))end- defp total_visitors_subquery(site, query, true) do+ defp total_visitors_subquery(site, %Query{include_imported: true} = query) dodynamic([e],selected_as(@@ -586,16 +597,14 @@ defmodule Plausible.Stats.Base do)end- defp total_visitors_subquery(site, query, false) do+ defp total_visitors_subquery(site, query) dodynamic([e], selected_as(subquery(total_visitors(site, query)), :__total_visitors))enddef add_percentage_metric(q, site, query, metrics) doif :percentage in metrics doq- |> select_merge(- ^%{__total_visitors: total_visitors_subquery(site, query, query.include_imported)}- )+ |> select_merge(^%{__total_visitors: total_visitors_subquery(site, query)})|> select_merge(%{percentage:fragment(@@ -609,32 +618,4 @@ defmodule Plausible.Stats.Base doqendend-- # Adds conversion_rate metric to query, calculated as- # X / Y where Y is the same breakdown value without goal or props- # filters.- def maybe_add_conversion_rate(q, site, query, metrics, opts) do- if :conversion_rate in metrics do- include_imported = Keyword.fetch!(opts, :include_imported)-- total_query = query |> Query.remove_event_filters([:goal, :props])-- # :TRICKY: Subquery is used due to event:goal breakdown above doing an UNION ALL- subquery(q)- |> select_merge(- ^%{total_visitors: total_visitors_subquery(site, total_query, include_imported)}- )- |> select_merge([e], %{- conversion_rate:- fragment(- "if(? > 0, round(? / ? * 100, 1), 0)",- selected_as(:__total_visitors),- e.visitors,- selected_as(:__total_visitors)- )- })- else- q- end- endenddiff --git a/lib/plausible/stats/breakdown.ex b/lib/plausible/stats/breakdown.exindex 822c00df1d..ae33e4e737 100644--- a/lib/plausible/stats/breakdown.ex+++ b/lib/plausible/stats/breakdown.ex@@ -4,7 +4,6 @@ defmodule Plausible.Stats.Breakdown douse Plausible.Stats.Fragmentsimport Plausible.Stats.{Base, Imported}- import Ecto.Queryrequire OpenTelemetry.Tracer, as: Traceralias Plausible.Stats.{Query, Util}@@ -53,74 +52,50 @@ defmodule Plausible.Stats.Breakdown dometrics_to_select = Util.maybe_add_visitors_metric(metrics) -- @computed_metrics- event_q =+ event_results =if Enum.any?(event_goals) dosite- |> breakdown_events(event_query, "event:name", metrics_to_select)- |> apply_pagination(pagination)+ |> breakdown(event_query, "event:name", metrics_to_select, pagination, skip_tracing: true)+ |> transform_keys(%{name: :goal})+ |> cast_revenue_metrics_to_money(revenue_goals)else- nil+ []end- page_q =+ {limit, page} = pagination+ offset = (page - 1) * limit++ page_results =if Enum.any?(pageview_goals) dopage_exprs = Enum.map(pageview_goals, & &1.page_path)page_regexes = Enum.map(page_exprs, &page_regex/1)- select_columns = metrics_to_select |> select_event_metrics |> mark_revenue_as_nil-from(e in base_event_query(site, query),order_by: [desc: fragment("uniq(?)", e.user_id)],+ limit: ^limit,+ offset: ^offset,where:fragment("notEmpty(multiMatchAllIndices(?, ?) as indices)",e.pathname,^page_regexes) and e.name == "pageview",- array_join: index in fragment("indices"),- group_by: index,- select: %{}+ group_by: fragment("index"),+ select: %{+ index: fragment("arrayJoin(indices) as index"),+ goal: fragment("concat('Visit ', ?[index])", ^page_exprs)+ })- |> select_merge(^select_columns)- # :TRICKY: name is added last to make sure both queries add columns in the same order- |> select_merge([_, index], %{- name: fragment("concat('Visit ', ?[?])", ^page_exprs, index)- })- |> apply_pagination(pagination)+ |> select_event_metrics(metrics_to_select -- @revenue_metrics)+ |> ClickhouseRepo.all()+ |> Enum.map(fn row -> Map.delete(row, :index) end)else- nil- end-- full_q =- case {event_q, page_q} do- {nil, nil} ->- nil-- {event_q, nil} ->- event_q-- {nil, page_q} ->- page_q-- {event_q, page_q} ->- from(- e in subquery(union_all(event_q, ^page_q)),- # :TODO: Handle other orderings- order_by: [desc: e.visitors]- )- |> apply_pagination(pagination)+ []end- if full_q do- full_q- |> maybe_add_conversion_rate(site, query, metrics, include_imported: false)- |> ClickhouseRepo.all()- |> transform_keys(%{name: :goal})- |> cast_revenue_metrics_to_money(revenue_goals)- |> Util.keep_requested_metrics(metrics)- else- []- end+ zip_results(event_results, page_results, :goal, metrics_to_select)+ |> maybe_add_cr(site, query, nil, metrics)+ |> Util.keep_requested_metrics(metrics)enddef breakdown(site, query, "event:props:" <> custom_prop = property, metrics, pagination, opts) do@@ -135,11 +110,11 @@ defmodule Plausible.Stats.Breakdown doif !Keyword.get(opts, :skip_tracing), do: trace(query, property, metrics)- breakdown_events(site, query, "event:props:" <> custom_prop, metrics_to_select)- |> maybe_add_conversion_rate(site, query, metrics, include_imported: false)- |> paginate_and_execute(metrics, pagination)- |> transform_keys(%{name: custom_prop})+ breakdown_events(site, query, "event:props:" <> custom_prop, metrics_to_select, pagination)|> Enum.map(&cast_revenue_metrics_to_money(&1, currency))+ |> sort_results(metrics_to_select)+ |> maybe_add_cr(site, query, nil, metrics)+ |> Util.keep_requested_metrics(metrics)enddef breakdown(site, query, "event:page" = property, metrics, pagination, opts) do@@ -150,10 +125,10 @@ defmodule Plausible.Stats.Breakdown doevent_result =site- |> breakdown_events(query, property, event_metrics)- |> maybe_add_group_conversion_rate(&breakdown_events/4, site, query, property, metrics)- |> paginate_and_execute(metrics, pagination)+ |> breakdown_events(query, "event:page", event_metrics, pagination)|> maybe_add_time_on_page(site, query, metrics)+ |> maybe_add_cr(site, query, property, metrics, pagination)+ |> Util.keep_requested_metrics(metrics)session_metrics = Enum.filter(metrics, &(&1 in @session_metrics))@@ -174,8 +149,7 @@ defmodule Plausible.Stats.Breakdown do{limit, _page} = paginationsession_result =- breakdown_sessions(site, new_query, "visit:entry_page", session_metrics)- |> paginate_and_execute(session_metrics, {limit, 1})+ breakdown_sessions(site, new_query, "visit:entry_page", session_metrics, {limit, 1})|> transform_keys(%{entry_page: :page})metrics = metrics ++ [:page]@@ -192,9 +166,7 @@ defmodule Plausible.Stats.Breakdown dodef breakdown(site, query, "event:name" = property, metrics, pagination, opts) doif !Keyword.get(opts, :skip_tracing), do: trace(query, property, metrics)-- breakdown_events(site, query, property, metrics)- |> paginate_and_execute(metrics, pagination)+ breakdown_events(site, query, property, metrics, pagination)enddef breakdown(site, query, property, metrics, pagination, opts) do@@ -202,9 +174,9 @@ defmodule Plausible.Stats.Breakdown dometrics_to_select = Util.maybe_add_visitors_metric(metrics) -- @computed_metrics- breakdown_sessions(site, query, property, metrics_to_select)- |> maybe_add_group_conversion_rate(&breakdown_sessions/4, site, query, property, metrics)- |> paginate_and_execute(metrics, pagination)+ breakdown_sessions(site, query, property, metrics_to_select, pagination)+ |> maybe_add_cr(site, query, property, metrics, pagination)+ |> Util.keep_requested_metrics(metrics)enddefp zip_results(event_result, session_result, property, metrics) do@@ -225,31 +197,35 @@ defmodule Plausible.Stats.Breakdown do|> sort_results(metrics)end- defp breakdown_sessions(site, query, property, metrics) do+ defp breakdown_sessions(_, _, _, [], _), do: []++ defp breakdown_sessions(site, query, property, metrics, pagination) dofrom(s in query_sessions(site, query),order_by: [desc: fragment("uniq(?)", s.user_id)],- select: ^select_session_metrics(metrics, query)+ select: %{})|> filter_converted_sessions(site, query)|> do_group_by(property)+ |> select_session_metrics(metrics, query)|> merge_imported(site, query, property, metrics)|> add_percentage_metric(site, query, metrics)+ |> apply_pagination(pagination)+ |> ClickhouseRepo.all()+ |> transform_keys(%{operating_system: :os})+ |> Util.keep_requested_metrics(metrics)end- defp breakdown_events(site, query, property, metrics) do+ defp breakdown_events(_, _, _, [], _), do: []++ defp breakdown_events(site, query, property, metrics, pagination) dofrom(e in base_event_query(site, query),order_by: [desc: fragment("uniq(?)", e.user_id)],- select: ^select_event_metrics(metrics)+ select: %{})|> do_group_by(property)+ |> select_event_metrics(metrics)|> merge_imported(site, query, property, metrics)|> add_percentage_metric(site, query, metrics)- end-- defp paginate_and_execute(_, [], _), do: []-- defp paginate_and_execute(q, metrics, pagination) do- q|> apply_pagination(pagination)|> ClickhouseRepo.all()|> transform_keys(%{operating_system: :os})@@ -447,18 +423,18 @@ defmodule Plausible.Stats.Breakdown dofrom(e in q,select_merge: %{- name:+ ^prop =>selected_as(fragment("if(not empty(?), ?, '(none)')",get_by_key(e, :meta, ^prop),get_by_key(e, :meta, ^prop)),- :name+ :breakdown_prop_value)},- group_by: selected_as(:name),- order_by: {:asc, selected_as(:name)}+ group_by: selected_as(:breakdown_prop_value),+ order_by: {:asc, selected_as(:breakdown_prop_value)})end@@ -676,26 +652,130 @@ defmodule Plausible.Stats.Breakdown do)end- defp group_by_field_names("event:props:" <> _prop), do: [:name]- defp group_by_field_names("visit:os"), do: [:operating_system]- defp group_by_field_names("visit:os_version"), do: [:os, :os_version]- defp group_by_field_names("visit:browser_version"), do: [:browser, :browser_version]+ defp maybe_add_cr(breakdown_results, site, query, property, metrics, pagination \\ nil) do+ cond do+ :conversion_rate not in metrics -> breakdown_results+ Enum.empty?(breakdown_results) -> breakdown_results+ is_nil(property) -> add_absolute_cr(breakdown_results, site, query)+ true -> add_cr(breakdown_results, site, query, property, metrics, pagination)+ end+ end++ # This function injects a conversion_rate metric into every+ # breakdown result map. It is calculated as X / Y, where:+ #+ # * X is the number of conversions for a breakdown+ # result (conversion = number of visitors who+ # completed the filtered goal with the filtered+ # custom properties).+ #+ # * Y is the number of all visitors for this breakdown+ # result without the `event:goal` and `event:props:*`+ # filters.+ defp add_cr(breakdown_results, site, query, property, metrics, pagination) do+ property_atom = Plausible.Stats.Filters.without_prefix(property)++ items =+ Enum.map(breakdown_results, fn item -> Map.fetch!(item, property_atom) end)++ query_without_goal =+ query+ |> Query.put_filter(property, {:member, items})+ |> Query.remove_event_filters([:goal, :props])++ # Here, we're always only interested in the first page of results+ # - the :member filter makes sure that the results always match with+ # the items in the given breakdown_results list+ page = 1++ # For browser/os versions we need to fetch a lot more entries than the+ # pagination limit. This is because many entries can correspond to a+ # single version number and we need to make sure that the results+ # without goal filter will include all those combinations of browsers/os-s+ # and their versions that were present in the `breakdown_results` array.+ {pagination_limit, find_match_fn} =+ case property_atom do+ :browser_version ->+ pagination_limit = min(elem(pagination, 0) * 10, 3_000)++ find_match_fn = fn total, conversion ->+ total[:browser_version] == conversion[:browser_version] &&+ total[:browser] == conversion[:browser]+ end++ {pagination_limit, find_match_fn}++ :os_version ->+ pagination_limit = min(elem(pagination, 0) * 5, 3_000)++ find_match_fn = fn total, conversion ->+ total[:os_version] == conversion[:os_version] &&+ total[:os] == conversion[:os]+ end++ {pagination_limit, find_match_fn}++ _ ->+ {elem(pagination, 0),+ fn total, conversion ->+ total[property_atom] == conversion[property_atom]+ end}+ end++ pagination = {pagination_limit, page}- defp group_by_field_names(property), do: [Plausible.Stats.Filters.without_prefix(property)]+ res_without_goal = breakdown(site, query_without_goal, property, [:visitors], pagination)- defp on_matches_group_by(fields) do- Enum.reduce(fields, nil, &fields_equal/2)- end+ Enum.map(breakdown_results, fn item ->+ without_goal = Enum.find(res_without_goal, &find_match_fn.(&1, item))- defp outer_order_by(fields) do- Enum.map(fields, fn field_name -> {:asc, dynamic([q], field(q, ^field_name))} end)+ {conversion_rate, total_visitors} =+ if without_goal do+ {Util.calculate_cr(without_goal.visitors, item.visitors), without_goal.visitors}+ else+ Sentry.capture_message(+ "Unable to find a conversion_rate divisor from a breakdown response",+ extra: %{+ domain: site.domain,+ query: inspect(query),+ property: property,+ pagination: inspect(pagination),+ item_not_found: inspect(item)+ }+ )++ {"N/A", "N/A"}+ end++ if :total_visitors in metrics do+ item+ |> Map.put(:conversion_rate, conversion_rate)+ |> Map.put(:total_visitors, total_visitors)+ else+ Map.put(item, :conversion_rate, conversion_rate)+ end+ end)end- defp fields_equal(field_name, nil),- do: dynamic([a, b], field(a, ^field_name) == field(b, ^field_name))+ # Similar to `add_cr/5`, injects a conversion_rate metric into+ # every breakdown result. However, a single divisor is used in+ # the CR calculation across all breakdown results. That is the+ # number of visitors without `event:goal` and `event:props:*`+ # filters.+ #+ # This is useful when we're only interested in the conversions+ # themselves - not how well a certain property such as browser+ # or page converted.+ defp add_absolute_cr(breakdown_results, site, query) do+ total_q = Query.remove_event_filters(query, [:goal, :props])++ %{visitors: %{value: total_visitors}} = Plausible.Stats.aggregate(site, total_q, [:visitors])- defp fields_equal(field_name, condition),- do: dynamic([a, b], field(a, ^field_name) == field(b, ^field_name) and ^condition)+ breakdown_results+ |> Enum.map(fn goal ->+ Map.put(goal, :conversion_rate, Util.calculate_cr(total_visitors, goal[:visitors]))+ end)+ enddefp sort_results(results, metrics) doEnum.sort_by(@@ -710,54 +790,6 @@ defmodule Plausible.Stats.Breakdown do)end- # This function injects a conversion_rate metric into- # a breakdown query. It is calculated as X / Y, where:- #- # * X is the number of conversions for a breakdown- # result (conversion = number of visitors who- # completed the filtered goal with the filtered- # custom properties).- #- # * Y is the number of all visitors for this breakdown- # result without the `event:goal` and `event:props:*`- # filters.- defp maybe_add_group_conversion_rate(q, breakdown_fn, site, query, property, metrics) do- if :conversion_rate in metrics do- breakdown_total_visitors_query = query |> Query.remove_event_filters([:goal, :props])-- breakdown_total_visitors_q =- breakdown_fn.(site, breakdown_total_visitors_query, property, [:visitors])-- from(e in subquery(q),- left_join: c in subquery(breakdown_total_visitors_q),- on: ^on_matches_group_by(group_by_field_names(property)),- select_merge: %{- total_visitors: c.visitors,- conversion_rate:- fragment(- "if(? > 0, round(? / ? * 100, 1), 0)",- c.visitors,- e.visitors,- c.visitors- )- },- order_by: [desc: e.visitors],- order_by: ^outer_order_by(group_by_field_names(property))- )- else- q- end- end-- # When querying custom event goals and pageviewgoals together, UNION ALL is used- # so the same fields must be present on both sides of the union. This change to the- # query will ensure that we don't unnecessarily read revenue column for pageview goals- defp mark_revenue_as_nil(select_columns) do- select_columns- |> Map.replace(:total_revenue, nil)- |> Map.replace(:average_revenue, nil)- end-defp sorting_key(metrics) doif Enum.member?(metrics, :visitors), do: :visitors, else: List.first(metrics)end@@ -775,8 +807,8 @@ defmodule Plausible.Stats.Breakdown dooffset = (page - 1) * limitq- |> limit(^limit)- |> offset(^offset)+ |> Ecto.Query.limit(^limit)+ |> Ecto.Query.offset(^offset)enddefp trace(query, property, metrics) dodiff --git a/lib/plausible/stats/timeseries.ex b/lib/plausible/stats/timeseries.exindex 053625a90a..4c31aabd03 100644--- a/lib/plausible/stats/timeseries.ex+++ b/lib/plausible/stats/timeseries.ex@@ -53,8 +53,9 @@ defmodule Plausible.Stats.Timeseries dodefp events_timeseries(_, _, []), do: []defp events_timeseries(site, query, metrics) do- from(e in base_event_query(site, query), select: ^select_event_metrics(metrics))+ from(e in base_event_query(site, query), select: %{})|> select_bucket(site, query)+ |> select_event_metrics(metrics)|> Plausible.Stats.Imported.merge_imported_timeseries(site, query, metrics)|> ClickhouseRepo.all()end@@ -62,9 +63,10 @@ defmodule Plausible.Stats.Timeseries dodefp sessions_timeseries(_, _, []), do: []defp sessions_timeseries(site, query, metrics) do- from(e in query_sessions(site, query), select: ^select_session_metrics(metrics, query))+ from(e in query_sessions(site, query), select: %{})|> filter_converted_sessions(site, query)|> select_bucket(site, query)+ |> select_session_metrics(metrics, query)|> Plausible.Stats.Imported.merge_imported_timeseries(site, query, metrics)|> ClickhouseRepo.all()|> Util.keep_requested_metrics(metrics)diff --git a/lib/plausible/stats/util.ex b/lib/plausible/stats/util.exindex 85c36888f8..10078ff6a3 100644--- a/lib/plausible/stats/util.ex+++ b/lib/plausible/stats/util.ex@@ -7,8 +7,7 @@ defmodule Plausible.Stats.Util do:__internal_visits,:visitors,:__total_visitors,- :__breakdown_value,- :total_visitors+ :__breakdown_value]@doc """@@ -49,4 +48,12 @@ defmodule Plausible.Stats.Util dometricsendend++ def calculate_cr(nil, _converted_visitors), do: nil++ def calculate_cr(unique_visitors, converted_visitors) do+ if unique_visitors > 0,+ do: Float.round(converted_visitors / unique_visitors * 100, 1),+ else: 0.0+ endenddiff --git a/test/plausible_web/controllers/api/external_stats_controller/aggregate_test.exs b/test/plausible_web/controllers/api/external_stats_controller/aggregate_test.exsindex 6f0e35092a..325987defc 100644--- a/test/plausible_web/controllers/api/external_stats_controller/aggregate_test.exs+++ b/test/plausible_web/controllers/api/external_stats_controller/aggregate_test.exs@@ -1569,21 +1569,5 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest do} =json_response(conn, 200)["results"]end-- test "conversion_rate for the filtered goal is 0 when no stats exist", %{- conn: conn,- site: site- } do- insert(:goal, %{site: site, event_name: "Signup"})-- conn =- get(conn, "/api/v1/stats/aggregate", %{- "site_id" => site.domain,- "metrics" => "conversion_rate",- "filters" => "event:goal==Signup"- })-- assert json_response(conn, 200)["results"] == %{"conversion_rate" => %{"value" => 0}}- endendenddiff --git a/test/plausible_web/controllers/api/stats_controller/conversions_test.exs b/test/plausible_web/controllers/api/stats_controller/conversions_test.exsindex 9dd09736ea..a454f59f69 100644--- a/test/plausible_web/controllers/api/stats_controller/conversions_test.exs+++ b/test/plausible_web/controllers/api/stats_controller/conversions_test.exs@@ -580,42 +580,6 @@ defmodule PlausibleWeb.Api.StatsController.ConversionsTest do}]end-- test "conversion_rate for goals should not be calculated with imported data", %{- conn: conn,- site: site- } do- site =- site- |> Plausible.Site.start_import(~D[2005-01-01], Timex.today(), "Google Analytics", "ok")- |> Plausible.Repo.update!()-- populate_stats(site, [- build(:pageview, pathname: "/"),- build(:pageview, pathname: "/another"),- build(:pageview, pathname: "/blog/post-1"),- build(:pageview, pathname: "/blog/post-2"),- build(:imported_pages, page: "/blog/post-1"),- build(:imported_visitors)- ])-- insert(:goal, %{site: site, page_path: "/blog**"})-- conn =- get(- conn,- "/api/stats/#{site.domain}/conversions?period=day"- )-- assert json_response(conn, 200) == [- %{- "name" => "Visit /blog**",- "visitors" => 2,- "events" => 2,- "conversion_rate" => 50- }- ]- endenddescribe "GET /api/stats/:domain/conversions - with goal and prop=(none) filter" do