Skip to content

Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.

Revert "Move conversion_rate logic from elixir to clickhouse (#3887)" (#3923)

Karl-Aksel Puulmann253fb5d67d30
diff --git a/extra/lib/plausible/stats/goal/revenue.ex b/extra/lib/plausible/stats/goal/revenue.ex
index 7b6e3368f5..46c4049b9f 100644
--- a/extra/lib/plausible/stats/goal/revenue.ex
+++ b/extra/lib/plausible/stats/goal/revenue.ex
@@ -10,17 +10,21 @@ defmodule Plausible.Stats.Goal.Revenue do
@revenue_metrics
end
- def total_revenue_query() do
- dynamic(
- [e],
- fragment("toDecimal64(sum(?) * any(_sample_factor), 3)", e.revenue_reporting_amount)
+ def total_revenue_query(query) do
+ from(e in query,
+ select_merge: %{
+ total_revenue:
+ fragment("toDecimal64(sum(?) * any(_sample_factor), 3)", e.revenue_reporting_amount)
+ }
)
end
- def average_revenue_query() do
- dynamic(
- [e],
- fragment("toDecimal64(avg(?) * any(_sample_factor), 3)", e.revenue_reporting_amount)
+ def average_revenue_query(query) do
+ from(e in query,
+ select_merge: %{
+ average_revenue:
+ fragment("toDecimal64(avg(?) * any(_sample_factor), 3)", e.revenue_reporting_amount)
+ }
)
end
diff --git a/lib/plausible/stats/aggregate.ex b/lib/plausible/stats/aggregate.ex
index 68b96183a2..56ded33f86 100644
--- a/lib/plausible/stats/aggregate.ex
+++ b/lib/plausible/stats/aggregate.ex
@@ -11,9 +11,7 @@ defmodule Plausible.Stats.Aggregate do
:visitors,
:pageviews,
:events,
- :sample_percent,
- :conversion_rate,
- :total_visitors
+ :sample_percent
] ++ @revenue_metrics
@session_metrics [:visits, :bounce_rate, :visit_duration, :views_per_visit, :sample_percent]
@@ -47,6 +45,7 @@ defmodule Plausible.Stats.Aggregate do
Plausible.ClickhouseRepo.parallel_tasks([session_task, event_task, time_on_page_task])
|> Enum.reduce(%{}, fn aggregate, task_result -> Map.merge(aggregate, task_result) end)
+ |> maybe_put_cr(site, query, metrics)
|> Util.keep_requested_metrics(metrics)
|> cast_revenue_metrics_to_money(currency)
|> Enum.map(&maybe_round_value/1)
@@ -54,20 +53,45 @@ defmodule Plausible.Stats.Aggregate do
|> Enum.into(%{})
end
+ defp maybe_put_cr(aggregate_result, site, query, metrics) do
+ if :conversion_rate in metrics do
+ all =
+ query
+ |> Query.remove_event_filters([:goal, :props])
+ |> then(fn query -> aggregate_events(site, query, [:visitors]) end)
+ |> Map.fetch!(:visitors)
+
+ converted = aggregate_result.visitors
+
+ cr = Util.calculate_cr(all, converted)
+
+ aggregate_result = Map.put(aggregate_result, :conversion_rate, cr)
+
+ if :total_visitors in metrics do
+ Map.put(aggregate_result, :total_visitors, all)
+ else
+ aggregate_result
+ end
+ else
+ aggregate_result
+ end
+ end
+
defp aggregate_events(_, _, []), do: %{}
defp aggregate_events(site, query, metrics) do
- from(e in base_event_query(site, query), select: ^select_event_metrics(metrics))
+ from(e in base_event_query(site, query), select: %{})
+ |> select_event_metrics(metrics)
|> merge_imported(site, query, :aggregate, metrics)
- |> maybe_add_conversion_rate(site, query, metrics, include_imported: query.include_imported)
|> ClickhouseRepo.one()
end
defp aggregate_sessions(_, _, []), do: %{}
defp aggregate_sessions(site, query, metrics) do
- from(e in query_sessions(site, query), select: ^select_session_metrics(metrics, query))
+ from(e in query_sessions(site, query), select: %{})
|> filter_converted_sessions(site, query)
+ |> select_session_metrics(metrics, query)
|> merge_imported(site, query, :aggregate, metrics)
|> ClickhouseRepo.one()
|> Util.keep_requested_metrics(metrics)
diff --git a/lib/plausible/stats/base.ex b/lib/plausible/stats/base.ex
index 463461bf8b..83b4ad6d28 100644
--- a/lib/plausible/stats/base.ex
+++ b/lib/plausible/stats/base.ex
@@ -209,152 +209,163 @@ defmodule Plausible.Stats.Base do
def apply_entry_prop_filter(sessions_q, _, _), do: sessions_q
- def select_event_metrics(metrics) do
- metrics
- |> Enum.map(&select_event_metric/1)
- |> Enum.reduce(%{}, &Map.merge/2)
- end
+ def select_event_metrics(q, []), do: q
- defp select_event_metric(:pageviews) do
- %{
- pageviews:
- dynamic(
- [e],
+ def select_event_metrics(q, [:pageviews | rest]) do
+ from(e in q,
+ select_merge: %{
+ pageviews:
fragment("toUInt64(round(countIf(? = 'pageview') * any(_sample_factor)))", e.name)
- )
- }
+ }
+ )
+ |> select_event_metrics(rest)
end
- defp select_event_metric(:events) do
- %{
- events: dynamic([], fragment("toUInt64(round(count(*) * any(_sample_factor)))"))
- }
+ def select_event_metrics(q, [:events | rest]) do
+ from(e in q,
+ select_merge: %{events: fragment("toUInt64(round(count(*) * any(_sample_factor)))")}
+ )
+ |> select_event_metrics(rest)
end
- defp select_event_metric(:visitors) do
- %{
- visitors: dynamic([e], selected_as(fragment(@uniq_users_expression, e.user_id), :visitors))
- }
+ def select_event_metrics(q, [:visitors | rest]) do
+ from(e in q,
+ select_merge: %{
+ visitors: selected_as(fragment(@uniq_users_expression, e.user_id), :visitors)
+ }
+ )
+ |> select_event_metrics(rest)
end
on_full_build do
- defp select_event_metric(:total_revenue) do
- %{total_revenue: Plausible.Stats.Goal.Revenue.total_revenue_query()}
+ def select_event_metrics(q, [:total_revenue | rest]) do
+ q
+ |> Plausible.Stats.Goal.Revenue.total_revenue_query()
+ |> select_event_metrics(rest)
end
- defp select_event_metric(:average_revenue) do
- %{average_revenue: Plausible.Stats.Goal.Revenue.average_revenue_query()}
+ def select_event_metrics(q, [:average_revenue | rest]) do
+ q
+ |> Plausible.Stats.Goal.Revenue.average_revenue_query()
+ |> select_event_metrics(rest)
end
end
- defp select_event_metric(:sample_percent) do
- %{
- sample_percent:
- dynamic(
- [],
+ def select_event_metrics(q, [:sample_percent | rest]) do
+ from(e in q,
+ select_merge: %{
+ sample_percent:
fragment("if(any(_sample_factor) > 1, round(100 / any(_sample_factor)), 100)")
- )
- }
+ }
+ )
+ |> select_event_metrics(rest)
end
- defp select_event_metric(:percentage), do: %{}
- defp select_event_metric(:conversion_rate), do: %{}
- defp select_event_metric(:total_visitors), do: %{}
+ def select_event_metrics(q, [:percentage | rest]) do
+ q |> select_event_metrics(rest)
+ end
- defp select_event_metric(unknown), do: raise("Unknown metric: #{unknown}")
+ def select_event_metrics(_, [unknown | _]), do: raise("Unknown metric: #{unknown}")
- def select_session_metrics(metrics, query) do
- metrics
- |> Enum.map(&select_session_metric(&1, query))
- |> Enum.reduce(%{}, &Map.merge/2)
- end
+ def select_session_metrics(q, [], _query), do: q
- defp select_session_metric(:bounce_rate, query) do
+ def select_session_metrics(q, [:bounce_rate | rest], query) do
condition = dynamic_filter_condition(query, "event:page", :entry_page)
- %{
- bounce_rate:
- dynamic(
- [],
- fragment(
- "toUInt32(ifNotFinite(round(sumIf(is_bounce * sign, ?) / sumIf(sign, ?) * 100), 0))",
- ^condition,
- ^condition
- )
- ),
- __internal_visits: dynamic([], fragment("toUInt32(sum(sign))"))
- }
+ from(s in q,
+ select_merge:
+ ^%{
+ bounce_rate:
+ dynamic(
+ [],
+ fragment(
+ "toUInt32(ifNotFinite(round(sumIf(is_bounce * sign, ?) / sumIf(sign, ?) * 100), 0))",
+ ^condition,
+ ^condition
+ )
+ ),
+ __internal_visits: dynamic([], fragment("toUInt32(sum(sign))"))
+ }
+ )
+ |> select_session_metrics(rest, query)
end
- defp select_session_metric(:visits, _query) do
- %{
- visits: dynamic([s], fragment("toUInt64(round(sum(?) * any(_sample_factor)))", s.sign))
- }
+ def select_session_metrics(q, [:visits | rest], query) do
+ from(s in q,
+ select_merge: %{
+ visits: fragment("toUInt64(round(sum(?) * any(_sample_factor)))", s.sign)
+ }
+ )
+ |> select_session_metrics(rest, query)
end
- defp select_session_metric(:pageviews, _query) do
- %{
- pageviews:
- dynamic(
- [s],
+ def select_session_metrics(q, [:pageviews | rest], query) do
+ from(s in q,
+ select_merge: %{
+ pageviews:
fragment("toUInt64(round(sum(? * ?) * any(_sample_factor)))", s.sign, s.pageviews)
- )
- }
+ }
+ )
+ |> select_session_metrics(rest, query)
end
- defp select_session_metric(:events, _query) do
- %{
- events:
- dynamic(
- [s],
- fragment("toUInt64(round(sum(? * ?) * any(_sample_factor)))", s.sign, s.events)
- )
- }
+ def select_session_metrics(q, [:events | rest], query) do
+ from(s in q,
+ select_merge: %{
+ events: fragment("toUInt64(round(sum(? * ?) * any(_sample_factor)))", s.sign, s.events)
+ }
+ )
+ |> select_session_metrics(rest, query)
end
- defp select_session_metric(:visitors, _query) do
- %{
- visitors:
- dynamic(
- [s],
+ def select_session_metrics(q, [:visitors | rest], query) do
+ from(s in q,
+ select_merge: %{
+ visitors:
selected_as(
fragment("toUInt64(round(uniq(?) * any(_sample_factor)))", s.user_id),
:visitors
)
- )
- }
+ }
+ )
+ |> select_session_metrics(rest, query)
end
- defp select_session_metric(:visit_duration, _query) do
- %{
- visit_duration:
- dynamic([], fragment("toUInt32(ifNotFinite(round(sum(duration * sign) / sum(sign)), 0))")),
- __internal_visits: dynamic([], fragment("toUInt32(sum(sign))"))
- }
+ def select_session_metrics(q, [:visit_duration | rest], query) do
+ from(s in q,
+ select_merge: %{
+ :visit_duration =>
+ fragment("toUInt32(ifNotFinite(round(sum(duration * sign) / sum(sign)), 0))"),
+ __internal_visits: fragment("toUInt32(sum(sign))")
+ }
+ )
+ |> select_session_metrics(rest, query)
end
- defp select_session_metric(:views_per_visit, _query) do
- %{
- views_per_visit:
- dynamic(
- [s],
- fragment("ifNotFinite(round(sum(? * ?) / sum(?), 2), 0)", s.sign, s.pageviews, s.sign)
- ),
- __internal_visits: dynamic([], fragment("toUInt32(sum(sign))"))
- }
+ def select_session_metrics(q, [:views_per_visit | rest], query) do
+ from(s in q,
+ select_merge: %{
+ views_per_visit:
+ fragment("ifNotFinite(round(sum(? * ?) / sum(?), 2), 0)", s.sign, s.pageviews, s.sign),
+ __internal_visits: fragment("toUInt32(sum(sign))")
+ }
+ )
+ |> select_session_metrics(rest, query)
end
- defp select_session_metric(:sample_percent, _query) do
- %{
- sample_percent:
- dynamic(
- [],
+ def select_session_metrics(q, [:sample_percent | rest], query) do
+ from(e in q,
+ select_merge: %{
+ sample_percent:
fragment("if(any(_sample_factor) > 1, round(100 / any(_sample_factor)), 100)")
- )
- }
+ }
+ )
+ |> select_session_metrics(rest, query)
end
- defp select_session_metric(:percentage, _query), do: %{}
+ def select_session_metrics(q, [:percentage | rest], query) do
+ q |> select_session_metrics(rest, query)
+ end
def dynamic_filter_condition(query, filter_key, db_field) do
case query && query.filters && query.filters[filter_key] do
@@ -575,7 +586,7 @@ defmodule Plausible.Stats.Base do
|> select([e], total_visitors: fragment(@uniq_users_expression, e.user_id))
end
- defp total_visitors_subquery(site, query, true) do
+ defp total_visitors_subquery(site, %Query{include_imported: true} = query) do
dynamic(
[e],
selected_as(
@@ -586,16 +597,14 @@ defmodule Plausible.Stats.Base do
)
end
- defp total_visitors_subquery(site, query, false) do
+ defp total_visitors_subquery(site, query) do
dynamic([e], selected_as(subquery(total_visitors(site, query)), :__total_visitors))
end
def add_percentage_metric(q, site, query, metrics) do
if :percentage in metrics do
q
- |> select_merge(
- ^%{__total_visitors: total_visitors_subquery(site, query, query.include_imported)}
- )
+ |> select_merge(^%{__total_visitors: total_visitors_subquery(site, query)})
|> select_merge(%{
percentage:
fragment(
@@ -609,32 +618,4 @@ defmodule Plausible.Stats.Base do
q
end
end
-
- # Adds conversion_rate metric to query, calculated as
- # X / Y where Y is the same breakdown value without goal or props
- # filters.
- def maybe_add_conversion_rate(q, site, query, metrics, opts) do
- if :conversion_rate in metrics do
- include_imported = Keyword.fetch!(opts, :include_imported)
-
- total_query = query |> Query.remove_event_filters([:goal, :props])
-
- # :TRICKY: Subquery is used due to event:goal breakdown above doing an UNION ALL
- subquery(q)
- |> select_merge(
- ^%{total_visitors: total_visitors_subquery(site, total_query, include_imported)}
- )
- |> select_merge([e], %{
- conversion_rate:
- fragment(
- "if(? > 0, round(? / ? * 100, 1), 0)",
- selected_as(:__total_visitors),
- e.visitors,
- selected_as(:__total_visitors)
- )
- })
- else
- q
- end
- end
end
diff --git a/lib/plausible/stats/breakdown.ex b/lib/plausible/stats/breakdown.ex
index 822c00df1d..ae33e4e737 100644
--- a/lib/plausible/stats/breakdown.ex
+++ b/lib/plausible/stats/breakdown.ex
@@ -4,7 +4,6 @@ defmodule Plausible.Stats.Breakdown do
use Plausible.Stats.Fragments
import Plausible.Stats.{Base, Imported}
- import Ecto.Query
require OpenTelemetry.Tracer, as: Tracer
alias Plausible.Stats.{Query, Util}
@@ -53,74 +52,50 @@ defmodule Plausible.Stats.Breakdown do
metrics_to_select = Util.maybe_add_visitors_metric(metrics) -- @computed_metrics
- event_q =
+ event_results =
if Enum.any?(event_goals) do
site
- |> breakdown_events(event_query, "event:name", metrics_to_select)
- |> apply_pagination(pagination)
+ |> breakdown(event_query, "event:name", metrics_to_select, pagination, skip_tracing: true)
+ |> transform_keys(%{name: :goal})
+ |> cast_revenue_metrics_to_money(revenue_goals)
else
- nil
+ []
end
- page_q =
+ {limit, page} = pagination
+ offset = (page - 1) * limit
+
+ page_results =
if Enum.any?(pageview_goals) do
page_exprs = Enum.map(pageview_goals, & &1.page_path)
page_regexes = Enum.map(page_exprs, &page_regex/1)
- select_columns = metrics_to_select |> select_event_metrics |> mark_revenue_as_nil
-
from(e in base_event_query(site, query),
order_by: [desc: fragment("uniq(?)", e.user_id)],
+ limit: ^limit,
+ offset: ^offset,
where:
fragment(
"notEmpty(multiMatchAllIndices(?, ?) as indices)",
e.pathname,
^page_regexes
) and e.name == "pageview",
- array_join: index in fragment("indices"),
- group_by: index,
- select: %{}
+ group_by: fragment("index"),
+ select: %{
+ index: fragment("arrayJoin(indices) as index"),
+ goal: fragment("concat('Visit ', ?[index])", ^page_exprs)
+ }
)
- |> select_merge(^select_columns)
- # :TRICKY: name is added last to make sure both queries add columns in the same order
- |> select_merge([_, index], %{
- name: fragment("concat('Visit ', ?[?])", ^page_exprs, index)
- })
- |> apply_pagination(pagination)
+ |> select_event_metrics(metrics_to_select -- @revenue_metrics)
+ |> ClickhouseRepo.all()
+ |> Enum.map(fn row -> Map.delete(row, :index) end)
else
- nil
- end
-
- full_q =
- case {event_q, page_q} do
- {nil, nil} ->
- nil
-
- {event_q, nil} ->
- event_q
-
- {nil, page_q} ->
- page_q
-
- {event_q, page_q} ->
- from(
- e in subquery(union_all(event_q, ^page_q)),
- # :TODO: Handle other orderings
- order_by: [desc: e.visitors]
- )
- |> apply_pagination(pagination)
+ []
end
- if full_q do
- full_q
- |> maybe_add_conversion_rate(site, query, metrics, include_imported: false)
- |> ClickhouseRepo.all()
- |> transform_keys(%{name: :goal})
- |> cast_revenue_metrics_to_money(revenue_goals)
- |> Util.keep_requested_metrics(metrics)
- else
- []
- end
+ zip_results(event_results, page_results, :goal, metrics_to_select)
+ |> maybe_add_cr(site, query, nil, metrics)
+ |> Util.keep_requested_metrics(metrics)
end
def breakdown(site, query, "event:props:" <> custom_prop = property, metrics, pagination, opts) do
@@ -135,11 +110,11 @@ defmodule Plausible.Stats.Breakdown do
if !Keyword.get(opts, :skip_tracing), do: trace(query, property, metrics)
- breakdown_events(site, query, "event:props:" <> custom_prop, metrics_to_select)
- |> maybe_add_conversion_rate(site, query, metrics, include_imported: false)
- |> paginate_and_execute(metrics, pagination)
- |> transform_keys(%{name: custom_prop})
+ breakdown_events(site, query, "event:props:" <> custom_prop, metrics_to_select, pagination)
|> Enum.map(&cast_revenue_metrics_to_money(&1, currency))
+ |> sort_results(metrics_to_select)
+ |> maybe_add_cr(site, query, nil, metrics)
+ |> Util.keep_requested_metrics(metrics)
end
def breakdown(site, query, "event:page" = property, metrics, pagination, opts) do
@@ -150,10 +125,10 @@ defmodule Plausible.Stats.Breakdown do
event_result =
site
- |> breakdown_events(query, property, event_metrics)
- |> maybe_add_group_conversion_rate(&breakdown_events/4, site, query, property, metrics)
- |> paginate_and_execute(metrics, pagination)
+ |> breakdown_events(query, "event:page", event_metrics, pagination)
|> maybe_add_time_on_page(site, query, metrics)
+ |> maybe_add_cr(site, query, property, metrics, pagination)
+ |> Util.keep_requested_metrics(metrics)
session_metrics = Enum.filter(metrics, &(&1 in @session_metrics))
@@ -174,8 +149,7 @@ defmodule Plausible.Stats.Breakdown do
{limit, _page} = pagination
session_result =
- breakdown_sessions(site, new_query, "visit:entry_page", session_metrics)
- |> paginate_and_execute(session_metrics, {limit, 1})
+ breakdown_sessions(site, new_query, "visit:entry_page", session_metrics, {limit, 1})
|> transform_keys(%{entry_page: :page})
metrics = metrics ++ [:page]
@@ -192,9 +166,7 @@ defmodule Plausible.Stats.Breakdown do
def breakdown(site, query, "event:name" = property, metrics, pagination, opts) do
if !Keyword.get(opts, :skip_tracing), do: trace(query, property, metrics)
-
- breakdown_events(site, query, property, metrics)
- |> paginate_and_execute(metrics, pagination)
+ breakdown_events(site, query, property, metrics, pagination)
end
def breakdown(site, query, property, metrics, pagination, opts) do
@@ -202,9 +174,9 @@ defmodule Plausible.Stats.Breakdown do
metrics_to_select = Util.maybe_add_visitors_metric(metrics) -- @computed_metrics
- breakdown_sessions(site, query, property, metrics_to_select)
- |> maybe_add_group_conversion_rate(&breakdown_sessions/4, site, query, property, metrics)
- |> paginate_and_execute(metrics, pagination)
+ breakdown_sessions(site, query, property, metrics_to_select, pagination)
+ |> maybe_add_cr(site, query, property, metrics, pagination)
+ |> Util.keep_requested_metrics(metrics)
end
defp zip_results(event_result, session_result, property, metrics) do
@@ -225,31 +197,35 @@ defmodule Plausible.Stats.Breakdown do
|> sort_results(metrics)
end
- defp breakdown_sessions(site, query, property, metrics) do
+ defp breakdown_sessions(_, _, _, [], _), do: []
+
+ defp breakdown_sessions(site, query, property, metrics, pagination) do
from(s in query_sessions(site, query),
order_by: [desc: fragment("uniq(?)", s.user_id)],
- select: ^select_session_metrics(metrics, query)
+ select: %{}
)
|> filter_converted_sessions(site, query)
|> do_group_by(property)
+ |> select_session_metrics(metrics, query)
|> merge_imported(site, query, property, metrics)
|> add_percentage_metric(site, query, metrics)
+ |> apply_pagination(pagination)
+ |> ClickhouseRepo.all()
+ |> transform_keys(%{operating_system: :os})
+ |> Util.keep_requested_metrics(metrics)
end
- defp breakdown_events(site, query, property, metrics) do
+ defp breakdown_events(_, _, _, [], _), do: []
+
+ defp breakdown_events(site, query, property, metrics, pagination) do
from(e in base_event_query(site, query),
order_by: [desc: fragment("uniq(?)", e.user_id)],
- select: ^select_event_metrics(metrics)
+ select: %{}
)
|> do_group_by(property)
+ |> select_event_metrics(metrics)
|> merge_imported(site, query, property, metrics)
|> add_percentage_metric(site, query, metrics)
- end
-
- defp paginate_and_execute(_, [], _), do: []
-
- defp paginate_and_execute(q, metrics, pagination) do
- q
|> apply_pagination(pagination)
|> ClickhouseRepo.all()
|> transform_keys(%{operating_system: :os})
@@ -447,18 +423,18 @@ defmodule Plausible.Stats.Breakdown do
from(
e in q,
select_merge: %{
- name:
+ ^prop =>
selected_as(
fragment(
"if(not empty(?), ?, '(none)')",
get_by_key(e, :meta, ^prop),
get_by_key(e, :meta, ^prop)
),
- :name
+ :breakdown_prop_value
)
},
- group_by: selected_as(:name),
- order_by: {:asc, selected_as(:name)}
+ group_by: selected_as(:breakdown_prop_value),
+ order_by: {:asc, selected_as(:breakdown_prop_value)}
)
end
@@ -676,26 +652,130 @@ defmodule Plausible.Stats.Breakdown do
)
end
- defp group_by_field_names("event:props:" <> _prop), do: [:name]
- defp group_by_field_names("visit:os"), do: [:operating_system]
- defp group_by_field_names("visit:os_version"), do: [:os, :os_version]
- defp group_by_field_names("visit:browser_version"), do: [:browser, :browser_version]
+ defp maybe_add_cr(breakdown_results, site, query, property, metrics, pagination \\ nil) do
+ cond do
+ :conversion_rate not in metrics -> breakdown_results
+ Enum.empty?(breakdown_results) -> breakdown_results
+ is_nil(property) -> add_absolute_cr(breakdown_results, site, query)
+ true -> add_cr(breakdown_results, site, query, property, metrics, pagination)
+ end
+ end
+
+ # This function injects a conversion_rate metric into every
+ # breakdown result map. It is calculated as X / Y, where:
+ #
+ # * X is the number of conversions for a breakdown
+ # result (conversion = number of visitors who
+ # completed the filtered goal with the filtered
+ # custom properties).
+ #
+ # * Y is the number of all visitors for this breakdown
+ # result without the `event:goal` and `event:props:*`
+ # filters.
+ defp add_cr(breakdown_results, site, query, property, metrics, pagination) do
+ property_atom = Plausible.Stats.Filters.without_prefix(property)
+
+ items =
+ Enum.map(breakdown_results, fn item -> Map.fetch!(item, property_atom) end)
+
+ query_without_goal =
+ query
+ |> Query.put_filter(property, {:member, items})
+ |> Query.remove_event_filters([:goal, :props])
+
+ # Here, we're always only interested in the first page of results
+ # - the :member filter makes sure that the results always match with
+ # the items in the given breakdown_results list
+ page = 1
+
+ # For browser/os versions we need to fetch a lot more entries than the
+ # pagination limit. This is because many entries can correspond to a
+ # single version number and we need to make sure that the results
+ # without goal filter will include all those combinations of browsers/os-s
+ # and their versions that were present in the `breakdown_results` array.
+ {pagination_limit, find_match_fn} =
+ case property_atom do
+ :browser_version ->
+ pagination_limit = min(elem(pagination, 0) * 10, 3_000)
+
+ find_match_fn = fn total, conversion ->
+ total[:browser_version] == conversion[:browser_version] &&
+ total[:browser] == conversion[:browser]
+ end
+
+ {pagination_limit, find_match_fn}
+
+ :os_version ->
+ pagination_limit = min(elem(pagination, 0) * 5, 3_000)
+
+ find_match_fn = fn total, conversion ->
+ total[:os_version] == conversion[:os_version] &&
+ total[:os] == conversion[:os]
+ end
+
+ {pagination_limit, find_match_fn}
+
+ _ ->
+ {elem(pagination, 0),
+ fn total, conversion ->
+ total[property_atom] == conversion[property_atom]
+ end}
+ end
+
+ pagination = {pagination_limit, page}
- defp group_by_field_names(property), do: [Plausible.Stats.Filters.without_prefix(property)]
+ res_without_goal = breakdown(site, query_without_goal, property, [:visitors], pagination)
- defp on_matches_group_by(fields) do
- Enum.reduce(fields, nil, &fields_equal/2)
- end
+ Enum.map(breakdown_results, fn item ->
+ without_goal = Enum.find(res_without_goal, &find_match_fn.(&1, item))
- defp outer_order_by(fields) do
- Enum.map(fields, fn field_name -> {:asc, dynamic([q], field(q, ^field_name))} end)
+ {conversion_rate, total_visitors} =
+ if without_goal do
+ {Util.calculate_cr(without_goal.visitors, item.visitors), without_goal.visitors}
+ else
+ Sentry.capture_message(
+ "Unable to find a conversion_rate divisor from a breakdown response",
+ extra: %{
+ domain: site.domain,
+ query: inspect(query),
+ property: property,
+ pagination: inspect(pagination),
+ item_not_found: inspect(item)
+ }
+ )
+
+ {"N/A", "N/A"}
+ end
+
+ if :total_visitors in metrics do
+ item
+ |> Map.put(:conversion_rate, conversion_rate)
+ |> Map.put(:total_visitors, total_visitors)
+ else
+ Map.put(item, :conversion_rate, conversion_rate)
+ end
+ end)
end
- defp fields_equal(field_name, nil),
- do: dynamic([a, b], field(a, ^field_name) == field(b, ^field_name))
+ # Similar to `add_cr/5`, injects a conversion_rate metric into
+ # every breakdown result. However, a single divisor is used in
+ # the CR calculation across all breakdown results. That is the
+ # number of visitors without `event:goal` and `event:props:*`
+ # filters.
+ #
+ # This is useful when we're only interested in the conversions
+ # themselves - not how well a certain property such as browser
+ # or page converted.
+ defp add_absolute_cr(breakdown_results, site, query) do
+ total_q = Query.remove_event_filters(query, [:goal, :props])
+
+ %{visitors: %{value: total_visitors}} = Plausible.Stats.aggregate(site, total_q, [:visitors])
- defp fields_equal(field_name, condition),
- do: dynamic([a, b], field(a, ^field_name) == field(b, ^field_name) and ^condition)
+ breakdown_results
+ |> Enum.map(fn goal ->
+ Map.put(goal, :conversion_rate, Util.calculate_cr(total_visitors, goal[:visitors]))
+ end)
+ end
defp sort_results(results, metrics) do
Enum.sort_by(
@@ -710,54 +790,6 @@ defmodule Plausible.Stats.Breakdown do
)
end
- # This function injects a conversion_rate metric into
- # a breakdown query. It is calculated as X / Y, where:
- #
- # * X is the number of conversions for a breakdown
- # result (conversion = number of visitors who
- # completed the filtered goal with the filtered
- # custom properties).
- #
- # * Y is the number of all visitors for this breakdown
- # result without the `event:goal` and `event:props:*`
- # filters.
- defp maybe_add_group_conversion_rate(q, breakdown_fn, site, query, property, metrics) do
- if :conversion_rate in metrics do
- breakdown_total_visitors_query = query |> Query.remove_event_filters([:goal, :props])
-
- breakdown_total_visitors_q =
- breakdown_fn.(site, breakdown_total_visitors_query, property, [:visitors])
-
- from(e in subquery(q),
- left_join: c in subquery(breakdown_total_visitors_q),
- on: ^on_matches_group_by(group_by_field_names(property)),
- select_merge: %{
- total_visitors: c.visitors,
- conversion_rate:
- fragment(
- "if(? > 0, round(? / ? * 100, 1), 0)",
- c.visitors,
- e.visitors,
- c.visitors
- )
- },
- order_by: [desc: e.visitors],
- order_by: ^outer_order_by(group_by_field_names(property))
- )
- else
- q
- end
- end
-
- # When querying custom event goals and pageviewgoals together, UNION ALL is used
- # so the same fields must be present on both sides of the union. This change to the
- # query will ensure that we don't unnecessarily read revenue column for pageview goals
- defp mark_revenue_as_nil(select_columns) do
- select_columns
- |> Map.replace(:total_revenue, nil)
- |> Map.replace(:average_revenue, nil)
- end
-
defp sorting_key(metrics) do
if Enum.member?(metrics, :visitors), do: :visitors, else: List.first(metrics)
end
@@ -775,8 +807,8 @@ defmodule Plausible.Stats.Breakdown do
offset = (page - 1) * limit
q
- |> limit(^limit)
- |> offset(^offset)
+ |> Ecto.Query.limit(^limit)
+ |> Ecto.Query.offset(^offset)
end
defp trace(query, property, metrics) do
diff --git a/lib/plausible/stats/timeseries.ex b/lib/plausible/stats/timeseries.ex
index 053625a90a..4c31aabd03 100644
--- a/lib/plausible/stats/timeseries.ex
+++ b/lib/plausible/stats/timeseries.ex
@@ -53,8 +53,9 @@ defmodule Plausible.Stats.Timeseries do
defp events_timeseries(_, _, []), do: []
defp events_timeseries(site, query, metrics) do
- from(e in base_event_query(site, query), select: ^select_event_metrics(metrics))
+ from(e in base_event_query(site, query), select: %{})
|> select_bucket(site, query)
+ |> select_event_metrics(metrics)
|> Plausible.Stats.Imported.merge_imported_timeseries(site, query, metrics)
|> ClickhouseRepo.all()
end
@@ -62,9 +63,10 @@ defmodule Plausible.Stats.Timeseries do
defp sessions_timeseries(_, _, []), do: []
defp sessions_timeseries(site, query, metrics) do
- from(e in query_sessions(site, query), select: ^select_session_metrics(metrics, query))
+ from(e in query_sessions(site, query), select: %{})
|> filter_converted_sessions(site, query)
|> select_bucket(site, query)
+ |> select_session_metrics(metrics, query)
|> Plausible.Stats.Imported.merge_imported_timeseries(site, query, metrics)
|> ClickhouseRepo.all()
|> Util.keep_requested_metrics(metrics)
diff --git a/lib/plausible/stats/util.ex b/lib/plausible/stats/util.ex
index 85c36888f8..10078ff6a3 100644
--- a/lib/plausible/stats/util.ex
+++ b/lib/plausible/stats/util.ex
@@ -7,8 +7,7 @@ defmodule Plausible.Stats.Util do
:__internal_visits,
:visitors,
:__total_visitors,
- :__breakdown_value,
- :total_visitors
+ :__breakdown_value
]
@doc """
@@ -49,4 +48,12 @@ defmodule Plausible.Stats.Util do
metrics
end
end
+
+ def calculate_cr(nil, _converted_visitors), do: nil
+
+ def calculate_cr(unique_visitors, converted_visitors) do
+ if unique_visitors > 0,
+ do: Float.round(converted_visitors / unique_visitors * 100, 1),
+ else: 0.0
+ end
end
diff --git a/test/plausible_web/controllers/api/external_stats_controller/aggregate_test.exs b/test/plausible_web/controllers/api/external_stats_controller/aggregate_test.exs
index 6f0e35092a..325987defc 100644
--- a/test/plausible_web/controllers/api/external_stats_controller/aggregate_test.exs
+++ b/test/plausible_web/controllers/api/external_stats_controller/aggregate_test.exs
@@ -1569,21 +1569,5 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest do
} =
json_response(conn, 200)["results"]
end
-
- test "conversion_rate for the filtered goal is 0 when no stats exist", %{
- conn: conn,
- site: site
- } do
- insert(:goal, %{site: site, event_name: "Signup"})
-
- conn =
- get(conn, "/api/v1/stats/aggregate", %{
- "site_id" => site.domain,
- "metrics" => "conversion_rate",
- "filters" => "event:goal==Signup"
- })
-
- assert json_response(conn, 200)["results"] == %{"conversion_rate" => %{"value" => 0}}
- end
end
end
diff --git a/test/plausible_web/controllers/api/stats_controller/conversions_test.exs b/test/plausible_web/controllers/api/stats_controller/conversions_test.exs
index 9dd09736ea..a454f59f69 100644
--- a/test/plausible_web/controllers/api/stats_controller/conversions_test.exs
+++ b/test/plausible_web/controllers/api/stats_controller/conversions_test.exs
@@ -580,42 +580,6 @@ defmodule PlausibleWeb.Api.StatsController.ConversionsTest do
}
]
end
-
- test "conversion_rate for goals should not be calculated with imported data", %{
- conn: conn,
- site: site
- } do
- site =
- site
- |> Plausible.Site.start_import(~D[2005-01-01], Timex.today(), "Google Analytics", "ok")
- |> Plausible.Repo.update!()
-
- populate_stats(site, [
- build(:pageview, pathname: "/"),
- build(:pageview, pathname: "/another"),
- build(:pageview, pathname: "/blog/post-1"),
- build(:pageview, pathname: "/blog/post-2"),
- build(:imported_pages, page: "/blog/post-1"),
- build(:imported_visitors)
- ])
-
- insert(:goal, %{site: site, page_path: "/blog**"})
-
- conn =
- get(
- conn,
- "/api/stats/#{site.domain}/conversions?period=day"
- )
-
- assert json_response(conn, 200) == [
- %{
- "name" => "Visit /blog**",
- "visitors" => 2,
- "events" => 2,
- "conversion_rate" => 50
- }
- ]
- end
end
describe "GET /api/stats/:domain/conversions - with goal and prop=(none) filter" do