Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.
TypeScript analyticsanalytics-dashboardclickhouseelixirgoogle-analyticsmarketingopen-source-analyticsphoenixplausible-analyticspostgresqlprivacyprivacy-friendlyself-hostedsimple-analyticsstatisticstailwindcssweb-analyticswebsite-analyticswebsite-statswebsite-tracking
Reapply conversion rate into Stats API + bugfixes (#3805)
RobertJoonasd74b1d5e6044
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex ece2d5aba2..be421dcacb 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -2,6 +2,9 @@All notable changes to this project will be documented in this file.### Added+- Display `Total visitors`, `Conversions`, and `CR` in the "Details" views of Countries, Regions and Cities (when filtering by a goal)+- Add `conversion_rate` to Regions and Cities reports (when filtering by a goal)+- Add the `conversion_rate` metric to Stats API Breakdown and Aggregate endpoints- IP Block List in Site Settings- Allow filtering with `contains`/`matches` operator for Sources, Browsers and Operating Systems.- Allow filtering by multiple custom properties@@ -37,6 +40,9 @@ All notable changes to this project will be documented in this file.- Replace `CLICKHOUSE_MAX_BUFFER_SIZE` with `CLICKHOUSE_MAX_BUFFER_SIZE_BYTES`### Fixed+- Fix `conversion_rate` metric in a `browser_versions` breakdown+- Calculate `conversion_rate` percentage change in the same way like `bounce_rate` (subtraction instead of division)+- Calculate `bounce_rate` percentage change in the Stats API in the same way as it's done in the dashboard- Stop returning custom events in goal breakdown with a pageview goal filter and vice versa- Only return `(none)` values in custom property breakdown for the first page (pagination) of results- Fixed weekly/monthly e-mail report [rendering issues](https://github.com/plausible/analytics/issues/284)diff --git a/assets/js/dashboard/router.js b/assets/js/dashboard/router.jsindex 5aedfde737..aaad57ff9f 100644--- a/assets/js/dashboard/router.js+++ b/assets/js/dashboard/router.js@@ -52,7 +52,7 @@ export default function Router({ site, loggedIn, currentUserRole }) {<ExitPagesModal site={site} /></Route><Route path="/:domain/countries">- <ModalTable title="Top countries" site={site} endpoint={url.apiPath(site, '/countries')} filter={{ country: 'code', country_labels: 'name' }} keyLabel="Country" renderIcon={renderCountryIcon} />+ <ModalTable title="Top countries" site={site} endpoint={url.apiPath(site, '/countries')} filter={{ country: 'code', country_labels: 'name' }} keyLabel="Country" renderIcon={renderCountryIcon} showPercentage={true}/></Route><Route path="/:domain/regions"><ModalTable title="Top regions" site={site} endpoint={url.apiPath(site, '/regions')} filter={{ region: 'code', region_labels: 'name' }} keyLabel="Region" renderIcon={renderRegionIcon} />diff --git a/assets/js/dashboard/stats/locations/index.js b/assets/js/dashboard/stats/locations/index.jsindex 4a86e515d0..fb96eea7d7 100644--- a/assets/js/dashboard/stats/locations/index.js+++ b/assets/js/dashboard/stats/locations/index.js@@ -57,7 +57,7 @@ function Regions({query, site, onClick}) {getFilterFor={getFilterFor}onClick={onClick}keyLabel="Region"- metrics={[VISITORS_METRIC]}+ metrics={maybeWithCR([VISITORS_METRIC], query)}detailsLink={sitePath(site, '/regions')}query={query}renderIcon={renderIcon}@@ -84,7 +84,7 @@ function Cities({query, site}) {fetchData={fetchData}getFilterFor={getFilterFor}keyLabel="City"- metrics={[VISITORS_METRIC]}+ metrics={maybeWithCR([VISITORS_METRIC], query)}detailsLink={sitePath(site, '/cities')}query={query}renderIcon={renderIcon}diff --git a/assets/js/dashboard/stats/modals/table.js b/assets/js/dashboard/stats/modals/table.jsindex f97655e6b5..0d7f157ab6 100644--- a/assets/js/dashboard/stats/modals/table.js+++ b/assets/js/dashboard/stats/modals/table.js@@ -20,8 +20,24 @@ class ModalTable extends React.Component {.then((res) => this.setState({loading: false, list: res}))}+ showConversionRate() {+ return !!this.state.query.filters.goal+ }++ showPercentage() {+ return this.props.showPercentage && !this.showConversionRate()+ }+label() {- return this.state.query.period === 'realtime' ? 'Current visitors' : 'Visitors'+ if (this.state.query.period === 'realtime') {+ return 'Current visitors'+ }++ if (this.showConversionRate()) {+ return 'Conversions'+ }++ return 'Visitors'}renderTableItem(tableItem) {@@ -40,11 +56,10 @@ class ModalTable extends React.Component {{tableItem.name}</Link></td>- <td className="p-2 w-32 font-medium" align="right">- {numberFormatter(tableItem.visitors)}- {tableItem.percentage >= 0 &&- <span className="inline-block text-xs w-8 pl-1 text-right">({tableItem.percentage}%)</span> }- </td>+ {this.showConversionRate() && <td className="p-2 w-32 font-medium" align="right">{numberFormatter(tableItem.total_visitors)}</td>}+ <td className="p-2 w-32 font-medium" align="right">{numberFormatter(tableItem.visitors)}</td>+ {this.showPercentage() && <td className="p-2 w-32 font-medium" align="right">{tableItem.percentage}</td>}+ {this.showConversionRate() && <td className="p-2 w-32 font-medium" align="right">{numberFormatter(tableItem.conversion_rate)}%</td>}</tr>)}@@ -66,19 +81,11 @@ class ModalTable extends React.Component {<table className="w-max overflow-x-auto md:w-full table-striped table-fixed"><thead><tr>- <th- className="p-2 w-48 lg:w-1/2 text-xs tracking-wide font-bold text-gray-500 dark:text-gray-400"- align="left"- >- {this.props.keyLabel}- </th>- <th- // eslint-disable-next-line max-len- className="p-2 w-32 lg:w-1/2 text-xs tracking-wide font-bold text-gray-500 dark:text-gray-400"- align="right"- >- {this.label()}- </th>+ <th className="p-2 w-48 md:w-56 lg:w-1/3 text-xs tracking-wide font-bold text-gray-500 dark:text-gray-400" align="left">{this.props.keyLabel}</th>+ {this.showConversionRate() && <th className="p-2 w-32 text-xs tracking-wide font-bold text-gray-500 dark:text-gray-400" align="right" >Total Visitors</th>}+ <th className="p-2 w-32 text-xs tracking-wide font-bold text-gray-500 dark:text-gray-400" align="right">{this.label()}</th>+ {this.showPercentage() && <th className="p-2 w-32 text-xs tracking-wide font-bold text-gray-500 dark:text-gray-400" align="right">%</th>}+ {this.showConversionRate() && <th className="p-2 w-32 text-xs tracking-wide font-bold text-gray-500 dark:text-gray-400" align="right">CR</th>}</tr></thead><tbody>diff --git a/lib/plausible/stats/aggregate.ex b/lib/plausible/stats/aggregate.exindex 1f14b87091..e356c189d5 100644--- a/lib/plausible/stats/aggregate.ex+++ b/lib/plausible/stats/aggregate.ex@@ -1,9 +1,9 @@defmodule Plausible.Stats.Aggregate do- alias Plausible.Stats.Queryuse Plausible.ClickhouseRepouse Plausible- import Plausible.Stats.{Base, Imported, Util}+ import Plausible.Stats.{Base, Imported}import Ecto.Query+ alias Plausible.Stats.{Query, Util}@revenue_metrics on_full_build(do: Plausible.Stats.Goal.Revenue.revenue_metrics(), else: [])@@ -26,8 +26,13 @@ defmodule Plausible.Stats.Aggregate doQuery.trace(query, metrics)- event_metrics = Enum.filter(metrics, &(&1 in @event_metrics))+ event_metrics =+ metrics+ |> Util.maybe_add_visitors_metric()+ |> Enum.filter(&(&1 in @event_metrics))+event_task = fn -> aggregate_events(site, query, event_metrics) end+session_metrics = Enum.filter(metrics, &(&1 in @session_metrics))session_task = fn -> aggregate_sessions(site, query, session_metrics) end@@ -40,12 +45,38 @@ defmodule Plausible.Stats.Aggregate doPlausible.ClickhouseRepo.parallel_tasks([session_task, event_task, time_on_page_task])|> Enum.reduce(%{}, fn aggregate, task_result -> Map.merge(aggregate, task_result) end)+ |> maybe_put_cr(site, query, metrics)+ |> Util.keep_requested_metrics(metrics)|> cast_revenue_metrics_to_money(currency)|> Enum.map(&maybe_round_value/1)|> Enum.map(fn {metric, value} -> {metric, %{value: value}} end)|> Enum.into(%{})end+ defp maybe_put_cr(aggregate_result, site, query, metrics) do+ if :conversion_rate in metrics do+ all =+ query+ |> Query.remove_event_filters([:goal, :props])+ |> then(fn query -> aggregate_events(site, query, [:visitors]) end)+ |> Map.fetch!(:visitors)++ converted = aggregate_result.visitors++ cr = Util.calculate_cr(all, converted)++ aggregate_result = Map.put(aggregate_result, :conversion_rate, cr)++ if :total_visitors in metrics do+ Map.put(aggregate_result, :total_visitors, all)+ else+ aggregate_result+ end+ else+ aggregate_result+ end+ end+defp aggregate_events(_, _, []), do: %{}defp aggregate_events(site, query, metrics) do@@ -63,7 +94,7 @@ defmodule Plausible.Stats.Aggregate do|> select_session_metrics(metrics, query)|> merge_imported(site, query, :aggregate, metrics)|> ClickhouseRepo.one()- |> remove_internal_visits_metric()+ |> Util.keep_requested_metrics(metrics)enddefp aggregate_time_on_page(site, query) dodiff --git a/lib/plausible/stats/breakdown.ex b/lib/plausible/stats/breakdown.exindex 6586d6cfd6..bbb89091ce 100644--- a/lib/plausible/stats/breakdown.ex+++ b/lib/plausible/stats/breakdown.ex@@ -3,9 +3,9 @@ defmodule Plausible.Stats.Breakdown douse Plausibleuse Plausible.Stats.Fragments- import Plausible.Stats.{Base, Imported, Util}+ import Plausible.Stats.{Base, Imported}require OpenTelemetry.Tracer, as: Tracer- alias Plausible.Stats.Query+ alias Plausible.Stats.{Query, Util}@no_ref "Direct / None"@not_set "(not set)"@@ -16,7 +16,12 @@ defmodule Plausible.Stats.Breakdown do@event_metrics [:visitors, :pageviews, :events] ++ @revenue_metrics- @event_props Plausible.Stats.Props.event_props()+ # These metrics can be asked from the `breakdown/5` function,+ # but they are different from regular metrics such as `visitors`,+ # or `bounce_rate` - we cannot currently "select them" directly in+ # the db queries. Instead, we need to artificially append them to+ # the breakdown results later on.+ @computed_metrics [:conversion_rate, :total_visitors]def breakdown(site, query, property, metrics, pagination, opts \\ [])@@ -29,15 +34,22 @@ defmodule Plausible.Stats.Breakdown doif !Keyword.get(opts, :skip_tracing), do: trace(query, property, metrics)+ {revenue_goals, metrics} =+ if full_build?() && Plausible.Billing.Feature.RevenueGoals.enabled?(site) do+ revenue_goals = Enum.filter(event_goals, &Plausible.Goal.Revenue.revenue?/1)+ metrics = if Enum.empty?(revenue_goals), do: metrics -- @revenue_metrics, else: metrics++ {revenue_goals, metrics}+ else+ {nil, metrics -- @revenue_metrics}+ end++ metrics_to_select = Util.maybe_add_visitors_metric(metrics) -- @computed_metrics+event_results =if Enum.any?(event_goals) do- revenue_goals =- on_full_build do- Enum.filter(event_goals, &Plausible.Goal.Revenue.revenue?/1)- end-site- |> breakdown(event_query, "event:name", metrics, pagination, skip_tracing: true)+ |> breakdown(event_query, "event:name", metrics_to_select, pagination, skip_tracing: true)|> transform_keys(%{name: :goal})|> cast_revenue_metrics_to_money(revenue_goals)else@@ -68,14 +80,16 @@ defmodule Plausible.Stats.Breakdown dogoal: fragment("concat('Visit ', ?[index])", ^page_exprs)})- |> select_event_metrics(metrics -- @revenue_metrics)+ |> select_event_metrics(metrics_to_select -- @revenue_metrics)|> ClickhouseRepo.all()|> Enum.map(fn row -> Map.delete(row, :index) end)else[]end- zip_results(event_results, page_results, :goal, metrics)+ zip_results(event_results, page_results, :goal, metrics_to_select)+ |> maybe_add_cr(site, query, nil, metrics)+ |> Util.keep_requested_metrics(metrics)enddef breakdown(site, query, "event:props:" <> custom_prop = property, metrics, pagination, opts) do@@ -86,6 +100,8 @@ defmodule Plausible.Stats.Breakdown do{nil, metrics}end+ metrics_to_select = Util.maybe_add_visitors_metric(metrics) -- @computed_metrics+{_limit, page} = paginationnone_result =@@ -97,7 +113,7 @@ defmodule Plausible.Stats.Breakdown doselect_merge: %{^custom_prop => "(none)"},having: fragment("uniq(?)", e.user_id) > 0)- |> select_event_metrics(metrics)+ |> select_event_metrics(metrics_to_select)|> ClickhouseRepo.all()else[]@@ -105,29 +121,28 @@ defmodule Plausible.Stats.Breakdown doif !Keyword.get(opts, :skip_tracing), do: trace(query, property, metrics)- breakdown_events(site, query, "event:props:" <> custom_prop, metrics, pagination)+ breakdown_events(site, query, "event:props:" <> custom_prop, metrics_to_select, pagination)|> Kernel.++(none_result)|> Enum.map(&cast_revenue_metrics_to_money(&1, currency))- |> Enum.sort_by(& &1[sorting_key(metrics)], :desc)+ |> Enum.sort_by(& &1[sorting_key(metrics_to_select)], :desc)+ |> maybe_add_cr(site, query, nil, metrics)+ |> Util.keep_requested_metrics(metrics)enddef breakdown(site, query, "event:page" = property, metrics, pagination, opts) do- event_metrics = Enum.filter(metrics, &(&1 in @event_metrics))- session_metrics = Enum.filter(metrics, &(&1 in @session_metrics))-- event_result = breakdown_events(site, query, "event:page", event_metrics, pagination)+ event_metrics =+ metrics+ |> Util.maybe_add_visitors_metric()+ |> Enum.filter(&(&1 in @event_metrics))event_result =- if :time_on_page in metrics do- pages = Enum.map(event_result, & &1[:page])- time_on_page_result = breakdown_time_on_page(site, query, pages)+ site+ |> breakdown_events(query, "event:page", event_metrics, pagination)+ |> maybe_add_time_on_page(site, query, metrics)+ |> maybe_add_cr(site, query, property, metrics, pagination)+ |> Util.keep_requested_metrics(metrics)- Enum.map(event_result, fn row ->- Map.put(row, :time_on_page, time_on_page_result[row[:page]])- end)- else- event_result- end+ session_metrics = Enum.filter(metrics, &(&1 in @session_metrics))new_query =case event_result do@@ -161,14 +176,19 @@ defmodule Plausible.Stats.Breakdown doendend- def breakdown(site, query, property, metrics, pagination, opts) when property in @event_props do+ def breakdown(site, query, "event:name" = property, metrics, pagination, opts) doif !Keyword.get(opts, :skip_tracing), do: trace(query, property, metrics)breakdown_events(site, query, property, metrics, pagination)enddef breakdown(site, query, property, metrics, pagination, opts) doif !Keyword.get(opts, :skip_tracing), do: trace(query, property, metrics)- breakdown_sessions(site, query, property, metrics, pagination)++ metrics_to_select = Util.maybe_add_visitors_metric(metrics) -- @computed_metrics++ breakdown_sessions(site, query, property, metrics_to_select, pagination)+ |> maybe_add_cr(site, query, property, metrics, pagination)+ |> Util.keep_requested_metrics(metrics)enddefp zip_results(event_result, session_result, property, metrics) do@@ -211,7 +231,7 @@ defmodule Plausible.Stats.Breakdown do|> apply_pagination(pagination)|> ClickhouseRepo.all()|> transform_keys(%{operating_system: :os})- |> remove_internal_visits_metric(metrics)+ |> Util.keep_requested_metrics(metrics)enddefp breakdown_events(_, _, _, [], _), do: []@@ -229,6 +249,19 @@ defmodule Plausible.Stats.Breakdown do|> transform_keys(%{operating_system: :os})end+ defp maybe_add_time_on_page(event_results, site, query, metrics) do+ if :time_on_page in metrics do+ pages = Enum.map(event_results, & &1[:page])+ time_on_page_result = breakdown_time_on_page(site, query, pages)++ Enum.map(event_results, fn row ->+ Map.put(row, :time_on_page, time_on_page_result[row[:page]])+ end)+ else+ event_results+ end+ end+defp breakdown_time_on_page(_site, _query, []) do%{}end@@ -626,6 +659,119 @@ defmodule Plausible.Stats.Breakdown do)end+ defp maybe_add_cr(breakdown_results, site, query, property, metrics, pagination \\ nil) do+ cond do+ :conversion_rate not in metrics -> breakdown_results+ Enum.empty?(breakdown_results) -> breakdown_results+ is_nil(property) -> add_absolute_cr(breakdown_results, site, query)+ true -> add_cr(breakdown_results, site, query, property, metrics, pagination)+ end+ end++ # This function injects a conversion_rate metric into every+ # breakdown result map. It is calculated as X / Y, where:+ #+ # * X is the number of conversions for a breakdown+ # result (conversion = number of visitors who+ # completed the filtered goal with the filtered+ # custom properties).+ #+ # * Y is the number of all visitors for this breakdown+ # result without the `event:goal` and `event:props:*`+ # filters.+ defp add_cr(breakdown_results, site, query, property, metrics, pagination) do+ property_atom = Plausible.Stats.Filters.without_prefix(property)++ items =+ Enum.map(breakdown_results, fn item -> Map.fetch!(item, property_atom) end)++ query_without_goal =+ query+ |> Query.put_filter(property, {:member, items})+ |> Query.remove_event_filters([:goal, :props])++ # Here, we're always only interested in the first page of results+ # - the :member filter makes sure that the results always match with+ # the items in the given breakdown_results list+ page = 1++ # For browser_versions we need to fetch a lot more entries than the+ # pagination limit. This is because many browsers can correspond to+ # a single version number and we need to make sure that the results+ # without goal filter will include all combinations of browsers and+ # their versions that were present in the `breakdown_results` array.+ {pagination_limit, find_match_fn} =+ if property_atom == :browser_version do+ pagination_limit = min(elem(pagination, 0) * 10, 3_000)++ find_match_fn = fn total, conversion ->+ total[:browser_version] == conversion[:browser_version] &&+ total[:browser] == conversion[:browser]+ end++ {pagination_limit, find_match_fn}+ else+ {elem(pagination, 0),+ fn total, conversion ->+ total[property_atom] == conversion[property_atom]+ end}+ end++ pagination = {pagination_limit, page}++ res_without_goal = breakdown(site, query_without_goal, property, [:visitors], pagination)++ Enum.map(breakdown_results, fn item ->+ without_goal = Enum.find(res_without_goal, &find_match_fn.(&1, item))++ {conversion_rate, total_visitors} =+ if without_goal do+ {Util.calculate_cr(without_goal.visitors, item.visitors), without_goal.visitors}+ else+ Sentry.capture_message(+ "Unable to find a conversion_rate divisor from a breakdown response",+ extra: %{+ domain: site.domain,+ query: inspect(query),+ property: property,+ pagination: inspect(pagination),+ item_not_found: inspect(item)+ }+ )++ {"N/A", "N/A"}+ end++ if :total_visitors in metrics do+ item+ |> Map.put(:conversion_rate, conversion_rate)+ |> Map.put(:total_visitors, total_visitors)+ else+ Map.put(item, :conversion_rate, conversion_rate)+ end+ end)+ end++ # Similar to `add_cr/5`, injects a conversion_rate metric into+ # every breakdown result. However, a single divisor is used in+ # the CR calculation across all breakdown results. That is the+ # number of visitors without `event:goal` and `event:props:*`+ # filters.+ #+ # This is useful when we're only interested in the conversions+ # themselves - not how well a certain property such as browser+ # or page converted.+ defp add_absolute_cr(breakdown_results, site, query) do+ total_q = Query.remove_event_filters(query, [:goal, :props])++ %{visitors: %{value: total_visitors}} = Plausible.Stats.aggregate(site, total_q, [:visitors])++ breakdown_results+ |> Enum.map(fn goal ->+ Map.put(goal, :conversion_rate, Util.calculate_cr(total_visitors, goal[:visitors]))+ end)+ end+defp sorting_key(metrics) doif Enum.member?(metrics, :visitors), do: :visitors, else: List.first(metrics)enddiff --git a/lib/plausible/stats/compare.ex b/lib/plausible/stats/compare.exindex 2424118b2b..d234d9aad2 100644--- a/lib/plausible/stats/compare.ex+++ b/lib/plausible/stats/compare.ex@@ -1,4 +1,8 @@defmodule Plausible.Stats.Compare do+ def calculate_change(:conversion_rate, old_value, new_value) do+ Float.round(new_value - old_value, 1)+ end+def calculate_change(:bounce_rate, old_count, new_count) doif old_count > 0, do: new_count - old_countenddiff --git a/lib/plausible/stats/filters/filters.ex b/lib/plausible/stats/filters/filters.exindex 19afaacf48..aeeeaeaa5c 100644--- a/lib/plausible/stats/filters/filters.ex+++ b/lib/plausible/stats/filters/filters.ex@@ -6,34 +6,30 @@ defmodule Plausible.Stats.Filters doalias Plausible.Stats.Filters.{DashboardFilterParser, StatsAPIFilterParser}@visit_props [- "source",- "referrer",- "utm_medium",- "utm_source",- "utm_campaign",- "utm_content",- "utm_term",- "screen",- "device",- "browser",- "browser_version",- "os",- "os_version",- "country",- "region",- "city",- "entry_page",- "exit_page"+ :source,+ :referrer,+ :utm_medium,+ :utm_source,+ :utm_campaign,+ :utm_content,+ :utm_term,+ :screen,+ :device,+ :browser,+ :browser_version,+ :os,+ :os_version,+ :country,+ :region,+ :city,+ :entry_page,+ :exit_page]- def visit_props(), do: @visit_props+ def visit_props(), do: @visit_props |> Enum.map(&to_string/1)- @event_props [- "name",- "page",- "goal"- ]+ @event_props [:name, :page, :goal]- def event_props(), do: @event_props+ def event_props(), do: @event_props |> Enum.map(&to_string/1)@doc """Parses different filter formats.@@ -67,4 +63,11 @@ defmodule Plausible.Stats.Filters dodef parse(filters) when is_map(filters), do: DashboardFilterParser.parse_and_prefix(filters)def parse(_), do: %{}++ def without_prefix(property) do+ property+ |> String.split(":")+ |> List.last()+ |> String.to_existing_atom()+ endenddiff --git a/lib/plausible/stats/timeseries.ex b/lib/plausible/stats/timeseries.exindex d87d354104..4c31aabd03 100644--- a/lib/plausible/stats/timeseries.ex+++ b/lib/plausible/stats/timeseries.ex@@ -1,8 +1,8 @@defmodule Plausible.Stats.Timeseries douse Plausible.ClickhouseRepouse Plausible- alias Plausible.Stats.Query- import Plausible.Stats.{Base, Util}+ alias Plausible.Stats.{Query, Util}+ import Plausible.Stats.{Base}use Plausible.Stats.Fragments@typep metric ::@@ -69,7 +69,7 @@ defmodule Plausible.Stats.Timeseries do|> select_session_metrics(metrics, query)|> Plausible.Stats.Imported.merge_imported_timeseries(site, query, metrics)|> ClickhouseRepo.all()- |> remove_internal_visits_metric(metrics)+ |> Util.keep_requested_metrics(metrics)enddefp buckets(%Query{interval: "month"} = query) dodiff --git a/lib/plausible/stats/util.ex b/lib/plausible/stats/util.exindex 0465506608..5c4c0de47a 100644--- a/lib/plausible/stats/util.ex+++ b/lib/plausible/stats/util.ex@@ -3,21 +3,59 @@ defmodule Plausible.Stats.Util doUtilities for modifying stat results"""+ @manually_removable_metrics [:__internal_visits, :visitors]+@doc """- `__internal_visits` is fetched when querying bounce rate and visit duration, as it- is needed to calculate these from imported data. This function removes that metric- from all entries in the results list.+ Sometimes we need to manually add metrics in order to calculate the value for+ other metrics. E.g:++ * `__internal_visits` is fetched when querying bounce rate and visit duration,+ as it is needed to calculate these from imported data.++ * `visitors` metric might be added manually via `maybe_add_visitors_metric/1`,+ in order to be able to calculate conversion rate.++ This function can be used for stripping those metrics from a breakdown (list),+ or an aggregate (map) result. We do not want to return metrics that we're not+ requested."""- def remove_internal_visits_metric(results, metrics) when is_list(results) do- if :bounce_rate in metrics or :visit_duration in metrics do- results- |> Enum.map(&remove_internal_visits_metric/1)+ def keep_requested_metrics(results, requested_metrics) when is_list(results) do+ Enum.map(results, fn results_map ->+ keep_requested_metrics(results_map, requested_metrics)+ end)+ end++ def keep_requested_metrics(results, requested_metrics) do+ Map.drop(results, @manually_removable_metrics -- requested_metrics)+ end++ @doc """+ This function adds the `visitors` metric into the list of+ given metrics if it's not already there and if there is a+ `conversion_rate` metric in the list.++ Currently, the conversion rate cannot be queried from the+ database with a simple select clause - instead, we need to+ fetch the database result first, and then manually add it+ into the aggregate map or every entry of thebreakdown list.++ In order for us to be able to calculate it based on the+ results returned by the database query, the visitors metric+ needs to be queried.+ """+ def maybe_add_visitors_metric(metrics) do+ if :conversion_rate in metrics and :visitors not in metrics do+ metrics ++ [:visitors]else- results+ metricsendend- def remove_internal_visits_metric(result) when is_map(result) do- Map.delete(result, :__internal_visits)+ def calculate_cr(nil, _converted_visitors), do: nil++ def calculate_cr(unique_visitors, converted_visitors) do+ if unique_visitors > 0,+ do: Float.round(converted_visitors / unique_visitors * 100, 1),+ else: 0.0endenddiff --git a/lib/plausible_web/controllers/api/external_stats_controller.ex b/lib/plausible_web/controllers/api/external_stats_controller.exindex a5e792bc26..c9cc9af837 100644--- a/lib/plausible_web/controllers/api/external_stats_controller.ex+++ b/lib/plausible_web/controllers/api/external_stats_controller.ex@@ -2,7 +2,20 @@ defmodule PlausibleWeb.Api.ExternalStatsController douse PlausibleWeb, :controlleruse Plausible.Repouse PlausibleWeb.Plugs.ErrorHandler- alias Plausible.Stats.Query+ alias Plausible.Stats.{Query, Compare, Comparisons}++ @metrics [+ :visitors,+ :visits,+ :pageviews,+ :views_per_visit,+ :bounce_rate,+ :visit_duration,+ :events,+ :conversion_rate+ ]++ @metric_mappings Enum.into(@metrics, %{}, fn metric -> {to_string(metric), metric} end)def realtime_visitors(conn, _params) dosite = conn.assigns.site@@ -21,7 +34,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController do:ok <- ensure_custom_props_access(site, query) doresults =if params["compare"] == "previous_period" do- {:ok, prev_query} = Plausible.Stats.Comparisons.compare(site, query, "previous_period")+ {:ok, prev_query} = Comparisons.compare(site, query, "previous_period")[prev_result, curr_result] =Plausible.ClickhouseRepo.parallel_tasks([@@ -31,12 +44,9 @@ defmodule PlausibleWeb.Api.ExternalStatsController doEnum.map(curr_result, fn {metric, %{value: current_val}} ->%{value: prev_val} = prev_result[metric]+ change = Compare.calculate_change(metric, prev_val, current_val)- {metric,- %{- value: current_val,- change: percent_change(prev_val, current_val)- }}+ {metric, %{value: current_val, change: change}}end)|> Enum.into(%{})else@@ -96,13 +106,6 @@ defmodule PlausibleWeb.Api.ExternalStatsController do@default_breakdown_limit 100defp validate_or_default_limit(_), do: {:ok, @default_breakdown_limit}- defp event_only_property?("event:name"), do: true- defp event_only_property?("event:goal"), do: true- defp event_only_property?("event:props:" <> _), do: true- defp event_only_property?(_), do: false-- @event_metrics ["visitors", "pageviews", "events"]- @session_metrics ["visits", "bounce_rate", "visit_duration", "views_per_visit"]defp parse_and_validate_metrics(params, property, query) dometrics =Map.get(params, "metrics", "visitors")@@ -113,7 +116,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController do{:error, reason}metrics ->- {:ok, Enum.map(metrics, &String.to_existing_atom/1)}+ {:ok, Enum.map(metrics, &Map.fetch!(@metric_mappings, &1))}endend@@ -155,26 +158,61 @@ defmodule PlausibleWeb.Api.ExternalStatsController doend)end- defp validate_metric("events", nil, %{include_imported: true}) do- {:error, "Metric `events` cannot be queried with imported data"}+ defp validate_metric("conversion_rate" = metric, property, query) do+ cond do+ property == "event:goal" ->+ {:ok, metric}++ query.filters["event:goal"] ->+ {:ok, metric}++ true ->+ {:error,+ "Metric `#{metric}` can only be queried in a goal breakdown or with a goal filter"}+ endend- defp validate_metric(metric, _, _) when metric in @event_metrics, do: {:ok, metric}+ defp validate_metric("events" = metric, _, query) do+ if query.include_imported do+ {:error, "Metric `#{metric}` cannot be queried with imported data"}+ else+ {:ok, metric}+ end+ end- defp validate_metric(metric, property, query) when metric in @session_metrics do- event_only_filter = Map.keys(query.filters) |> Enum.find(&event_only_property?/1)+ defp validate_metric(metric, _, _) when metric in ["visitors", "pageviews"] do+ {:ok, metric}+ end+ defp validate_metric("views_per_visit" = metric, property, query) docond do- metric == "views_per_visit" && query.filters["event:page"] ->+ query.filters["event:page"] ->{:error, "Metric `#{metric}` cannot be queried with a filter on `event:page`."}- metric == "views_per_visit" && property != nil ->+ property != nil ->{:error, "Metric `#{metric}` is not supported in breakdown queries."}+ true ->+ validate_session_metric(metric, property, query)+ end+ end++ defp validate_metric(metric, property, query)+ when metric in ["visits", "bounce_rate", "visit_duration"] do+ validate_session_metric(metric, property, query)+ end++ defp validate_metric(metric, _, _) do+ {:error,+ "The metric `#{metric}` is not recognized. Find valid metrics from the documentation: https://plausible.io/docs/stats-api#metrics"}+ end++ defp validate_session_metric(metric, property, query) do+ cond doevent_only_property?(property) ->{:error, "Session metric `#{metric}` cannot be queried for breakdown by `#{property}`."}- event_only_filter ->+ event_only_filter = find_event_only_filter(query) ->{:error,"Session metric `#{metric}` cannot be queried when using a filter on `#{event_only_filter}`."}@@ -183,11 +221,15 @@ defmodule PlausibleWeb.Api.ExternalStatsController doendend- defp validate_metric(metric, _, _) do- {:error,- "The metric `#{metric}` is not recognized. Find valid metrics from the documentation: https://plausible.io/docs/stats-api#metrics"}+ defp find_event_only_filter(query) do+ Map.keys(query.filters) |> Enum.find(&event_only_property?/1)end+ defp event_only_property?("event:name"), do: true+ defp event_only_property?("event:goal"), do: true+ defp event_only_property?("event:props:" <> _), do: true+ defp event_only_property?(_), do: false+def timeseries(conn, params) dosite = Repo.preload(conn.assigns.site, :owner)@@ -206,19 +248,6 @@ defmodule PlausibleWeb.Api.ExternalStatsController doendend- defp percent_change(old_count, new_count) do- cond do- old_count == 0 and new_count > 0 ->- 100-- old_count == 0 and new_count == 0 ->- 0-- true ->- round((new_count - old_count) / old_count * 100)- end- end-defp validate_date(%{"period" => "custom"} = params) dowith {:ok, date} <- Map.fetch(params, "date"),[from, to] <- String.split(date, ","),diff --git a/lib/plausible_web/controllers/api/stats_controller.ex b/lib/plausible_web/controllers/api/stats_controller.exindex 97f9692efb..c3c0a6168c 100644--- a/lib/plausible_web/controllers/api/stats_controller.ex+++ b/lib/plausible_web/controllers/api/stats_controller.ex@@ -346,61 +346,23 @@ defmodule PlausibleWeb.Api.StatsController doenddefp fetch_top_stats(site, %Query{filters: %{"event:goal" => _}} = query, comparison_query) do- query_without_filters = Query.remove_event_filters(query, [:goal, :props])- metrics = [:visitors, :events] ++ @revenue_metrics-- results_without_filters =- site- |> Stats.aggregate(query_without_filters, [:visitors])- |> transform_keys(%{visitors: :unique_visitors})-- results =- site- |> Stats.aggregate(query, metrics)- |> transform_keys(%{visitors: :converted_visitors, events: :completions})- |> Map.merge(results_without_filters)-- comparison =- if comparison_query do- comparison_query_without_filters =- Query.remove_event_filters(comparison_query, [:goal, :props])-- comparison_without_filters =- site- |> Stats.aggregate(comparison_query_without_filters, [:visitors])- |> transform_keys(%{visitors: :unique_visitors})-- site- |> Stats.aggregate(comparison_query, metrics)- |> transform_keys(%{visitors: :converted_visitors, events: :completions})- |> Map.merge(comparison_without_filters)- end-- conversion_rate = %{- cr: %{value: calculate_cr(results.unique_visitors.value, results.converted_visitors.value)}- }-- comparison_conversion_rate =- if comparison do- value =- calculate_cr(comparison.unique_visitors.value, comparison.converted_visitors.value)+ metrics =+ [:total_visitors, :visitors, :events, :conversion_rate] ++ @revenue_metrics- %{cr: %{value: value}}- else- nil- end+ results = Stats.aggregate(site, query, metrics)+ comparison = if comparison_query, do: Stats.aggregate(site, comparison_query, metrics)[- top_stats_entry(results, comparison, "Unique visitors", :unique_visitors),- top_stats_entry(results, comparison, "Unique conversions", :converted_visitors),- top_stats_entry(results, comparison, "Total conversions", :completions),+ top_stats_entry(results, comparison, "Unique visitors", :total_visitors),+ top_stats_entry(results, comparison, "Unique conversions", :visitors),+ top_stats_entry(results, comparison, "Total conversions", :events),on_full_build dotop_stats_entry(results, comparison, "Average revenue", :average_revenue, &format_money/1)end,on_full_build dotop_stats_entry(results, comparison, "Total revenue", :total_revenue, &format_money/1)end,- top_stats_entry(conversion_rate, comparison_conversion_rate, "Conversion rate", :cr)+ top_stats_entry(results, comparison, "Conversion rate", :conversion_rate)]|> Enum.reject(&is_nil/1)|> then(&{&1, 100})@@ -469,17 +431,16 @@ defmodule PlausibleWeb.Api.StatsController dodef sources(conn, params) dosite = conn.assigns[:site]-query = Query.from(site, params)-pagination = parse_pagination(params)- metrics =- if params["detailed"], do: [:visitors, :bounce_rate, :visit_duration], else: [:visitors]+ extra_metrics =+ if params["detailed"], do: [:bounce_rate, :visit_duration], else: []++ metrics = breakdown_metrics(query, extra_metrics)res =Stats.breakdown(site, query, "visit:source", metrics, pagination)- |> add_cr(site, query, pagination, :source, "visit:source")|> transform_keys(%{source: :name})if params["csv"] do@@ -552,16 +513,12 @@ defmodule PlausibleWeb.Api.StatsController dodef utm_mediums(conn, params) dosite = conn.assigns[:site]-query = Query.from(site, params)-pagination = parse_pagination(params)-- metrics = [:visitors, :bounce_rate, :visit_duration]+ metrics = breakdown_metrics(query, [:bounce_rate, :visit_duration])res =Stats.breakdown(site, query, "visit:utm_medium", metrics, pagination)- |> add_cr(site, query, pagination, :utm_medium, "visit:utm_medium")|> transform_keys(%{utm_medium: :name})if params["csv"] do@@ -579,16 +536,12 @@ defmodule PlausibleWeb.Api.StatsController dodef utm_campaigns(conn, params) dosite = conn.assigns[:site]-query = Query.from(site, params)-pagination = parse_pagination(params)-- metrics = [:visitors, :bounce_rate, :visit_duration]+ metrics = breakdown_metrics(query, [:bounce_rate, :visit_duration])res =Stats.breakdown(site, query, "visit:utm_campaign", metrics, pagination)- |> add_cr(site, query, pagination, :utm_campaign, "visit:utm_campaign")|> transform_keys(%{utm_campaign: :name})if params["csv"] do@@ -606,15 +559,12 @@ defmodule PlausibleWeb.Api.StatsController dodef utm_contents(conn, params) dosite = conn.assigns[:site]-query = Query.from(site, params)-pagination = parse_pagination(params)- metrics = [:visitors, :bounce_rate, :visit_duration]+ metrics = breakdown_metrics(query, [:bounce_rate, :visit_duration])res =Stats.breakdown(site, query, "visit:utm_content", metrics, pagination)- |> add_cr(site, query, pagination, :utm_content, "visit:utm_content")|> transform_keys(%{utm_content: :name})if params["csv"] do@@ -632,15 +582,12 @@ defmodule PlausibleWeb.Api.StatsController dodef utm_terms(conn, params) dosite = conn.assigns[:site]-query = Query.from(site, params)-pagination = parse_pagination(params)- metrics = [:visitors, :bounce_rate, :visit_duration]+ metrics = breakdown_metrics(query, [:bounce_rate, :visit_duration])res =Stats.breakdown(site, query, "visit:utm_term", metrics, pagination)- |> add_cr(site, query, pagination, :utm_term, "visit:utm_term")|> transform_keys(%{utm_term: :name})if params["csv"] do@@ -658,16 +605,12 @@ defmodule PlausibleWeb.Api.StatsController dodef utm_sources(conn, params) dosite = conn.assigns[:site]-query = Query.from(site, params)-pagination = parse_pagination(params)-- metrics = [:visitors, :bounce_rate, :visit_duration]+ metrics = breakdown_metrics(query, [:bounce_rate, :visit_duration])res =Stats.breakdown(site, query, "visit:utm_source", metrics, pagination)- |> add_cr(site, query, pagination, :utm_source, "visit:utm_source")|> transform_keys(%{utm_source: :name})if params["csv"] do@@ -685,16 +628,12 @@ defmodule PlausibleWeb.Api.StatsController dodef referrers(conn, params) dosite = conn.assigns[:site]-query = Query.from(site, params)-pagination = parse_pagination(params)-- metrics = [:visitors, :bounce_rate, :visit_duration]+ metrics = breakdown_metrics(query, [:bounce_rate, :visit_duration])res =Stats.breakdown(site, query, "visit:referrer", metrics, pagination)- |> add_cr(site, query, pagination, :referrer, "visit:referrer")|> transform_keys(%{referrer: :name})if params["csv"] do@@ -754,12 +693,13 @@ defmodule PlausibleWeb.Api.StatsController dopagination = parse_pagination(params)- metrics =- if params["detailed"], do: [:visitors, :bounce_rate, :visit_duration], else: [:visitors]+ extra_metrics =+ if params["detailed"], do: [:bounce_rate, :visit_duration], else: []++ metrics = breakdown_metrics(query, extra_metrics)referrers =Stats.breakdown(site, query, "visit:referrer", metrics, pagination)- |> add_cr(site, query, pagination, :referrer, "visit:referrer")|> transform_keys(%{referrer: :name})json(conn, referrers)@@ -769,16 +709,16 @@ defmodule PlausibleWeb.Api.StatsController dosite = conn.assigns[:site]query = Query.from(site, params)- metrics =+ extra_metrics =if params["detailed"],- do: [:visitors, :pageviews, :bounce_rate, :time_on_page],- else: [:visitors]+ do: [:pageviews, :bounce_rate, :time_on_page],+ else: []+ metrics = breakdown_metrics(query, extra_metrics)pagination = parse_pagination(params)pages =Stats.breakdown(site, query, "event:page", metrics, pagination)- |> add_cr(site, query, pagination, :page, "event:page")|> transform_keys(%{page: :name})if params["csv"] do@@ -798,11 +738,10 @@ defmodule PlausibleWeb.Api.StatsController dosite = conn.assigns[:site]query = Query.from(site, params)pagination = parse_pagination(params)- metrics = [:visitors, :visits, :visit_duration]+ metrics = breakdown_metrics(query, [:visits, :visit_duration])entry_pages =Stats.breakdown(site, query, "visit:entry_page", metrics, pagination)- |> add_cr(site, query, pagination, :entry_page, "visit:entry_page")|> transform_keys(%{entry_page: :name})if params["csv"] do@@ -829,11 +768,10 @@ defmodule PlausibleWeb.Api.StatsController dosite = conn.assigns[:site]query = Query.from(site, params){limit, page} = parse_pagination(params)- metrics = [:visitors, :visits]+ metrics = breakdown_metrics(query, [:visits])exit_pages =Stats.breakdown(site, query, "visit:exit_page", metrics, {limit, page})- |> add_cr(site, query, {limit, page}, :exit_page, "visit:exit_page")|> add_exit_rate(site, query, limit)|> transform_keys(%{exit_page: :name})@@ -889,10 +827,10 @@ defmodule PlausibleWeb.Api.StatsController dosite = conn.assigns[:site]query = site |> Query.from(params)pagination = parse_pagination(params)+ metrics = breakdown_metrics(query)countries =- Stats.breakdown(site, query, "visit:country", [:visitors], pagination)- |> add_cr(site, query, {300, 1}, :country, "visit:country")+ Stats.breakdown(site, query, "visit:country", metrics, pagination)|> transform_keys(%{country: :code})|> add_percentages(site, query)@@ -941,9 +879,10 @@ defmodule PlausibleWeb.Api.StatsController dosite = conn.assigns[:site]query = site |> Query.from(params)pagination = parse_pagination(params)+ metrics = breakdown_metrics(query)regions =- Stats.breakdown(site, query, "visit:region", [:visitors], pagination)+ Stats.breakdown(site, query, "visit:region", metrics, pagination)|> transform_keys(%{region: :code})|> Enum.map(fn region ->region_entry = Location.get_subdivision(region[:code])@@ -974,9 +913,10 @@ defmodule PlausibleWeb.Api.StatsController dosite = conn.assigns[:site]query = site |> Query.from(params)pagination = parse_pagination(params)+ metrics = breakdown_metrics(query)cities =- Stats.breakdown(site, query, "visit:city", [:visitors], pagination)+ Stats.breakdown(site, query, "visit:city", metrics, pagination)|> transform_keys(%{city: :code})|> Enum.map(fn city ->city_info = Location.get_city(city[:code])@@ -1012,10 +952,10 @@ defmodule PlausibleWeb.Api.StatsController dosite = conn.assigns[:site]query = Query.from(site, params)pagination = parse_pagination(params)+ metrics = breakdown_metrics(query)browsers =- Stats.breakdown(site, query, "visit:browser", [:visitors], pagination)- |> add_cr(site, query, pagination, :browser, "visit:browser")+ Stats.breakdown(site, query, "visit:browser", metrics, pagination)|> transform_keys(%{browser: :name})|> add_percentages(site, query)@@ -1036,10 +976,10 @@ defmodule PlausibleWeb.Api.StatsController dosite = conn.assigns[:site]query = Query.from(site, params)pagination = parse_pagination(params)+ metrics = breakdown_metrics(query)versions =- Stats.breakdown(site, query, "visit:browser_version", [:visitors], pagination)- |> add_cr(site, query, pagination, :browser_version, "visit:browser_version")+ Stats.breakdown(site, query, "visit:browser_version", metrics, pagination)|> transform_keys(%{browser_version: :name})|> add_percentages(site, query)@@ -1066,10 +1006,10 @@ defmodule PlausibleWeb.Api.StatsController dosite = conn.assigns[:site]query = Query.from(site, params)pagination = parse_pagination(params)+ metrics = breakdown_metrics(query)systems =- Stats.breakdown(site, query, "visit:os", [:visitors], pagination)- |> add_cr(site, query, pagination, :os, "visit:os")+ Stats.breakdown(site, query, "visit:os", metrics, pagination)|> transform_keys(%{os: :name})|> add_percentages(site, query)@@ -1090,10 +1030,10 @@ defmodule PlausibleWeb.Api.StatsController dosite = conn.assigns[:site]query = Query.from(site, params)pagination = parse_pagination(params)+ metrics = breakdown_metrics(query)versions =- Stats.breakdown(site, query, "visit:os_version", [:visitors], pagination)- |> add_cr(site, query, pagination, :os_version, "visit:os_version")+ Stats.breakdown(site, query, "visit:os_version", metrics, pagination)|> transform_keys(%{os_version: :name})|> add_percentages(site, query)@@ -1104,10 +1044,10 @@ defmodule PlausibleWeb.Api.StatsController dosite = conn.assigns[:site]query = Query.from(site, params)pagination = parse_pagination(params)+ metrics = breakdown_metrics(query)sizes =- Stats.breakdown(site, query, "visit:device", [:visitors], pagination)- |> add_cr(site, query, pagination, :device, "visit:device")+ Stats.breakdown(site, query, "visit:device", metrics, pagination)|> transform_keys(%{device: :name})|> add_percentages(site, query)@@ -1124,14 +1064,6 @@ defmodule PlausibleWeb.Api.StatsController doendend- defp calculate_cr(nil, _converted_visitors), do: nil-- defp calculate_cr(unique_visitors, converted_visitors) do- if unique_visitors > 0,- do: Float.round(converted_visitors / unique_visitors * 100, 1),- else: 0.0- end-def conversions(conn, params) dopagination = parse_pagination(params)site = Plausible.Repo.preload(conn.assigns.site, :goals)@@ -1144,21 +1076,7 @@ defmodule PlausibleWeb.Api.StatsController doqueryend- total_q = Query.remove_event_filters(query, [:goal, :props])-- %{visitors: %{value: total_visitors}} = Stats.aggregate(site, total_q, [:visitors])-- metrics =- on_full_build do- if Enum.any?(site.goals, &Plausible.Goal.Revenue.revenue?/1) and- Plausible.Billing.Feature.RevenueGoals.enabled?(site) do- [:visitors, :events] ++ @revenue_metrics- else- [:visitors, :events]- end- else- [:visitors, :events]- end+ metrics = [:visitors, :events, :conversion_rate] ++ @revenue_metricsconversions =site@@ -1166,7 +1084,6 @@ defmodule PlausibleWeb.Api.StatsController do|> transform_keys(%{goal: :name})|> Enum.map(fn goal ->goal- |> Map.put(:conversion_rate, calculate_cr(total_visitors, goal[:visitors]))|> Enum.map(&format_revenue_metric/1)|> Map.new()end)@@ -1240,32 +1157,19 @@ defmodule PlausibleWeb.Api.StatsController do|> Map.put(:include_imported, false)metrics =- if full_build?() and Map.has_key?(query.filters, "event:goal") do- [:visitors, :events] ++ @revenue_metrics+ if query.filters["event:goal"] do+ [:visitors, :events, :conversion_rate] ++ @revenue_metricselse- [:visitors, :events]+ [:visitors, :events] ++ @revenue_metricsend- props =- Stats.breakdown(site, query, prefixed_prop, metrics, pagination)- |> transform_keys(%{prop_key => :name})- |> Enum.map(fn entry ->- Enum.map(entry, &format_revenue_metric/1)- |> Map.new()- end)- |> add_percentages(site, query)-- if Map.has_key?(query.filters, "event:goal") do- total_q = Query.remove_event_filters(query, [:goal, :props])-- %{visitors: %{value: total_unique_visitors}} = Stats.aggregate(site, total_q, [:visitors])-- Enum.map(props, fn prop ->- Map.put(prop, :conversion_rate, calculate_cr(total_unique_visitors, prop.visitors))- end)- else- props- end+ Stats.breakdown(site, query, prefixed_prop, metrics, pagination)+ |> transform_keys(%{prop_key => :name})+ |> Enum.map(fn entry ->+ Enum.map(entry, &format_revenue_metric/1)+ |> Map.new()+ end)+ |> add_percentages(site, query)enddef current_visitors(conn, _) do@@ -1324,37 +1228,6 @@ defmodule PlausibleWeb.Api.StatsController dodefp add_percentages(breakdown_result, _, _), do: breakdown_result- defp add_cr([_ | _] = breakdown_results, site, query, pagination, key_name, filter_name)- when is_map_key(query.filters, "event:goal") do- items = Enum.map(breakdown_results, fn item -> Map.fetch!(item, key_name) end)-- query_without_goal =- query- |> Query.put_filter(filter_name, {:member, items})- |> Query.remove_event_filters([:goal, :props])-- # Here, we're always only interested in the first page of results- # - the :member filter makes sure that the results always match with- # the items in the given breakdown_results list- pagination = {elem(pagination, 0), 1}-- res_without_goal =- Stats.breakdown(site, query_without_goal, filter_name, [:visitors], pagination)-- Enum.map(breakdown_results, fn item ->- without_goal =- Enum.find(res_without_goal, fn s ->- Map.fetch!(s, key_name) == Map.fetch!(item, key_name)- end)-- item- |> Map.put(:total_visitors, without_goal.visitors)- |> Map.put(:conversion_rate, calculate_cr(without_goal.visitors, item.visitors))- end)- end-- defp add_cr(breakdown_results, _, _, _, _, _), do: breakdown_results-defp to_csv(list, columns), do: to_csv(list, columns, columns)defp to_csv(list, columns, column_names) do@@ -1485,4 +1358,12 @@ defmodule PlausibleWeb.Api.StatsController do{metric, value}endend++ defp breakdown_metrics(query, extra_metrics \\ []) do+ if query.filters["event:goal"] do+ [:visitors, :conversion_rate, :total_visitors]+ else+ [:visitors] ++ extra_metrics+ end+ endenddiff --git a/test/plausible_web/controllers/api/external_stats_controller/aggregate_test.exs b/test/plausible_web/controllers/api/external_stats_controller/aggregate_test.exsindex b7e84b7a58..40e6f7a264 100644--- a/test/plausible_web/controllers/api/external_stats_controller/aggregate_test.exs+++ b/test/plausible_web/controllers/api/external_stats_controller/aggregate_test.exs@@ -140,6 +140,22 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest doendend+ test "validates that conversion_rate cannot be queried without a goal filter", %{+ conn: conn,+ site: site+ } do+ conn =+ get(conn, "/api/v1/stats/aggregate", %{+ "site_id" => site.domain,+ "metrics" => "conversion_rate"+ })++ assert %{"error" => msg} = json_response(conn, 400)++ assert msg =~+ "Metric `conversion_rate` can only be queried in a goal breakdown or with a goal filter"+ end+test "validates that views_per_visit cannot be used with event:page filter", %{conn: conn,site: site@@ -156,6 +172,23 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest do"Metric `views_per_visit` cannot be queried with a filter on `event:page`."}end++ test "validates that views_per_visit cannot be used with an event only filter", %{+ conn: conn,+ site: site+ } do+ conn =+ get(conn, "/api/v1/stats/aggregate", %{+ "site_id" => site.domain,+ "filters" => "event:name==Something",+ "metrics" => "views_per_visit"+ })++ assert json_response(conn, 400) == %{+ "error" =>+ "Session metric `views_per_visit` cannot be queried when using a filter on `event:name`."+ }+ endendtest "aggregates a single metric", %{conn: conn, site: site} do@@ -289,10 +322,39 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest doassert json_response(conn, 200)["results"] == %{"pageviews" => %{"value" => 4, "change" => 100},"visitors" => %{"value" => 3, "change" => 100},- "bounce_rate" => %{"value" => 100, "change" => 100},+ "bounce_rate" => %{"value" => 100, "change" => nil},"visit_duration" => %{"value" => 0, "change" => 0}}end++ test "can compare conversion_rate with previous period", %{conn: conn, site: site} do+ today = ~N[2023-05-05 12:00:00]+ yesterday = Timex.shift(today, days: -1)++ populate_stats(site, [+ build(:event, name: "Signup", timestamp: yesterday),+ build(:pageview, timestamp: yesterday),+ build(:pageview, timestamp: yesterday),+ build(:event, name: "Signup", timestamp: today),+ build(:pageview, timestamp: today)+ ])++ insert(:goal, %{site: site, event_name: "Signup"})++ conn =+ get(conn, "/api/v1/stats/aggregate", %{+ "site_id" => site.domain,+ "period" => "day",+ "date" => "2023-05-05",+ "metrics" => "conversion_rate",+ "filters" => "event:goal==Signup",+ "compare" => "previous_period"+ })++ assert json_response(conn, 200)["results"] == %{+ "conversion_rate" => %{"value" => 50.0, "change" => 16.7}+ }+ endenddescribe "with imported data" do@@ -357,7 +419,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest do"visitors" => %{"value" => 2, "change" => 100},"visits" => %{"value" => 5, "change" => 150},"pageviews" => %{"value" => 9, "change" => -10},- "bounce_rate" => %{"value" => 40, "change" => -20},+ "bounce_rate" => %{"value" => 40, "change" => -10},"views_per_visit" => %{"value" => 1.0, "change" => 100},"visit_duration" => %{"value" => 20, "change" => -80}}@@ -1234,4 +1296,111 @@ defmodule PlausibleWeb.Api.ExternalStatsController.AggregateTest doassert json_response(conn, 200)["results"] == %{"pageviews" => %{"value" => 3}}endend++ describe "metrics" do+ test "conversion_rate when goal filter is applied", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:event, name: "Signup"),+ build(:pageview)+ ])++ insert(:goal, %{site: site, event_name: "Signup"})++ conn =+ get(conn, "/api/v1/stats/aggregate", %{+ "site_id" => site.domain,+ "metrics" => "conversion_rate",+ "filters" => "event:goal==Signup"+ })++ assert json_response(conn, 200)["results"] == %{"conversion_rate" => %{"value" => 50}}+ end++ test "conversion_rate when goal + custom prop filter applied", %{+ conn: conn,+ site: site+ } do+ populate_stats(site, [+ build(:event, name: "Signup"),+ build(:event, name: "Signup", "meta.key": ["author"], "meta.value": ["Uku"]),+ build(:event, name: "Signup", "meta.key": ["author"], "meta.value": ["Marko"]),+ build(:pageview)+ ])++ insert(:goal, %{site: site, event_name: "Signup"})++ conn =+ get(conn, "/api/v1/stats/aggregate", %{+ "site_id" => site.domain,+ "metrics" => "conversion_rate,visitors,events",+ "filters" => "event:goal==Signup;event:props:author==Uku"+ })++ assert %{+ "conversion_rate" => %{"value" => 25.0},+ "visitors" => %{"value" => 1},+ "events" => %{"value" => 1}+ } = json_response(conn, 200)["results"]+ end++ test "conversion_rate when goal + visit property filter applied", %{+ conn: conn,+ site: site+ } do+ populate_stats(site, [+ build(:event, name: "Signup"),+ build(:event, name: "Signup", browser: "Chrome"),+ build(:event, name: "Signup", browser: "Firefox", user_id: 123),+ build(:event, name: "Signup", browser: "Firefox", user_id: 123),+ build(:pageview, browser: "Firefox"),+ build(:pageview)+ ])++ insert(:goal, %{site: site, event_name: "Signup"})++ conn =+ get(conn, "/api/v1/stats/aggregate", %{+ "site_id" => site.domain,+ "metrics" => "conversion_rate,visitors,events",+ "filters" => "visit:browser==Firefox;event:goal==Signup"+ })++ assert %{+ "conversion_rate" => %{"value" => 50.0},+ "visitors" => %{"value" => 1},+ "events" => %{"value" => 2}+ } =+ json_response(conn, 200)["results"]+ end++ test "conversion_rate when goal + page filter applied", %{+ conn: conn,+ site: site+ } do+ populate_stats(site, [+ build(:event, name: "Signup"),+ build(:event, name: "Signup", pathname: "/not-this"),+ build(:event, name: "Signup", pathname: "/this", user_id: 123),+ build(:event, name: "Signup", pathname: "/this", user_id: 123),+ build(:pageview, pathname: "/this"),+ build(:pageview)+ ])++ insert(:goal, %{site: site, event_name: "Signup"})++ conn =+ get(conn, "/api/v1/stats/aggregate", %{+ "site_id" => site.domain,+ "metrics" => "conversion_rate,visitors,events",+ "filters" => "event:page==/this;event:goal==Signup"+ })++ assert %{+ "conversion_rate" => %{"value" => 50.0},+ "visitors" => %{"value" => 1},+ "events" => %{"value" => 2}+ } =+ json_response(conn, 200)["results"]+ end+ endenddiff --git a/test/plausible_web/controllers/api/external_stats_controller/breakdown_test.exs b/test/plausible_web/controllers/api/external_stats_controller/breakdown_test.exsindex 1d2cfca625..369992f000 100644--- a/test/plausible_web/controllers/api/external_stats_controller/breakdown_test.exs+++ b/test/plausible_web/controllers/api/external_stats_controller/breakdown_test.exs@@ -83,6 +83,23 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest doenddescribe "param validation" do+ test "does not allow querying conversion_rate without a goal filter", %{+ conn: conn,+ site: site+ } do+ conn =+ get(conn, "/api/v1/stats/breakdown", %{+ "site_id" => site.domain,+ "property" => "event:page",+ "metrics" => "conversion_rate"+ })++ assert json_response(conn, 400) == %{+ "error" =>+ "Metric `conversion_rate` can only be queried in a goal breakdown or with a goal filter"+ }+ end+test "validates that property is required", %{conn: conn, site: site} doconn =get(conn, "/api/v1/stats/breakdown", %{@@ -2001,6 +2018,347 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest doenddescribe "metrics" do+ test "returns conversion_rate in an event:goal breakdown", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:event, name: "Signup", user_id: 1),+ build(:event, name: "Signup", user_id: 1),+ build(:pageview, pathname: "/blog"),+ build(:pageview, pathname: "/blog/post"),+ build(:pageview)+ ])++ insert(:goal, %{site: site, event_name: "Signup"})+ insert(:goal, %{site: site, page_path: "/blog**"})++ conn =+ get(conn, "/api/v1/stats/breakdown", %{+ "site_id" => site.domain,+ "period" => "day",+ "property" => "event:goal",+ "metrics" => "visitors,events,conversion_rate"+ })++ assert json_response(conn, 200) == %{+ "results" => [+ %{+ "goal" => "Visit /blog**",+ "visitors" => 2,+ "events" => 2,+ "conversion_rate" => 50+ },+ %{+ "goal" => "Signup",+ "visitors" => 1,+ "events" => 2,+ "conversion_rate" => 25+ }+ ]+ }+ end++ test "returns conversion_rate alone in an event:goal breakdown", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:event, name: "Signup", user_id: 1),+ build(:pageview)+ ])++ insert(:goal, %{site: site, event_name: "Signup"})++ conn =+ get(conn, "/api/v1/stats/breakdown", %{+ "site_id" => site.domain,+ "period" => "day",+ "property" => "event:goal",+ "metrics" => "conversion_rate"+ })++ assert json_response(conn, 200) == %{+ "results" => [+ %{+ "goal" => "Signup",+ "conversion_rate" => 50+ }+ ]+ }+ end++ test "returns conversion_rate in a goal filtered custom prop breakdown", %{+ conn: conn,+ site: site+ } do+ populate_stats(site, [+ build(:pageview, pathname: "/blog/1", "meta.key": ["author"], "meta.value": ["Uku"]),+ build(:pageview, pathname: "/blog/2", "meta.key": ["author"], "meta.value": ["Uku"]),+ build(:pageview, pathname: "/blog/3", "meta.key": ["author"], "meta.value": ["Uku"]),+ build(:pageview, pathname: "/blog/1", "meta.key": ["author"], "meta.value": ["Marko"]),+ build(:pageview,+ pathname: "/blog/2",+ "meta.key": ["author"],+ "meta.value": ["Marko"],+ user_id: 1+ ),+ build(:pageview,+ pathname: "/blog/3",+ "meta.key": ["author"],+ "meta.value": ["Marko"],+ user_id: 1+ ),+ build(:pageview, pathname: "/blog"),+ build(:pageview, "meta.key": ["author"], "meta.value": ["Marko"]),+ build(:pageview)+ ])++ insert(:goal, %{site: site, page_path: "/blog**"})++ conn =+ get(conn, "/api/v1/stats/breakdown", %{+ "site_id" => site.domain,+ "period" => "day",+ "property" => "event:props:author",+ "filters" => "event:goal==Visit /blog**",+ "metrics" => "visitors,events,conversion_rate"+ })++ assert json_response(conn, 200) == %{+ "results" => [+ %{+ "author" => "Uku",+ "visitors" => 3,+ "events" => 3,+ "conversion_rate" => 37.5+ },+ %{+ "author" => "Marko",+ "visitors" => 2,+ "events" => 3,+ "conversion_rate" => 25+ },+ %{+ "author" => "(none)",+ "visitors" => 1,+ "events" => 1,+ "conversion_rate" => 12.5+ }+ ]+ }+ end++ test "returns conversion_rate alone in a goal filtered custom prop breakdown", %{+ conn: conn,+ site: site+ } do+ populate_stats(site, [+ build(:pageview, pathname: "/blog/1", "meta.key": ["author"], "meta.value": ["Uku"]),+ build(:pageview)+ ])++ insert(:goal, %{site: site, page_path: "/blog**"})++ conn =+ get(conn, "/api/v1/stats/breakdown", %{+ "site_id" => site.domain,+ "period" => "day",+ "property" => "event:props:author",+ "filters" => "event:goal==Visit /blog**",+ "metrics" => "conversion_rate"+ })++ assert json_response(conn, 200) == %{+ "results" => [+ %{+ "author" => "Uku",+ "conversion_rate" => 50+ }+ ]+ }+ end++ test "returns conversion_rate in a goal filtered event:page breakdown", %{+ conn: conn,+ site: site+ } do+ populate_stats(site, [+ build(:event, pathname: "/en/register"),+ build(:event, pathname: "/en/register", name: "Signup"),+ build(:event, pathname: "/en/register", name: "Signup"),+ build(:event, pathname: "/it/register", name: "Signup", user_id: 1),+ build(:event, pathname: "/it/register", name: "Signup", user_id: 1),+ build(:event, pathname: "/it/register")+ ])++ insert(:goal, %{site: site, event_name: "Signup"})++ conn =+ get(conn, "/api/v1/stats/breakdown", %{+ "site_id" => site.domain,+ "period" => "day",+ "property" => "event:page",+ "filters" => "event:goal==Signup",+ "metrics" => "visitors,events,conversion_rate"+ })++ assert json_response(conn, 200) == %{+ "results" => [+ %{+ "page" => "/en/register",+ "visitors" => 2,+ "events" => 2,+ "conversion_rate" => 66.7+ },+ %{+ "page" => "/it/register",+ "visitors" => 1,+ "events" => 2,+ "conversion_rate" => 50+ }+ ]+ }+ end++ test "returns conversion_rate alone in a goal filtered event:page breakdown", %{+ conn: conn,+ site: site+ } do+ populate_stats(site, [+ build(:event, pathname: "/en/register"),+ build(:event, pathname: "/en/register", name: "Signup")+ ])++ insert(:goal, %{site: site, event_name: "Signup"})++ conn =+ get(conn, "/api/v1/stats/breakdown", %{+ "site_id" => site.domain,+ "period" => "day",+ "property" => "event:page",+ "filters" => "event:goal==Signup",+ "metrics" => "conversion_rate"+ })++ assert json_response(conn, 200) == %{+ "results" => [+ %{+ "page" => "/en/register",+ "conversion_rate" => 50+ }+ ]+ }+ end++ test "returns conversion_rate in a multi-goal filtered visit:screen_size breakdown", %{+ conn: conn,+ site: site+ } do+ populate_stats(site, [+ build(:event, screen_size: "Mobile"),+ build(:event, screen_size: "Mobile", name: "AddToCart"),+ build(:event, screen_size: "Mobile", name: "AddToCart"),+ build(:event, screen_size: "Desktop", name: "AddToCart", user_id: 1),+ build(:event, screen_size: "Desktop", name: "Purchase", user_id: 1),+ build(:event, screen_size: "Desktop")+ ])++ # Make sure that revenue goals are treated the same+ # way as regular custom event goals+ insert(:goal, %{site: site, event_name: "Purchase", currency: :EUR})+ insert(:goal, %{site: site, event_name: "AddToCart"})++ conn =+ get(conn, "/api/v1/stats/breakdown", %{+ "site_id" => site.domain,+ "period" => "day",+ "property" => "visit:device",+ "filters" => "event:goal==AddToCart|Purchase",+ "metrics" => "visitors,events,conversion_rate"+ })++ assert json_response(conn, 200) == %{+ "results" => [+ %{+ "device" => "Mobile",+ "visitors" => 2,+ "events" => 2,+ "conversion_rate" => 66.7+ },+ %{+ "device" => "Desktop",+ "visitors" => 1,+ "events" => 2,+ "conversion_rate" => 50+ }+ ]+ }+ end++ test "returns conversion_rate alone in a goal filtered visit:screen_size breakdown", %{+ conn: conn,+ site: site+ } do+ populate_stats(site, [+ build(:event, screen_size: "Mobile"),+ build(:event, screen_size: "Mobile", name: "AddToCart")+ ])++ insert(:goal, %{site: site, event_name: "AddToCart"})++ conn =+ get(conn, "/api/v1/stats/breakdown", %{+ "site_id" => site.domain,+ "period" => "day",+ "property" => "visit:device",+ "filters" => "event:goal==AddToCart",+ "metrics" => "conversion_rate"+ })++ assert json_response(conn, 200) == %{+ "results" => [+ %{+ "device" => "Mobile",+ "conversion_rate" => 50+ }+ ]+ }+ end++ test "returns conversion_rate for a browser_version breakdown with pagination limit", %{+ site: site,+ conn: conn+ } do+ populate_stats(site, [+ build(:pageview, browser: "Firefox", browser_version: "110"),+ build(:pageview, browser: "Firefox", browser_version: "110"),+ build(:pageview, browser: "Chrome", browser_version: "110"),+ build(:pageview, browser: "Chrome", browser_version: "110"),+ build(:pageview, browser: "Avast Secure Browser", browser_version: "110"),+ build(:pageview, browser: "Avast Secure Browser", browser_version: "110"),+ build(:event, name: "Signup", browser: "Edge", browser_version: "110")+ ])++ insert(:goal, site: site, event_name: "Signup")++ conn =+ get(conn, "/api/v1/stats/breakdown", %{+ "site_id" => site.domain,+ "period" => "day",+ "property" => "visit:browser_version",+ "filters" => "event:goal==Signup",+ "metrics" => "visitors,conversion_rate",+ "page" => 1,+ "limit" => 1+ })++ assert json_response(conn, 200) == %{+ "results" => [+ %{+ "browser" => "Edge",+ "browser_version" => "110",+ "visitors" => 1,+ "conversion_rate" => 100.0+ }+ ]+ }+ end+test "all metrics for breakdown by visit prop", %{conn: conn, site: site} dopopulate_stats(site, [build(:pageview,diff --git a/test/plausible_web/controllers/api/external_stats_controller/timeseries_test.exs b/test/plausible_web/controllers/api/external_stats_controller/timeseries_test.exsindex ac13e1051a..bfbfb37269 100644--- a/test/plausible_web/controllers/api/external_stats_controller/timeseries_test.exs+++ b/test/plausible_web/controllers/api/external_stats_controller/timeseries_test.exs@@ -1093,6 +1093,22 @@ defmodule PlausibleWeb.Api.ExternalStatsController.TimeseriesTest doenddescribe "metrics" do+ test "validates that conversion_rate cannot be queried without a goal filter", %{+ conn: conn,+ site: site+ } do+ conn =+ get(conn, "/api/v1/stats/timeseries", %{+ "site_id" => site.domain,+ "metrics" => "conversion_rate"+ })++ assert %{"error" => msg} = json_response(conn, 400)++ assert msg ==+ "Metric `conversion_rate` can only be queried in a goal breakdown or with a goal filter"+ end+test "shows pageviews,visits,views_per_visit for last 7d", %{conn: conn, site: site} dopopulate_stats(site, [build(:pageview,diff --git a/test/plausible_web/controllers/api/stats_controller/browsers_test.exs b/test/plausible_web/controllers/api/stats_controller/browsers_test.exsindex 9e4a0ca142..7d4e94890d 100644--- a/test/plausible_web/controllers/api/stats_controller/browsers_test.exs+++ b/test/plausible_web/controllers/api/stats_controller/browsers_test.exs@@ -171,6 +171,50 @@ defmodule PlausibleWeb.Api.StatsController.BrowsersTest dodescribe "GET /api/stats/:domain/browser-versions" dosetup [:create_user, :log_in, :create_new_site]+ test "returns correct conversion_rate when browser_version clashes across browsers", %{+ conn: conn,+ site: site+ } do+ populate_stats(site, [+ build(:event, browser: "Chrome", browser_version: "110", name: "Signup"),+ build(:event, browser: "Chrome", browser_version: "110", name: "Signup"),+ build(:pageview, browser: "Chrome", browser_version: "110"),+ build(:pageview, browser: "Chrome", browser_version: "121"),+ build(:pageview, browser: "Chrome", browser_version: "121"),+ build(:event, browser: "Firefox", browser_version: "121", name: "Signup"),+ build(:pageview, browser: "Firefox", browser_version: "110"),+ build(:pageview, browser: "Firefox", browser_version: "110"),+ build(:pageview, browser: "Firefox", browser_version: "110"),+ build(:pageview, browser: "Firefox", browser_version: "110")+ ])++ insert(:goal, site: site, event_name: "Signup")+ filters = Jason.encode!(%{goal: "Signup"})++ json_response =+ get(+ conn,+ "/api/stats/#{site.domain}/browser-versions?period=day&filters=#{filters}"+ )+ |> json_response(200)++ assert %{+ "browser" => "Chrome",+ "conversion_rate" => 66.7,+ "name" => "110",+ "total_visitors" => 3,+ "visitors" => 2+ } == List.first(json_response)++ assert %{+ "browser" => "Firefox",+ "conversion_rate" => 100.0,+ "name" => "121",+ "total_visitors" => 1,+ "visitors" => 1+ } == List.last(json_response)+ end+test "returns top browser versions by unique visitors", %{conn: conn, site: site} dopopulate_stats(site, [build(:pageview, browser: "Chrome", browser_version: "78.0"),diff --git a/test/plausible_web/controllers/api/stats_controller/pages_test.exs b/test/plausible_web/controllers/api/stats_controller/pages_test.exsindex f8a1f411aa..976745ee51 100644--- a/test/plausible_web/controllers/api/stats_controller/pages_test.exs+++ b/test/plausible_web/controllers/api/stats_controller/pages_test.exs@@ -1313,17 +1313,13 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do%{"total_visitors" => 2,"visitors" => 1,- "visits" => 1,"name" => "/page1",- "visit_duration" => 0,"conversion_rate" => 50.0},%{"total_visitors" => 1,"visitors" => 1,- "visits" => 1,"name" => "/page2",- "visit_duration" => 900,"conversion_rate" => 100.0}]@@ -1508,14 +1504,12 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do"name" => "/exit1","visitors" => 1,"total_visitors" => 1,- "visits" => 1,"conversion_rate" => 100.0},%{"name" => "/exit2","visitors" => 1,"total_visitors" => 1,- "visits" => 1,"conversion_rate" => 100.0}]diff --git a/test/plausible_web/controllers/api/stats_controller/top_stats_test.exs b/test/plausible_web/controllers/api/stats_controller/top_stats_test.exsindex 66b078b44c..c2dd91541e 100644--- a/test/plausible_web/controllers/api/stats_controller/top_stats_test.exs+++ b/test/plausible_web/controllers/api/stats_controller/top_stats_test.exs@@ -1094,7 +1094,7 @@ defmodule PlausibleWeb.Api.StatsController.TopStatsTest dores = json_response(conn, 200)assert %{- "change" => -50,+ "change" => -33.4,"comparison_value" => 66.7,"name" => "Conversion rate","value" => 33.3