Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.
TypeScript analyticsanalytics-dashboardclickhouseelixirgoogle-analyticsmarketingopen-source-analyticsphoenixplausible-analyticspostgresqlprivacyprivacy-friendlyself-hostedsimple-analyticsstatisticstailwindcssweb-analyticswebsite-analyticswebsite-statswebsite-tracking
APIv2: Replace breakdown module with QueryBuilder (#4293)
Karl-Aksel Puulmann0594478addcd
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 065196ce6b..75f271eeb9 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -16,6 +16,7 @@ All notable changes to this project will be documented in this file.- Increase hourly request limit for API keys in CE from 600 to 1000000 (practically removing the limit) plausible/analytics#4200- Make TCP connections try IPv6 first with IPv4 fallback in CE plausible/analytics#4245- `is` and `is not` filters in dashboard no longer support wildcards. Use contains/does not contain filter instead.+- `bounce_rate` metric now returns 0 instead of null for event:page breakdown when page has never been entry page.### Fixeddiff --git a/lib/plausible/stats/base.ex b/lib/plausible/stats/base.exindex 3b3dfd2e33..5354833a1f 100644--- a/lib/plausible/stats/base.ex+++ b/lib/plausible/stats/base.ex@@ -250,6 +250,8 @@ defmodule Plausible.Stats.Base doenddefp select_session_metric(:percentage, _query), do: %{}+ defp select_session_metric(:conversion_rate, _query), do: %{}+ defp select_session_metric(:group_conversion_rate, _query), do: %{}def filter_converted_sessions(db_query, site, query) doif Query.has_event_filters?(query) dodiff --git a/lib/plausible/stats/breakdown.ex b/lib/plausible/stats/breakdown.exindex d68e9fcad0..a0bdaa1509 100644--- a/lib/plausible/stats/breakdown.ex+++ b/lib/plausible/stats/breakdown.ex@@ -3,344 +3,63 @@ defmodule Plausible.Stats.Breakdown douse Plausibleuse Plausible.Stats.Fragments- import Plausible.Stats.{Base, Imported}+ import Plausible.Stats.Baseimport Ecto.Query- alias Plausible.Stats.{Query, Util, TableDecider}-- @no_ref "Direct / None"- @not_set "(not set)"-- @session_metrics [:bounce_rate, :visit_duration]-- @revenue_metrics on_ee(do: Plausible.Stats.Goal.Revenue.revenue_metrics(), else: [])-- @event_metrics [:visits, :visitors, :pageviews, :events, :percentage] ++ @revenue_metrics-- # These metrics can be asked from the `breakdown/5` function,- # but they are different from regular metrics such as `visitors`,- # or `bounce_rate` - we cannot currently "select them" directly in- # the db queries. Instead, we need to artificially append them to- # the breakdown results later on.- @computed_metrics [:conversion_rate, :total_visitors]-- def breakdown(site, query, metrics, pagination, opts \\ [])-- def breakdown(- site,- %Query{dimensions: ["event:goal"]} = query,- metrics,- pagination,- opts- ) do- site = Plausible.Repo.preload(site, :goals)-- {event_goals, pageview_goals} = Enum.split_with(site.goals, & &1.event_name)- events = Enum.map(event_goals, & &1.event_name)-- event_query =- query- |> Query.put_filter([:is, "event:name", events])- |> Query.set_dimensions(["event:name"])-- if !Keyword.get(opts, :skip_tracing), do: Query.trace(query, metrics)-- no_revenue = {nil, metrics -- @revenue_metrics}-- {revenue_goals, metrics} =- on_ee do- if Plausible.Billing.Feature.RevenueGoals.enabled?(site) do- revenue_goals = Enum.filter(event_goals, &Plausible.Goal.Revenue.revenue?/1)- metrics = if Enum.empty?(revenue_goals), do: metrics -- @revenue_metrics, else: metrics-- {revenue_goals, metrics}- else- no_revenue- end- else- no_revenue- end-- metrics_to_select = Util.maybe_add_visitors_metric(metrics) -- @computed_metrics-- event_q =- if Enum.any?(event_goals) do- site- |> breakdown_events(event_query, metrics_to_select)- |> apply_pagination(pagination)- else- nil- end-- page_q =- if Enum.any?(pageview_goals) do- page_query = Query.set_dimensions(query, ["event:page"])-- page_exprs = Enum.map(pageview_goals, & &1.page_path)- page_regexes = Enum.map(page_exprs, &page_regex/1)-- select_columns = metrics_to_select |> select_event_metrics |> mark_revenue_as_nil-- from(e in base_event_query(site, page_query),- order_by: [desc: fragment("uniq(?)", e.user_id)],- where:- fragment(- "notEmpty(multiMatchAllIndices(?, ?) as indices)",- e.pathname,- ^page_regexes- ) and e.name == "pageview",- array_join: index in fragment("indices"),- group_by: index,- select: %{- name: fragment("concat('Visit ', ?[?])", ^page_exprs, index)- }- )- |> select_merge(^select_columns)- |> merge_imported_pageview_goals(site, page_query, page_exprs, metrics_to_select)- |> apply_pagination(pagination)- else- nil- end-- full_q =- case {event_q, page_q} do- {nil, nil} ->- nil-- {event_q, nil} ->- event_q-- {nil, page_q} ->- page_q-- {event_q, page_q} ->- from(- e in subquery(union_all(event_q, ^page_q)),- order_by: [desc: e.visitors]- )- |> apply_pagination(pagination)- end-- if full_q do- full_q- |> maybe_add_conversion_rate(site, query, metrics)- |> ClickhouseRepo.all()- |> transform_keys(%{name: :goal})- |> cast_revenue_metrics_to_money(revenue_goals)- |> Util.keep_requested_metrics(metrics)- else- []- end- end-- def breakdown(- site,- %Query{dimensions: ["event:props:" <> custom_prop]} = query,- metrics,- pagination,- opts- ) do- {currency, metrics} =- on_ee do- Plausible.Stats.Goal.Revenue.get_revenue_tracking_currency(site, query, metrics)- else- {nil, metrics}- end-- metrics_to_select = Util.maybe_add_visitors_metric(metrics) -- @computed_metrics-- if !Keyword.get(opts, :skip_tracing), do: Query.trace(query, metrics)-- breakdown_events(site, query, metrics_to_select)- |> maybe_add_conversion_rate(site, query, metrics)- |> paginate_and_execute(metrics, pagination)- |> transform_keys(%{breakdown_prop_value: custom_prop})- |> Enum.map(&cast_revenue_metrics_to_money(&1, currency))- end-- def breakdown(site, %Query{dimensions: ["event:page"]} = query, metrics, pagination, opts) do- event_metrics =- metrics- |> Util.maybe_add_visitors_metric()- |> Enum.filter(&(&1 in @event_metrics))-- if !Keyword.get(opts, :skip_tracing), do: Query.trace(query, metrics)-- event_result =- site- |> breakdown_events(query, event_metrics)- |> maybe_add_group_conversion_rate(&breakdown_events/3, site, query, metrics)- |> paginate_and_execute(metrics, pagination)- |> maybe_add_time_on_page(site, query, metrics)-- session_metrics = Enum.filter(metrics, &(&1 in @session_metrics))-- entry_page_query =- case event_result do- [] ->- query-- pages ->- query- |> Query.remove_filters(["event:page"])- |> Query.put_filter([:is, "visit:entry_page", Enum.map(pages, & &1[:page])])- |> Query.set_dimensions(["visit:entry_page"])- end-- if Enum.any?(event_metrics) && Enum.empty?(event_result) do- []- else- {limit, _page} = pagination-- session_result =- breakdown_sessions(site, entry_page_query, session_metrics)- |> paginate_and_execute(session_metrics, {limit, 1})- |> transform_keys(%{entry_page: :page})-- metrics = metrics ++ [:page]-- zip_results(- event_result,- session_result,- :page,- metrics+ alias Plausible.Stats.{Query, QueryOptimizer, QueryResult, SQL}+ alias Plausible.Stats.Filters.QueryParser++ def breakdown(site, %Query{dimensions: [dimension]} = query, metrics, pagination, _opts \\ []) do+ transformed_metrics = transform_metrics(metrics, dimension)++ query_with_metrics =+ Query.set(+ query,+ metrics: transformed_metrics,+ order_by: infer_order_by(transformed_metrics, dimension),+ dimensions: transform_dimensions(dimension),+ filters: query.filters ++ dimension_filters(dimension),+ preloaded_goals: QueryParser.preload_goals_if_needed(site, query.filters, [dimension]),+ v2: true,+ # Allow pageview and event metrics to be queried off of sessions table+ legacy_breakdown: true)- |> Enum.map(&Map.take(&1, metrics))- end- end-- def breakdown(site, %Query{dimensions: ["event:name"]} = query, metrics, pagination, opts) do- if !Keyword.get(opts, :skip_tracing), do: Query.trace(query, metrics)-- breakdown_events(site, query, metrics)- |> paginate_and_execute(metrics, pagination)- end-- def breakdown(site, query, metrics, pagination, opts) do- query = maybe_update_breakdown_filters(query)- if !Keyword.get(opts, :skip_tracing), do: Query.trace(query, metrics)-- metrics_to_select = Util.maybe_add_visitors_metric(metrics) -- @computed_metrics-- case breakdown_table(query, metrics) do- :session ->- breakdown_sessions(site, query, metrics_to_select)- |> maybe_add_group_conversion_rate(&breakdown_sessions/3, site, query, metrics)- |> paginate_and_execute(metrics, pagination)-- :event ->- breakdown_events(site, query, metrics_to_select)- |> maybe_add_group_conversion_rate(&breakdown_events/3, site, query, metrics)- |> paginate_and_execute(metrics, pagination)- end- end-- defp maybe_update_breakdown_filters(%Query{dimensions: [visit_entry_prop]} = query)- when visit_entry_prop in [- "visit:source",- "visit:entry_page",- "visit:utm_medium",- "visit:utm_source",- "visit:utm_campaign",- "visit:utm_content",- "visit:utm_term",- "visit:entry_page",- "visit:referrer"- ] do- update_hostname_filter_prop(query, "visit:entry_page_hostname")- end-- defp maybe_update_breakdown_filters(%Query{dimensions: ["visit:exit_page"]} = query) do- update_hostname_filter_prop(query, "visit:exit_page_hostname")- end-- defp maybe_update_breakdown_filters(query) do- query- end+ |> QueryOptimizer.optimize()- defp update_hostname_filter_prop(query, visit_prop) do- case Query.get_filter(query, "event:hostname") do- nil ->- query+ q = SQL.QueryBuilder.build(query_with_metrics, site)- [op, "event:hostname", value] ->- query- |> Query.put_filter([op, visit_prop, value])- end- end-- # Backwards compatibility- defp breakdown_table(%Query{experimental_reduced_joins?: false}, _), do: :session-- defp breakdown_table(%Query{dimensions: ["visit:entry_page"]}, _metrics), do: :session- defp breakdown_table(%Query{dimensions: ["visit:entry_page_hostname"]}, _metrics), do: :session- defp breakdown_table(%Query{dimensions: ["visit:exit_page"]}, _metrics), do: :session- defp breakdown_table(%Query{dimensions: ["visit:exit_page_hostname"]}, _metrics), do: :session-- defp breakdown_table(%Query{dimensions: [_dimension]} = query, metrics) do- {_, session_metrics, _} = TableDecider.partition_metrics(metrics, query)-- if not Enum.empty?(session_metrics) do- :session- else- :event- end+ q+ |> apply_pagination(pagination)+ |> ClickhouseRepo.all()+ |> QueryResult.from(query_with_metrics)+ |> build_breakdown_result(query_with_metrics, metrics)+ |> maybe_add_time_on_page(site, query_with_metrics, metrics)+ |> update_currency_metrics(site, query_with_metrics)end- defp zip_results(event_result, session_result, property, metrics) do- null_row = Enum.map(metrics, fn metric -> {metric, nil} end) |> Enum.into(%{})+ defp build_breakdown_result(query_result, query, metrics) do+ query_result.results+ |> Enum.map(fn %{dimensions: dimensions, metrics: entry_metrics} ->+ dimension_map =+ query.dimensions |> Enum.map(&result_key/1) |> Enum.zip(dimensions) |> Enum.into(%{})- prop_values =- Enum.map(event_result ++ session_result, fn row -> row[property] end)- |> Enum.uniq()+ metrics_map = Enum.zip(metrics, entry_metrics) |> Enum.into(%{})- Enum.map(prop_values, fn value ->- event_row = Enum.find(event_result, fn row -> row[property] == value end) || %{}- session_row = Enum.find(session_result, fn row -> row[property] == value end) || %{}-- null_row- |> Map.merge(event_row)- |> Map.merge(session_row)+ Map.merge(dimension_map, metrics_map)end)- |> sort_results(metrics)- end-- defp breakdown_sessions(site, %Query{dimensions: [dimension]} = query, metrics) do- from(s in query_sessions(site, query),- order_by: [desc: fragment("uniq(?)", s.user_id)],- select: ^select_session_metrics(metrics, query)- )- |> filter_converted_sessions(site, query)- |> do_group_by(dimension)- |> merge_imported(site, query, metrics)- |> add_percentage_metric(site, query, metrics)- end-- defp breakdown_events(site, %Query{dimensions: [dimension]} = query, metrics) do- from(e in base_event_query(site, query),- order_by: [desc: fragment("uniq(?)", e.user_id)],- select: %{}- )- |> do_group_by(dimension)- |> select_merge(^select_event_metrics(metrics))- |> merge_imported(site, query, metrics)- |> add_percentage_metric(site, query, metrics)end- defp paginate_and_execute(_, [], _), do: []-- defp paginate_and_execute(q, metrics, pagination) do- q- |> apply_pagination(pagination)- |> ClickhouseRepo.all()- |> Util.keep_requested_metrics(metrics)- end+ defp result_key("event:props:" <> custom_property), do: custom_property+ defp result_key("event:" <> key), do: key |> String.to_existing_atom()+ defp result_key("visit:" <> key), do: key |> String.to_existing_atom()+ defp result_key(dimension), do: dimensiondefp maybe_add_time_on_page(event_results, site, query, metrics) do- if :time_on_page in metrics do+ if query.dimensions == ["event:page"] and :time_on_page in metrics dopages = Enum.map(event_results, & &1[:page])time_on_page_result = breakdown_time_on_page(site, query, pages)- Enum.map(event_results, fn row ->+ event_results+ |> Enum.map(fn row ->Map.put(row, :time_on_page, time_on_page_result[row[:page]])end)else@@ -431,359 +150,102 @@ defmodule Plausible.Stats.Breakdown do|> Map.new()end- defp do_group_by(- %Ecto.Query{from: %Ecto.Query.FromExpr{source: {"events" <> _, _}}} = q,- "event:props:" <> prop- ) do- from(- e in q,- select_merge: %{- breakdown_prop_value:- selected_as(- fragment(- "if(not empty(?), ?, '(none)')",- get_by_key(e, :meta, ^prop),- get_by_key(e, :meta, ^prop)- ),- :breakdown_prop_value- )- },- group_by: selected_as(:breakdown_prop_value),- order_by: {:asc, selected_as(:breakdown_prop_value)}- )- end-- defp do_group_by(- %Ecto.Query{from: %Ecto.Query.FromExpr{source: {"events" <> _, _}}} = q,- "event:name"- ) do- from(- e in q,- group_by: e.name,- select_merge: %{name: e.name},- order_by: {:asc, e.name}- )- end-- defp do_group_by(- %Ecto.Query{from: %Ecto.Query.FromExpr{source: {"events" <> _, _}}} = q,- "event:page"- ) do- from(- e in q,- group_by: e.pathname,- select_merge: %{page: e.pathname},- order_by: {:asc, e.pathname}- )- end-- defp do_group_by(q, "visit:source") do- from(- s in q,- group_by: s.source,- select_merge: %{- source: fragment("if(empty(?), ?, ?)", s.source, @no_ref, s.source)- },- order_by: {:asc, s.source}- )- end-- defp do_group_by(q, "visit:country") do- from(- s in q,- where: s.country != "\0\0" and s.country != "ZZ",- group_by: s.country,- select_merge: %{country: s.country},- order_by: {:asc, s.country}- )- end-- defp do_group_by(q, "visit:region") do- from(- s in q,- where: s.region != "",- group_by: s.region,- select_merge: %{region: s.region},- order_by: {:asc, s.region}- )- end-- defp do_group_by(q, "visit:city") do- from(- s in q,- where: s.city != 0,- group_by: s.city,- select_merge: %{city: s.city},- order_by: {:asc, s.city}- )- end-- defp do_group_by(q, "visit:entry_page") do- from(- s in q,- # Sessions without pageviews don't get entry_page assigned, hence they should get ignored- where: s.entry_page != "",- group_by: s.entry_page,- select_merge: %{entry_page: s.entry_page},- order_by: {:asc, s.entry_page}- )- end-- defp do_group_by(q, "visit:exit_page") do- from(- s in q,- # Sessions without pageviews don't get entry_page assigned, hence they should get ignored- where: s.entry_page != "",- group_by: s.exit_page,- select_merge: %{exit_page: s.exit_page},- order_by: {:asc, s.exit_page}- )- end-- defp do_group_by(q, "visit:referrer") do- from(- s in q,- group_by: s.referrer,- select_merge: %{- referrer: fragment("if(empty(?), ?, ?)", s.referrer, @no_ref, s.referrer)- },- order_by: {:asc, s.referrer}- )- end-- defp do_group_by(q, "visit:utm_medium") do- from(- s in q,- where: fragment("not empty(?)", s.utm_medium),- group_by: s.utm_medium,- select_merge: %{- utm_medium: s.utm_medium- }- )- end+ defp transform_metrics(metrics, dimension) do+ metrics =+ if is_nil(metric_to_order_by(metrics)) do+ metrics ++ [:visitors]+ else+ metrics+ end- defp do_group_by(q, "visit:utm_source") do- from(- s in q,- where: fragment("not empty(?)", s.utm_source),- group_by: s.utm_source,- select_merge: %{- utm_source: s.utm_source- }- )+ Enum.map(metrics, fn metric ->+ case {metric, dimension} do+ {:conversion_rate, "event:props:" <> _} -> :conversion_rate+ {:conversion_rate, "event:goal"} -> :conversion_rate+ {:conversion_rate, _} -> :group_conversion_rate+ _ -> metric+ end+ end)end- defp do_group_by(q, "visit:utm_campaign") do- from(- s in q,- where: fragment("not empty(?)", s.utm_campaign),- group_by: s.utm_campaign,- select_merge: %{- utm_campaign: s.utm_campaign- }- )- end+ defp infer_order_by(metrics, "event:goal"), do: [{metric_to_order_by(metrics), :desc}]- defp do_group_by(q, "visit:utm_content") do- from(- s in q,- where: fragment("not empty(?)", s.utm_content),- group_by: s.utm_content,- select_merge: %{- utm_content: s.utm_content- }- )- end+ defp infer_order_by(metrics, dimension),+ do: [{metric_to_order_by(metrics), :desc}, {dimension, :asc}]- defp do_group_by(q, "visit:utm_term") do- from(- s in q,- where: fragment("not empty(?)", s.utm_term),- group_by: s.utm_term,- select_merge: %{- utm_term: s.utm_term- }- )+ defp metric_to_order_by(metrics) do+ Enum.find(metrics, &(&1 != :time_on_page))end- defp do_group_by(q, "visit:device") do- from(- s in q,- group_by: s.device,- select_merge: %{- device: fragment("if(empty(?), ?, ?)", s.device, @not_set, s.device)- },- order_by: {:asc, s.device}- )- end+ def transform_dimensions("visit:browser_version"),+ do: ["visit:browser", "visit:browser_version"]- defp do_group_by(q, "visit:os") do- from(- s in q,- group_by: s.os,- select_merge: %{- os: fragment("if(empty(?), ?, ?)", s.os, @not_set, s.os)- },- order_by: {:asc, s.os}- )- end+ def transform_dimensions("visit:os_version"), do: ["visit:os", "visit:os_version"]+ def transform_dimensions(dimension), do: [dimension]- defp do_group_by(q, "visit:os_version") do- from(- s in q,- group_by: [s.os, s.os_version],- select_merge: %{- os: fragment("if(empty(?), ?, ?)", s.os, @not_set, s.os),- os_version:- fragment(- "if(empty(?), ?, ?)",- s.os_version,- @not_set,- s.os_version- )- },- order_by: {:asc, s.os_version}- )- end+ @filter_dimensions_not %{+ "visit:city" => [0],+ "visit:country" => ["\0\0", "ZZ"],+ "visit:region" => [""],+ "visit:utm_medium" => [""],+ "visit:utm_source" => [""],+ "visit:utm_campaign" => [""],+ "visit:utm_content" => [""],+ "visit:utm_term" => [""],+ "visit:entry_page" => [""],+ "visit:exit_page" => [""]+ }- defp do_group_by(q, "visit:browser") do- from(- s in q,- group_by: s.browser,- select_merge: %{- browser: fragment("if(empty(?), ?, ?)", s.browser, @not_set, s.browser)- },- order_by: {:asc, s.browser}- )- end+ @extra_filter_dimensions Map.keys(@filter_dimensions_not)- defp do_group_by(q, "visit:browser_version") do- from(- s in q,- group_by: [s.browser, s.browser_version],- select_merge: %{- browser: fragment("if(empty(?), ?, ?)", s.browser, @not_set, s.browser),- browser_version:- fragment("if(empty(?), ?, ?)", s.browser_version, @not_set, s.browser_version)- },- order_by: {:asc, s.browser_version}- )+ defp dimension_filters(dimension) when dimension in @extra_filter_dimensions do+ [[:is_not, dimension, Map.get(@filter_dimensions_not, dimension)]]end- defp group_by_field_names("event:props:" <> _prop), do: [:name]- defp group_by_field_names("visit:os_version"), do: [:os, :os_version]- defp group_by_field_names("visit:browser_version"), do: [:browser, :browser_version]+ defp dimension_filters(_), do: []- defp group_by_field_names(property), do: [Plausible.Stats.Filters.without_prefix(property)]+ defp apply_pagination(q, {limit, page}) do+ offset = (page - 1) * limit- defp on_matches_group_by(fields) do- Enum.reduce(fields, nil, &fields_equal/2)+ q+ |> limit(^limit)+ |> offset(^offset)end- defp outer_order_by(fields) do- Enum.map(fields, fn field_name -> {:asc, dynamic([q], field(q, ^field_name))} end)- end+ on_ee do+ defp update_currency_metrics(results, site, %Query{dimensions: ["event:goal"]}) do+ site = Plausible.Repo.preload(site, :goals)- defp fields_equal(field_name, nil),- do: dynamic([a, b], field(a, ^field_name) == field(b, ^field_name))-- defp fields_equal(field_name, condition),- do: dynamic([a, b], field(a, ^field_name) == field(b, ^field_name) and ^condition)-- defp sort_results(results, metrics) do- Enum.sort_by(- results,- fn entry ->- case entry[sorting_key(metrics)] do- nil -> 0- n -> n- end- end,- :desc- )- end+ {event_goals, _pageview_goals} = Enum.split_with(site.goals, & &1.event_name)+ revenue_goals = Enum.filter(event_goals, &Plausible.Goal.Revenue.revenue?/1)- # This function injects a conversion_rate metric into- # a breakdown query. It is calculated as X / Y, where:- #- # * X is the number of conversions for a breakdown- # result (conversion = number of visitors who- # completed the filtered goal with the filtered- # custom properties).- #- # * Y is the number of all visitors for this breakdown- # result without the `event:goal` and `event:props:*`- # filters.- defp maybe_add_group_conversion_rate(- q,- breakdown_fn,- site,- %Query{dimensions: [dimension]} = query,- metrics- ) do- if :conversion_rate in metrics do- breakdown_total_visitors_query =- query |> Query.remove_filters(["event:goal", "event:props"])-- breakdown_total_visitors_q =- breakdown_fn.(site, breakdown_total_visitors_query, [:visitors])-- from(e in subquery(q),- left_join: c in subquery(breakdown_total_visitors_q),- on: ^on_matches_group_by(group_by_field_names(dimension)),- select_merge: %{- total_visitors: c.visitors,- conversion_rate:- fragment(- "if(? > 0, round(? / ? * 100, 1), 0)",- c.visitors,- e.visitors,- c.visitors- )- },- order_by: [desc: e.visitors],- order_by: ^outer_order_by(group_by_field_names(dimension))- )- else- q+ if length(revenue_goals) > 0 and Plausible.Billing.Feature.RevenueGoals.enabled?(site) do+ Plausible.Stats.Goal.Revenue.cast_revenue_metrics_to_money(results, revenue_goals)+ else+ remove_revenue_metrics(results)+ endend- end- # When querying custom event goals and pageviewgoals together, UNION ALL is used- # so the same fields must be present on both sides of the union. This change to the- # query will ensure that we don't unnecessarily read revenue column for pageview goals- defp mark_revenue_as_nil(select_columns) do- select_columns- |> Map.replace(:total_revenue, nil)- |> Map.replace(:average_revenue, nil)- end+ defp update_currency_metrics(results, site, query) do+ {currency, _metrics} =+ Plausible.Stats.Goal.Revenue.get_revenue_tracking_currency(site, query, query.metrics)- defp sorting_key(metrics) do- if Enum.member?(metrics, :visitors), do: :visitors, else: List.first(metrics)+ if currency do+ results+ |> Enum.map(&Plausible.Stats.Goal.Revenue.cast_revenue_metrics_to_money(&1, currency))+ else+ remove_revenue_metrics(results)+ end+ end+ else+ defp update_currency_metrics(results, _site, _query), do: remove_revenue_metrics(results)end- defp transform_keys(results, keys_to_replace) do+ defp remove_revenue_metrics(results) doEnum.map(results, fn map ->- Enum.map(map, fn {key, val} ->- {Map.get(keys_to_replace, key, key), val}- end)- |> Enum.into(%{})+ map+ |> Map.delete(:total_revenue)+ |> Map.delete(:average_revenue)end)end-- defp apply_pagination(q, {limit, page}) do- offset = (page - 1) * limit-- q- |> limit(^limit)- |> offset(^offset)- end-- on_ee do- defp cast_revenue_metrics_to_money(results, revenue_goals) do- Plausible.Stats.Goal.Revenue.cast_revenue_metrics_to_money(results, revenue_goals)- end- else- defp cast_revenue_metrics_to_money(results, _revenue_goals), do: results- endenddiff --git a/lib/plausible/stats/filters/query_parser.ex b/lib/plausible/stats/filters/query_parser.exindex 0184478905..3f802dcb64 100644--- a/lib/plausible/stats/filters/query_parser.ex+++ b/lib/plausible/stats/filters/query_parser.ex@@ -4,6 +4,7 @@ defmodule Plausible.Stats.Filters.QueryParser doalias Plausible.Stats.TableDecideralias Plausible.Stats.Filtersalias Plausible.Stats.Query+ alias Plausible.Stats.Metricsdef parse(site, params, now \\ nil) when is_map(params) dowith {:ok, metrics} <- parse_metrics(Map.get(params, "metrics", [])),@@ -44,17 +45,12 @@ defmodule Plausible.Stats.Filters.QueryParser dodefp parse_metrics(_invalid_metrics), do: {:error, "Invalid metrics passed"}- defp parse_metric("time_on_page"), do: {:ok, :time_on_page}- defp parse_metric("conversion_rate"), do: {:ok, :conversion_rate}- defp parse_metric("group_conversion_rate"), do: {:ok, :group_conversion_rate}- defp parse_metric("visitors"), do: {:ok, :visitors}- defp parse_metric("pageviews"), do: {:ok, :pageviews}- defp parse_metric("events"), do: {:ok, :events}- defp parse_metric("visits"), do: {:ok, :visits}- defp parse_metric("bounce_rate"), do: {:ok, :bounce_rate}- defp parse_metric("visit_duration"), do: {:ok, :visit_duration}- defp parse_metric("views_per_visit"), do: {:ok, :views_per_visit}- defp parse_metric(unknown_metric), do: {:error, "Unknown metric '#{inspect(unknown_metric)}'"}+ defp parse_metric(metric_str) do+ case Metrics.from_string(metric_str) do+ {:ok, metric} -> {:ok, metric}+ _ -> {:error, "Unknown metric '#{inspect(metric_str)}'"}+ end+ enddef parse_filters(filters) when is_list(filters) doparse_list(filters, &parse_filter/1)@@ -284,7 +280,7 @@ defmodule Plausible.Stats.Filters.QueryParser doendend- defp preload_goals_if_needed(site, filters, dimensions) do+ def preload_goals_if_needed(site, filters, dimensions) dogoal_filters? =Enum.any?(filters, fn [_, filter_key | _rest] -> filter_key == "event:goal" end)diff --git a/lib/plausible/stats/filters/where_builder.ex b/lib/plausible/stats/filters/where_builder.exindex e04396d999..257d4257a0 100644--- a/lib/plausible/stats/filters/where_builder.ex+++ b/lib/plausible/stats/filters/where_builder.ex@@ -10,6 +10,8 @@ defmodule Plausible.Stats.Filters.WhereBuilder douse Plausible.Stats.Fragments+ require Logger+@sessions_only_visit_fields [:entry_page,:exit_page,@@ -150,6 +152,15 @@ defmodule Plausible.Stats.Filters.WhereBuilder dotrueend+ defp add_filter(table, _query, filter) do+ Logger.info("Unable to process garbage filter. No results are returned",+ table: table,+ filter: filter+ )++ false+ end+defp filter_custom_prop(prop_name, column_name, [:is, _, values]) donone_value_included = Enum.member?(values, "(none)")diff --git a/lib/plausible/stats/imported/base.ex b/lib/plausible/stats/imported/base.exindex a446817a37..5b774d9ea9 100644--- a/lib/plausible/stats/imported/base.ex+++ b/lib/plausible/stats/imported/base.ex@@ -54,10 +54,12 @@ defmodule Plausible.Stats.Imported.Base dodef property_to_table_mappings(), do: @property_to_table_mappingsdef query_imported(site, query) do- query- |> transform_filters()- |> decide_table()- |> query_imported(site, query)+ [table] =+ query+ |> transform_filters()+ |> decide_tables()++ query_imported(table, site, query)enddef query_imported(table, site, query) do@@ -75,13 +77,13 @@ defmodule Plausible.Stats.Imported.Base do|> apply_filter(query)end- def decide_table(query) do+ def decide_tables(query) doquery = transform_filters(query)if custom_prop_query?(query) dodo_decide_custom_prop_table(query)else- do_decide_table(query)+ do_decide_tables(query)endend@@ -92,19 +94,17 @@ defmodule Plausible.Stats.Imported.Base do[:is, "event:name", ["pageview"]] -> true_ -> falseend)- |> Enum.flat_map(fn filter ->- case filter do- [op, "event:goal", events] ->- events- |> Enum.group_by(&elem(&1, 0), &elem(&1, 1))- |> Enum.map(fn- {:event, names} -> [op, "event:name", names]- {:page, pages} -> [op, "event:page", pages]- end)-- filter ->- [filter]- end+ |> Enum.flat_map(fn+ [op, "event:goal", clauses] ->+ clauses+ |> Enum.group_by(&elem(&1, 0), &elem(&1, 1))+ |> Enum.map(fn+ {:event, names} -> [op, "event:name", names]+ {:page, pages} -> [op, "event:page", pages]+ end)++ filter ->+ [filter]end)struct!(query, filters: new_filters)@@ -136,10 +136,10 @@ defmodule Plausible.Stats.Imported.Base dodo_decide_custom_prop_table(query, custom_prop_filter)_ ->- nil+ []endelse- nil+ []endend@@ -158,23 +158,27 @@ defmodule Plausible.Stats.Imported.Base doend)if has_required_name_filter? and not has_unsupported_filters? do- "imported_custom_events"+ ["imported_custom_events"]else- nil+ []endend- defp do_decide_table(%Query{filters: [], dimensions: []}), do: "imported_visitors"+ defp do_decide_tables(%Query{filters: [], dimensions: []}), do: ["imported_visitors"]- defp do_decide_table(%Query{filters: [], dimensions: ["event:goal"]}) do- "imported_custom_events"+ defp do_decide_tables(%Query{filters: [], dimensions: ["event:goal"]}) do+ ["imported_pages", "imported_custom_events"]end- defp do_decide_table(%Query{filters: [], dimensions: [dimension]}) do- @property_to_table_mappings[dimension]+ defp do_decide_tables(%Query{filters: [], dimensions: [dimension]}) do+ if Map.has_key?(@property_to_table_mappings, dimension) do+ [@property_to_table_mappings[dimension]]+ else+ []+ endend- defp do_decide_table(%Query{filters: filters, dimensions: ["event:goal"]}) do+ defp do_decide_tables(%Query{filters: filters, dimensions: ["event:goal"]}) dofilter_props = Enum.map(filters, &Enum.at(&1, 1))any_event_name_filters? = "event:name" in filter_props@@ -182,27 +186,28 @@ defmodule Plausible.Stats.Imported.Base doany_other_filters? = Enum.any?(filter_props, &(&1 not in ["event:page", "event:name"]))cond do- any_other_filters? -> nil- any_event_name_filters? and not any_page_filters? -> "imported_custom_events"- any_page_filters? and not any_event_name_filters? -> "imported_pages"- true -> nil+ any_other_filters? -> []+ any_event_name_filters? and not any_page_filters? -> ["imported_custom_events"]+ any_page_filters? and not any_event_name_filters? -> ["imported_pages"]+ true -> []endend- defp do_decide_table(%Query{filters: filters, dimensions: dimensions}) do+ defp do_decide_tables(%Query{filters: filters, dimensions: dimensions}) dotable_candidates =filters|> Enum.map(fn [_, filter_key | _] -> filter_key end)|> Enum.concat(dimensions)|> Enum.map(fn"visit:screen" -> "visit:device"- prop -> prop+ dimension -> dimensionend)|> Enum.map(&@property_to_table_mappings[&1])case Enum.uniq(table_candidates) do- [candidate] -> candidate- _ -> nil+ [nil] -> []+ [candidate] -> [candidate]+ _ -> []endenddiff --git a/lib/plausible/stats/imported/imported.ex b/lib/plausible/stats/imported/imported.exindex 6542ab44f3..57cd6737bb 100644--- a/lib/plausible/stats/imported/imported.ex+++ b/lib/plausible/stats/imported/imported.ex@@ -4,10 +4,11 @@ defmodule Plausible.Stats.Imported doimport Ecto.Queryimport Plausible.Stats.Fragments+ import Plausible.Stats.Util, only: [shortname: 2]- alias Plausible.Stats.Basealias Plausible.Stats.Importedalias Plausible.Stats.Query+ alias Plausible.Stats.SQL.QueryBuilder@no_ref "Direct / None"@not_set "(not set)"@@ -15,7 +16,7 @@ defmodule Plausible.Stats.Imported do@property_to_table_mappings Imported.Base.property_to_table_mappings()- @imported_properties Map.keys(@property_to_table_mappings) +++ @imported_dimensions Map.keys(@property_to_table_mappings) ++Plausible.Imported.imported_custom_props()@goals_with_url Plausible.Imported.goals_with_url()@@ -37,7 +38,7 @@ defmodule Plausible.Stats.Imported do(see `@goals_with_url` and `@goals_with_path`)."""def schema_supports_query?(query) do- not is_nil(Imported.Base.decide_table(query))+ length(Imported.Base.decide_tables(query)) > 0enddef merge_imported_country_suggestions(native_q, _site, %Plausible.Stats.Query{@@ -267,78 +268,6 @@ defmodule Plausible.Stats.Imported dodef merge_imported(q, _, %Query{include_imported: false}, _), do: q- # Note: Only called for APIv2, old APIs use merge_imported_pageview_goals- def merge_imported(q, site, %Query{dimensions: ["event:goal"]} = query, metrics)- when query.v2 do- {events, page_regexes} = Filters.Utils.split_goals_query_expressions(query.preloaded_goals)-- events_q =- "imported_custom_events"- |> Imported.Base.query_imported(site, query)- |> where([i], i.visitors > 0)- |> select_merge([i], %{- dim0: selected_as(fragment("-indexOf(?, ?)", ^events, i.name), :dim0)- })- |> select_imported_metrics(metrics)- |> group_by([], selected_as(:dim0))- |> where([], selected_as(:dim0) != 0)-- pages_q =- "imported_pages"- |> Imported.Base.query_imported(site, query)- |> where([i], i.visitors > 0)- |> where(- [i],- fragment("notEmpty(multiMatchAllIndices(?, ?) as indices)", i.page, ^page_regexes)- )- |> join(:array, index in fragment("indices"))- |> group_by([_i, index], index)- |> select_merge([_i, index], %{- dim0: type(fragment("?", index), :integer)- })- |> select_imported_metrics(metrics)-- q- |> naive_dimension_join(events_q, metrics)- |> naive_dimension_join(pages_q, metrics)- end-- def merge_imported(q, site, %Query{dimensions: [dimension]} = query, metrics)- when dimension in @imported_properties do- dim = Plausible.Stats.Filters.without_prefix(dimension)-- imported_q =- site- |> Imported.Base.query_imported(query)- |> where([i], i.visitors > 0)- |> group_imported_by(dim, query)- |> select_imported_metrics(metrics)-- join_on =- case dim do- _ when dim in [:url, :path] and not query.v2 ->- dynamic([s, i], s.breakdown_prop_value == i.breakdown_prop_value)-- :os_version when not query.v2 ->- dynamic([s, i], s.os == i.os and s.os_version == i.os_version)-- :browser_version when not query.v2 ->- dynamic([s, i], s.browser == i.browser and s.browser_version == i.browser_version)-- dim ->- dynamic([s, i], field(s, ^shortname(query, dim)) == field(i, ^shortname(query, dim)))- end-- from(s in Ecto.Query.subquery(q),- full_join: i in subquery(imported_q),- on: ^join_on,- select: %{}- )- |> select_joined_dimension(dim, query)- |> select_joined_metrics(metrics)- |> apply_order_by(query, metrics)- end-def merge_imported(q, site, %Query{dimensions: []} = query, metrics) doimported_q =site@@ -353,51 +282,82 @@ defmodule Plausible.Stats.Imported do|> select_joined_metrics(metrics)end- def merge_imported(q, _, _, _), do: q-- def merge_imported_pageview_goals(q, _, %Query{include_imported: false}, _, _), do: q+ def merge_imported(q, site, %Query{dimensions: ["event:goal"]} = query, metrics) do+ {events, page_regexes} = Filters.Utils.split_goals_query_expressions(query.preloaded_goals)- def merge_imported_pageview_goals(q, site, query, page_exprs, metrics) do- if Imported.Base.decide_table(query) == "imported_pages" do- page_regexes = Enum.map(page_exprs, &Base.page_regex/1)+ Imported.Base.decide_tables(query)+ |> Enum.map(fn+ "imported_custom_events" ->+ Imported.Base.query_imported("imported_custom_events", site, query)+ |> where([i], i.visitors > 0)+ |> select_merge([i], %{+ dim0:+ selected_as(+ fragment("-indexOf(?, ?)", type(^events, {:array, :string}), i.name),+ :dim0+ )+ })+ |> select_imported_metrics(metrics)+ |> group_by([], selected_as(:dim0))+ |> where([], selected_as(:dim0) != 0)- imported_q =- "imported_pages"- |> Imported.Base.query_imported(site, query)+ "imported_pages" ->+ Imported.Base.query_imported("imported_pages", site, query)|> where([i], i.visitors > 0)|> where([i],- fragment("notEmpty(multiMatchAllIndices(?, ?) as indices)", i.page, ^page_regexes)+ fragment(+ "notEmpty(multiMatchAllIndices(?, ?) as indices)",+ i.page,+ type(^page_regexes, {:array, :string})+ ))|> join(:array, index in fragment("indices"))|> group_by([_i, index], index)|> select_merge([_i, index], %{- name: fragment("concat('Visit ', ?[?])", ^page_exprs, index)+ dim0: selected_as(type(fragment("?", index), :integer), :dim0)})|> select_imported_metrics(metrics)+ end)+ |> Enum.reduce(q, fn imports_q, q ->+ naive_dimension_join(q, imports_q, metrics)+ end)+ end++ def merge_imported(q, site, %Query{dimensions: dimensions} = query, metrics) do+ if merge_imported_dimensions?(dimensions) do+ imported_q =+ site+ |> Imported.Base.query_imported(query)+ |> where([i], i.visitors > 0)+ |> group_imported_by(query)+ |> select_imported_metrics(metrics)- from(s in Ecto.Query.subquery(q),+ from(s in subquery(q),full_join: i in subquery(imported_q),- on: s.name == i.name,+ on: ^QueryBuilder.build_group_by_join(query),select: %{})- |> select_joined_dimension(:name, query)+ |> select_joined_dimensions(query)|> select_joined_metrics(metrics)elseqendend+ def merge_imported(q, _, _, _), do: q++ defp merge_imported_dimensions?(dimensions) do+ dimensions in [["visit:browser", "visit:browser_version"], ["visit:os", "visit:os_version"]] or+ (length(dimensions) == 1 and hd(dimensions) in @imported_dimensions)+ end+def total_imported_visitors(site, query) dosite|> Imported.Base.query_imported(query)|> select_merge([i], %{total_visitors: fragment("sum(?)", i.visitors)})end- # :TRICKY: Handle backwards compatibility with old breakdown module- defp shortname(query, _dim) when query.v2, do: :dim0- defp shortname(_query, dim), do: dim-defp select_imported_metrics(q, []), do: qdefp select_imported_metrics(q, [:visitors | rest]) do@@ -592,190 +552,175 @@ defmodule Plausible.Stats.Imported do|> select_imported_metrics(rest)end- defp group_imported_by(q, dim, query) when dim in [:source, :referrer] do+ defp group_imported_by(q, query) do+ Enum.reduce(query.dimensions, q, fn dimension, q ->+ dim = Plausible.Stats.Filters.without_prefix(dimension)++ group_imported_by(q, dim, shortname(query, dimension))+ end)+ end++ defp group_imported_by(q, dim, key) when dim in [:source, :referrer] doq|> group_by([i], field(i, ^dim))|> select_merge([i], %{- ^shortname(query, dim) =>- fragment(- "if(empty(?), ?, ?)",- field(i, ^dim),- @no_ref,- field(i, ^dim)+ ^key =>+ selected_as(+ fragment(+ "if(empty(?), ?, ?)",+ field(i, ^dim),+ @no_ref,+ field(i, ^dim)+ ),+ ^key)})end- defp group_imported_by(q, dim, query)+ defp group_imported_by(q, dim, key)when dim in [:utm_source, :utm_medium, :utm_campaign, :utm_term, :utm_content] doq|> group_by([i], field(i, ^dim))|> where([i], fragment("not empty(?)", field(i, ^dim)))- |> select_merge([i], %{^shortname(query, dim) => field(i, ^dim)})+ |> select_merge([i], %{^key => selected_as(field(i, ^dim), ^key)})end- defp group_imported_by(q, :page, query) do+ defp group_imported_by(q, :page, key) doq|> group_by([i], i.page)- |> select_merge([i], %{^shortname(query, :page) => i.page, time_on_page: sum(i.time_on_page)})+ |> select_merge([i], %{^key => selected_as(i.page, ^key), time_on_page: sum(i.time_on_page)})end- defp group_imported_by(q, :country, query) do+ defp group_imported_by(q, :country, key) doq|> group_by([i], i.country)|> where([i], i.country != "ZZ")- |> select_merge([i], %{^shortname(query, :country) => i.country})+ |> select_merge([i], %{^key => selected_as(i.country, ^key)})end- defp group_imported_by(q, :region, query) do+ defp group_imported_by(q, :region, key) doq|> group_by([i], i.region)|> where([i], i.region != "")- |> select_merge([i], %{^shortname(query, :region) => i.region})+ |> select_merge([i], %{^key => selected_as(i.region, ^key)})end- defp group_imported_by(q, :city, query) do+ defp group_imported_by(q, :city, key) doq|> group_by([i], i.city)|> where([i], i.city != 0 and not is_nil(i.city))- |> select_merge([i], %{^shortname(query, :city) => i.city})+ |> select_merge([i], %{^key => selected_as(i.city, ^key)})end- defp group_imported_by(q, dim, query) when dim in [:device, :browser] do+ defp group_imported_by(q, dim, key) when dim in [:device, :browser] doq|> group_by([i], field(i, ^dim))|> select_merge([i], %{- ^shortname(query, dim) =>- fragment("if(empty(?), ?, ?)", field(i, ^dim), @not_set, field(i, ^dim))+ ^key =>+ selected_as(+ fragment("if(empty(?), ?, ?)", field(i, ^dim), @not_set, field(i, ^dim)),+ ^key+ )})end- defp group_imported_by(q, :browser_version, query) do+ defp group_imported_by(q, :browser_version, key) doq- |> group_by([i], [i.browser, i.browser_version])+ |> group_by([i], [i.browser_version])|> select_merge([i], %{- ^shortname(query, :browser) =>- fragment("if(empty(?), ?, ?)", i.browser, @not_set, i.browser),- ^shortname(query, :browser_version) =>- fragment(- "if(empty(?), ?, ?)",- i.browser_version,- @not_set,- i.browser_version+ ^key =>+ selected_as(+ fragment(+ "if(empty(?), ?, ?)",+ i.browser_version,+ @not_set,+ i.browser_version+ ),+ ^key)})end- defp group_imported_by(q, :os, query) do+ defp group_imported_by(q, :os, key) doq|> group_by([i], i.operating_system)|> select_merge([i], %{- ^shortname(query, :os) =>- fragment("if(empty(?), ?, ?)", i.operating_system, @not_set, i.operating_system)+ ^key =>+ selected_as(+ fragment("if(empty(?), ?, ?)", i.operating_system, @not_set, i.operating_system),+ ^key+ )})end- defp group_imported_by(q, :os_version, query) do+ defp group_imported_by(q, :os_version, key) doq- |> group_by([i], [i.operating_system, i.operating_system_version])+ |> group_by([i], [i.operating_system_version])|> select_merge([i], %{- ^shortname(query, :os) =>- fragment("if(empty(?), ?, ?)", i.operating_system, @not_set, i.operating_system),- ^shortname(query, :os_version) =>- fragment(- "if(empty(?), ?, ?)",- i.operating_system_version,- @not_set,- i.operating_system_version+ ^key =>+ selected_as(+ fragment(+ "if(empty(?), ?, ?)",+ i.operating_system_version,+ @not_set,+ i.operating_system_version+ ),+ ^key)})end- defp group_imported_by(q, dim, query) when dim in [:entry_page, :exit_page] do+ defp group_imported_by(q, dim, key) when dim in [:entry_page, :exit_page] doq|> group_by([i], field(i, ^dim))- |> select_merge([i], %{^shortname(query, dim) => field(i, ^dim)})+ |> select_merge([i], %{^key => selected_as(field(i, ^dim), ^key)})end- defp group_imported_by(q, :name, query) do+ defp group_imported_by(q, :name, key) doq|> group_by([i], i.name)- |> select_merge([i], %{^shortname(query, :name) => i.name})- end-- defp group_imported_by(q, :url, query) when query.v2 do- q- |> group_by([i], i.link_url)- |> select_merge([i], %{- ^shortname(query, :url) => fragment("if(not empty(?), ?, ?)", i.link_url, i.link_url, @none)- })+ |> select_merge([i], %{^key => selected_as(i.name, ^key)})end- defp group_imported_by(q, :url, _query) do+ defp group_imported_by(q, :url, key) doq|> group_by([i], i.link_url)|> select_merge([i], %{- breakdown_prop_value: fragment("if(not empty(?), ?, ?)", i.link_url, i.link_url, @none)+ ^key => selected_as(fragment("if(not empty(?), ?, ?)", i.link_url, i.link_url, @none), ^key)})end- defp group_imported_by(q, :path, query) when query.v2 do+ defp group_imported_by(q, :path, key) doq|> group_by([i], i.path)|> select_merge([i], %{- ^shortname(query, :path) => fragment("if(not empty(?), ?, ?)", i.path, i.path, @none)+ ^key => selected_as(fragment("if(not empty(?), ?, ?)", i.path, i.path, @none), ^key)})end- defp group_imported_by(q, :path, _query) do- q- |> group_by([i], i.path)- |> select_merge([i], %{- breakdown_prop_value: fragment("if(not empty(?), ?, ?)", i.path, i.path, @none)- })- end-- defp select_joined_dimension(q, :city, query) do- select_merge(q, [s, i], %{- ^shortname(query, :city) => fragment("greatest(?,?)", i.city, s.city)- })- end-- defp select_joined_dimension(q, :os_version, query) when not query.v2 do- select_merge(q, [s, i], %{- os: fragment("if(empty(?), ?, ?)", s.os, i.os, s.os),- os_version: fragment("if(empty(?), ?, ?)", s.os_version, i.os_version, s.os_version)- })- end-- defp select_joined_dimension(q, :browser_version, query) when not query.v2 do- select_merge(q, [s, i], %{- browser: fragment("if(empty(?), ?, ?)", s.browser, i.browser, s.browser),- browser_version:- fragment("if(empty(?), ?, ?)", s.browser_version, i.browser_version, s.browser_version)- })+ defp select_joined_dimensions(q, query) do+ Enum.reduce(query.dimensions, q, fn dimension, q ->+ select_joined_dimension(q, dimension, shortname(query, dimension))+ end)end- defp select_joined_dimension(q, dim, query) when dim in [:url, :path] and not query.v2 do+ defp select_joined_dimension(q, "visit:city", key) doselect_merge(q, [s, i], %{- breakdown_prop_value:- fragment(- "if(empty(?), ?, ?)",- s.breakdown_prop_value,- i.breakdown_prop_value,- s.breakdown_prop_value- )+ ^key => selected_as(fragment("greatest(?,?)", field(i, ^key), field(s, ^key)), ^key)})end- defp select_joined_dimension(q, dim, query) do+ defp select_joined_dimension(q, _dimension, key) doselect_merge(q, [s, i], %{- ^shortname(query, dim) =>- fragment(- "if(empty(?), ?, ?)",- field(s, ^shortname(query, dim)),- field(i, ^shortname(query, dim)),- field(s, ^shortname(query, dim))+ ^key =>+ selected_as(+ fragment(+ "if(empty(?), ?, ?)",+ field(s, ^key),+ field(i, ^key),+ field(s, ^key)+ ),+ ^key)})end@@ -882,21 +827,12 @@ defmodule Plausible.Stats.Imported do|> select_joined_metrics(rest)end- defp apply_order_by(q, %Query{v2: true}, _), do: q-- defp apply_order_by(q, query, [:visitors | rest]) do- order_by(q, [s, i], desc: s.visitors + i.visitors)- |> apply_order_by(query, rest)- end-- defp apply_order_by(q, _query, _), do: q-defp naive_dimension_join(q1, q2, metrics) do- from(a in Ecto.Query.subquery(q1),+ from(a in subquery(q1),full_join: b in subquery(q2),on: a.dim0 == b.dim0,select: %{- dim0: fragment("coalesce(?, ?)", a.dim0, b.dim0)+ dim0: selected_as(fragment("if(? != 0, ?, ?)", a.dim0, a.dim0, b.dim0), :dim0)})|> select_joined_metrics(metrics)diff --git a/lib/plausible/stats/metrics.ex b/lib/plausible/stats/metrics.exindex 8ed71dd2e3..29319d07a4 100644--- a/lib/plausible/stats/metrics.ex+++ b/lib/plausible/stats/metrics.ex@@ -17,7 +17,8 @@ defmodule Plausible.Stats.Metrics do:events,:conversion_rate,:group_conversion_rate,- :time_on_page+ :time_on_page,+ :percentage] ++ on_ee(do: Plausible.Stats.Goal.Revenue.revenue_metrics(), else: [])@metric_mappings Enum.into(@all_metrics, %{}, fn metric -> {to_string(metric), metric} end)@@ -27,4 +28,8 @@ defmodule Plausible.Stats.Metrics dodef from_string!(str) doMap.fetch!(@metric_mappings, str)end++ def from_string(str) do+ Map.fetch(@metric_mappings, str)+ endenddiff --git a/lib/plausible/stats/query.ex b/lib/plausible/stats/query.exindex 86fc13220c..75fb8c503c 100644--- a/lib/plausible/stats/query.ex+++ b/lib/plausible/stats/query.ex@@ -18,7 +18,12 @@ defmodule Plausible.Stats.Query doorder_by: nil,timezone: nil,v2: false,- preloaded_goals: []+ legacy_breakdown: false,+ preloaded_goals: [],+ include: %{+ imports: false,+ time_labels: false+ }require OpenTelemetry.Tracer, as: Traceralias Plausible.Stats.{Filters, Interval, Imported}@@ -225,6 +230,12 @@ defmodule Plausible.Stats.Query dostruct!(query, filters: Filters.parse(params["filters"]))end+ def set(query, keywords) do+ query+ |> struct!(keywords)+ |> refresh_imported_opts()+ end+@spec set_dimensions(t(), list(String.t())) :: t()def set_dimensions(query, dimensions) doquerydiff --git a/lib/plausible/stats/query_optimizer.ex b/lib/plausible/stats/query_optimizer.exindex c1c38e5fb1..25146d950d 100644--- a/lib/plausible/stats/query_optimizer.ex+++ b/lib/plausible/stats/query_optimizer.ex@@ -3,6 +3,7 @@ defmodule Plausible.Stats.QueryOptimizer doMethods to manipulate Query for business logic reasons before building an ecto query."""+ use Plausiblealias Plausible.Stats.{Query, TableDecider, Util}@doc """@@ -148,8 +149,17 @@ defmodule Plausible.Stats.QueryOptimizer dodimension -> dimensionend)- query- |> Query.set_metrics(session_metrics)- |> Query.set_dimensions(dimensions)+ filters =+ if "event:page" in query.dimensions do+ query.filters+ |> Enum.map(fn+ [op, "event:page" | rest] -> [op, "visit:entry_page" | rest]+ filter -> filter+ end)+ else+ query.filters+ end++ Query.set(query, filters: filters, metrics: session_metrics, dimensions: dimensions)endenddiff --git a/lib/plausible/stats/sql/expression.ex b/lib/plausible/stats/sql/expression.exindex e74348daed..d3f38a1428 100644--- a/lib/plausible/stats/sql/expression.ex+++ b/lib/plausible/stats/sql/expression.ex@@ -11,84 +11,121 @@ defmodule Plausible.Stats.SQL.Expression do@no_ref "Direct / None"@not_set "(not set)"- defmacrop field_or_blank_value(expr, empty_value) do+ defmacrop field_or_blank_value(expr, empty_value, select_alias) doquote dodynamic([t],- fragment("if(empty(?), ?, ?)", unquote(expr), unquote(empty_value), unquote(expr))+ selected_as(+ fragment("if(empty(?), ?, ?)", unquote(expr), unquote(empty_value), unquote(expr)),+ ^unquote(select_alias)+ ))endend- def dimension("time:hour", query) do- dynamic([t], fragment("toStartOfHour(toTimeZone(?, ?))", t.timestamp, ^query.timezone))+ def dimension("time:hour", query, select_alias) do+ dynamic(+ [t],+ selected_as(+ fragment("toStartOfHour(toTimeZone(?, ?))", t.timestamp, ^query.timezone),+ ^select_alias+ )+ )end- def dimension("time:day", query) do- dynamic([t], fragment("toDate(toTimeZone(?, ?))", t.timestamp, ^query.timezone))+ def dimension("time:day", query, select_alias) do+ dynamic(+ [t],+ selected_as(+ fragment("toDate(toTimeZone(?, ?))", t.timestamp, ^query.timezone),+ ^select_alias+ )+ )end- def dimension("time:month", query) do- dynamic([t], fragment("toStartOfMonth(toTimeZone(?, ?))", t.timestamp, ^query.timezone))+ def dimension("time:month", query, select_alias) do+ dynamic(+ [t],+ selected_as(+ fragment("toStartOfMonth(toTimeZone(?, ?))", t.timestamp, ^query.timezone),+ ^select_alias+ )+ )end- def dimension("event:name", _query), do: dynamic([t], t.name)- def dimension("event:page", _query), do: dynamic([t], t.pathname)- def dimension("event:hostname", _query), do: dynamic([t], t.hostname)+ def dimension("event:name", _query, select_alias),+ do: dynamic([t], selected_as(t.name, ^select_alias))++ def dimension("event:page", _query, select_alias),+ do: dynamic([t], selected_as(t.pathname, ^select_alias))++ def dimension("event:hostname", _query, select_alias),+ do: dynamic([t], selected_as(t.hostname, ^select_alias))- def dimension("event:props:" <> property_name, _query) do+ def dimension("event:props:" <> property_name, _query, select_alias) dodynamic([t],- fragment(- "if(not empty(?), ?, '(none)')",- get_by_key(t, :meta, ^property_name),- get_by_key(t, :meta, ^property_name)+ selected_as(+ fragment(+ "if(not empty(?), ?, '(none)')",+ get_by_key(t, :meta, ^property_name),+ get_by_key(t, :meta, ^property_name)+ ),+ ^select_alias))end- def dimension("visit:entry_page", _query), do: dynamic([t], t.entry_page)- def dimension("visit:exit_page", _query), do: dynamic([t], t.exit_page)+ def dimension("visit:entry_page", _query, select_alias),+ do: dynamic([t], selected_as(t.entry_page, ^select_alias))++ def dimension("visit:exit_page", _query, select_alias),+ do: dynamic([t], selected_as(t.exit_page, ^select_alias))++ def dimension("visit:utm_medium", _query, select_alias),+ do: field_or_blank_value(t.utm_medium, @not_set, select_alias)++ def dimension("visit:utm_source", _query, select_alias),+ do: field_or_blank_value(t.utm_source, @not_set, select_alias)- def dimension("visit:utm_medium", _query),- do: field_or_blank_value(t.utm_medium, @not_set)+ def dimension("visit:utm_campaign", _query, select_alias),+ do: field_or_blank_value(t.utm_campaign, @not_set, select_alias)- def dimension("visit:utm_source", _query),- do: field_or_blank_value(t.utm_source, @not_set)+ def dimension("visit:utm_content", _query, select_alias),+ do: field_or_blank_value(t.utm_content, @not_set, select_alias)- def dimension("visit:utm_campaign", _query),- do: field_or_blank_value(t.utm_campaign, @not_set)+ def dimension("visit:utm_term", _query, select_alias),+ do: field_or_blank_value(t.utm_term, @not_set, select_alias)- def dimension("visit:utm_content", _query),- do: field_or_blank_value(t.utm_content, @not_set)+ def dimension("visit:source", _query, select_alias),+ do: field_or_blank_value(t.source, @no_ref, select_alias)- def dimension("visit:utm_term", _query),- do: field_or_blank_value(t.utm_term, @not_set)+ def dimension("visit:referrer", _query, select_alias),+ do: field_or_blank_value(t.referrer, @no_ref, select_alias)- def dimension("visit:source", _query),- do: field_or_blank_value(t.source, @no_ref)+ def dimension("visit:device", _query, select_alias),+ do: field_or_blank_value(t.device, @not_set, select_alias)- def dimension("visit:referrer", _query),- do: field_or_blank_value(t.referrer, @no_ref)+ def dimension("visit:os", _query, select_alias),+ do: field_or_blank_value(t.os, @not_set, select_alias)- def dimension("visit:device", _query),- do: field_or_blank_value(t.device, @not_set)+ def dimension("visit:os_version", _query, select_alias),+ do: field_or_blank_value(t.os_version, @not_set, select_alias)- def dimension("visit:os", _query), do: field_or_blank_value(t.os, @not_set)+ def dimension("visit:browser", _query, select_alias),+ do: field_or_blank_value(t.browser, @not_set, select_alias)- def dimension("visit:os_version", _query),- do: field_or_blank_value(t.os_version, @not_set)+ def dimension("visit:browser_version", _query, select_alias),+ do: field_or_blank_value(t.browser_version, @not_set, select_alias)- def dimension("visit:browser", _query),- do: field_or_blank_value(t.browser, @not_set)+ def dimension("visit:country", _query, select_alias),+ do: dynamic([t], selected_as(t.country, ^select_alias))- def dimension("visit:browser_version", _query),- do: field_or_blank_value(t.browser_version, @not_set)+ def dimension("visit:region", _query, select_alias),+ do: dynamic([t], selected_as(t.region, ^select_alias))- # :TODO: Locations also set extra filters- def dimension("visit:country", _query), do: dynamic([t], t.country)- def dimension("visit:region", _query), do: dynamic([t], t.region)- def dimension("visit:city", _query), do: dynamic([t], t.city)+ def dimension("visit:city", _query, select_alias),+ do: dynamic([t], selected_as(t.city, ^select_alias))defmacro event_goal_join(events, page_regexes) doquote dodiff --git a/lib/plausible/stats/sql/query_builder.ex b/lib/plausible/stats/sql/query_builder.exindex 40db95f28f..7e5e49a887 100644--- a/lib/plausible/stats/sql/query_builder.ex+++ b/lib/plausible/stats/sql/query_builder.ex@@ -7,7 +7,7 @@ defmodule Plausible.Stats.SQL.QueryBuilder doimport Plausible.Stats.Importedimport Plausible.Stats.Util- alias Plausible.Stats.{Base, Query, QueryOptimizer, TableDecider, Filters, Metrics}+ alias Plausible.Stats.{Base, Query, QueryOptimizer, TableDecider, Filters}alias Plausible.Stats.SQL.Expressionrequire Plausible.Stats.SQL.Expression@@ -44,6 +44,7 @@ defmodule Plausible.Stats.SQL.QueryBuilder do|> merge_imported(site, events_query, events_query.metrics)|> maybe_add_global_conversion_rate(site, events_query)|> maybe_add_group_conversion_rate(site, events_query)+ |> Base.add_percentage_metric(site, events_query, events_query.metrics)enddefp join_sessions_if_needed(q, site, query) do@@ -84,6 +85,9 @@ defmodule Plausible.Stats.SQL.QueryBuilder do|> join_events_if_needed(site, sessions_query)|> build_group_by(sessions_query)|> merge_imported(site, sessions_query, sessions_query.metrics)+ |> maybe_add_global_conversion_rate(site, sessions_query)+ |> maybe_add_group_conversion_rate(site, sessions_query)+ |> Base.add_percentage_metric(site, sessions_query, sessions_query.metrics)enddef join_events_if_needed(q, site, query) do@@ -123,36 +127,23 @@ defmodule Plausible.Stats.SQL.QueryBuilder do^shortname(query, dimension) => fragment("?", goal)},group_by: goal,- where: goal != 0+ where: goal != 0 and (e.name == "pageview" or goal < 0))enddefp dimension_group_by(q, query, dimension) do- q- |> select_merge(^%{shortname(query, dimension) => Expression.dimension(dimension, query)})- |> group_by(^Expression.dimension(dimension, query))- end+ key = shortname(query, dimension)- defp build_order_by(q, query, mode) do- Enum.reduce(query.order_by || [], q, &build_order_by(&2, query, &1, mode))+ q+ |> select_merge(^%{key => Expression.dimension(dimension, query, key)})+ |> group_by([], selected_as(^key))end- def build_order_by(q, query, {metric_or_dimension, order_direction}, :inner) do- order_by(- q,- [t],- ^{- order_direction,- if(- Metrics.metric?(metric_or_dimension),- do: dynamic([], selected_as(^shortname(query, metric_or_dimension))),- else: Expression.dimension(metric_or_dimension, query)- )- }- )+ defp build_order_by(q, query) do+ Enum.reduce(query.order_by || [], q, &build_order_by(&2, query, &1))end- def build_order_by(q, query, {metric_or_dimension, order_direction}, :outer) do+ def build_order_by(q, query, {metric_or_dimension, order_direction}) doorder_by(q,[t],@@ -262,10 +253,10 @@ defmodule Plausible.Stats.SQL.QueryBuilder dodefp join_query_results({nil, _}, {nil, _}), do: nildefp join_query_results({events_q, events_query}, {nil, _}),- do: events_q |> build_order_by(events_query, :inner)+ do: events_q |> build_order_by(events_query)- defp join_query_results({nil, _}, {sessions_q, sessions_query}),- do: sessions_q |> build_order_by(sessions_query, :inner)+ defp join_query_results({nil, events_query}, {sessions_q, _}),+ do: sessions_q |> build_order_by(events_query)defp join_query_results({events_q, events_query}, {sessions_q, sessions_query}) dojoin(subquery(events_q), :left, [e], s in subquery(sessions_q),@@ -274,12 +265,12 @@ defmodule Plausible.Stats.SQL.QueryBuilder do|> select_join_fields(events_query, events_query.dimensions, e)|> select_join_fields(events_query, events_query.metrics, e)|> select_join_fields(sessions_query, List.delete(sessions_query.metrics, :sample_percent), s)- |> build_order_by(events_query, :outer)+ |> build_order_by(events_query)end- defp build_group_by_join(%Query{dimensions: []}), do: true+ def build_group_by_join(%Query{dimensions: []}), do: true- defp build_group_by_join(query) do+ def build_group_by_join(query) doquery.dimensions|> Enum.map(fn dim ->dynamic([e, s], field(e, ^shortname(query, dim)) == field(s, ^shortname(query, dim)))diff --git a/lib/plausible/stats/table_decider.ex b/lib/plausible/stats/table_decider.exindex c0176d417f..7f3efcf25e 100644--- a/lib/plausible/stats/table_decider.ex+++ b/lib/plausible/stats/table_decider.ex@@ -29,34 +29,45 @@ defmodule Plausible.Stats.TableDecider do|> filter_keys()|> partition(query, &filters_partitioner/2)+ %{event: event_only_dimensions, session: session_only_dimensions} =+ partition(query.dimensions, query, &filters_partitioner/2)+cond do# Only one table needs to be queried- empty?(event_only_metrics) && empty?(event_only_filters) ->+ empty?(event_only_metrics) && empty?(event_only_filters) && empty?(event_only_dimensions) ->{[], session_only_metrics ++ either_metrics ++ sample_percent, other_metrics}- empty?(session_only_metrics) && empty?(session_only_filters) ->+ empty?(session_only_metrics) && empty?(session_only_filters) &&+ empty?(session_only_dimensions) ->{event_only_metrics ++ either_metrics ++ sample_percent, [], other_metrics}- # Filters on both events and sessions, but only one kind of metric- empty?(event_only_metrics) ->+ # Filters and/or dimensions on both events and sessions, but only one kind of metric+ empty?(event_only_metrics) && empty?(event_only_dimensions) ->{[], session_only_metrics ++ either_metrics ++ sample_percent, other_metrics}- empty?(session_only_metrics) ->+ empty?(session_only_metrics) && empty?(session_only_dimensions) ->{event_only_metrics ++ either_metrics ++ sample_percent, [], other_metrics}- # Default: prefer sessions+ # Default: prefer eventstrue ->- {event_only_metrics ++ sample_percent,- session_only_metrics ++ either_metrics ++ sample_percent, other_metrics}+ {event_only_metrics ++ either_metrics ++ sample_percent,+ session_only_metrics ++ sample_percent, other_metrics}endenddefp filter_keys(query) doquery.filters|> Enum.map(fn [_, filter_key | _rest] -> filter_key end)- |> Enum.concat(query.dimensions)end+ defp metric_partitioner(%Query{v2: true}, :conversion_rate), do: :either+ defp metric_partitioner(%Query{v2: true}, :group_conversion_rate), do: :either+ defp metric_partitioner(%Query{v2: true}, :visitors), do: :either+ defp metric_partitioner(%Query{v2: true}, :visits), do: :either+ # Note: This is inaccurate when filtering but required for old backwards compatibility+ defp metric_partitioner(%Query{legacy_breakdown: true}, :pageviews), do: :either+ defp metric_partitioner(%Query{legacy_breakdown: true}, :events), do: :either+defp metric_partitioner(_, :conversion_rate), do: :eventdefp metric_partitioner(_, :group_conversion_rate), do: :eventdefp metric_partitioner(_, :average_revenue), do: :event@@ -76,7 +87,7 @@ defmodule Plausible.Stats.TableDecider do# Calculated metrics - handled on callsite separately from other metrics.defp metric_partitioner(_, :time_on_page), do: :otherdefp metric_partitioner(_, :total_visitors), do: :other- defp metric_partitioner(_, :percentage), do: :other+ defp metric_partitioner(_, :percentage), do: :either# Sample percentage is included in both tables if queried.defp metric_partitioner(_, :sample_percent), do: :sample_percentdiff --git a/test/plausible/imported/csv_importer_test.exs b/test/plausible/imported/csv_importer_test.exsindex 18087bddf7..3bb6f8ca08 100644--- a/test/plausible/imported/csv_importer_test.exs+++ b/test/plausible/imported/csv_importer_test.exs@@ -622,7 +622,7 @@ defmodule Plausible.Imported.CSVImporterTest docase params_or_site do%Plausible.Site{} = site ->common_params.(site)- |> Map.put("metrics", "visitors,visits,pageviews,visit_duration,bounce_rate")+ |> Map.put("metrics", "visitors,visits,visit_duration,bounce_rate")|> Map.put("limit", 1000)|> Map.put("property", by)@@ -669,7 +669,7 @@ defmodule Plausible.Imported.CSVImporterTest doassert exported["pageviews"] == imported["pageviews"]assert exported["bounce_rate"] == imported["bounce_rate"]assert_in_delta exported["visitors"], imported["visitors"], 1- assert exported["visits"] == imported["visits"]+ assert_in_delta exported["visits"], imported["visits"], 1assert_in_delta exported["visit_duration"], imported["visit_duration"], 1end)@@ -810,7 +810,7 @@ defmodule Plausible.Imported.CSVImporterTest do_no_diff = 0endend)- ) == [0.0, 0.0, 0.0, 0.0, 0.03614457831325302]+ ) == [0.0, 0.0, 0.0, 0.0, 0.0]# NOTE: city breakdown's visit duration difference is up to 14%assert summary(field(exported_cities, "visit_duration")) == [0, 0.0, 0.0, 1.0, 1718]@@ -829,7 +829,7 @@ defmodule Plausible.Imported.CSVImporterTest do_no_diff = 0endend)- ) == [0, 0.0, 0.0, 0.0, 0.1428571428571429]+ ) == [0, 0.0, 0.0, 0.0, 0.0]# NOTE: city breakdown's visitors relative difference is up to 27%assert summary(field(exported_cities, "visitors")) == [1, 1.0, 1.0, 2.0, 22]diff --git a/test/plausible/imported/google_analytics4_test.exs b/test/plausible/imported/google_analytics4_test.exsindex 7253877c4c..728378787e 100644--- a/test/plausible/imported/google_analytics4_test.exs+++ b/test/plausible/imported/google_analytics4_test.exs@@ -117,7 +117,7 @@ defmodule Plausible.Imported.GoogleAnalytics4Test dobreakdown_params =common_params- |> Map.put("metrics", "visitors,visits,pageviews,visit_duration,bounce_rate")+ |> Map.put("metrics", "visitors,visits,visit_duration,bounce_rate")|> Map.put("limit", 1000)%{key: api_key} = insert(:api_key, user: user)@@ -467,7 +467,6 @@ defmodule Plausible.Imported.GoogleAnalytics4Test doassert List.first(results) == %{"bounce_rate" => 35.0,- "pageviews" => 6229,"visit_duration" => 40.0,"visitors" => 4671,"visits" => 4917,@@ -475,12 +474,11 @@ defmodule Plausible.Imported.GoogleAnalytics4Test do}assert List.last(results) == %{- "bounce_rate" => 100.0,- "pageviews" => 1,- "visit_duration" => 0.0,+ "bounce_rate" => 0.0,+ "source" => "yahoo",+ "visit_duration" => 41.0,"visitors" => 1,- "visits" => 1,- "source" => "petalsearch.com"+ "visits" => 1}end@@ -493,7 +491,6 @@ defmodule Plausible.Imported.GoogleAnalytics4Test doassert [%{"bounce_rate" => 35.0,- "pageviews" => 6399,"utm_medium" => "organic","visit_duration" => 40.0,"visitors" => 4787,@@ -501,7 +498,6 @@ defmodule Plausible.Imported.GoogleAnalytics4Test do},%{"bounce_rate" => 58.0,- "pageviews" => 491,"utm_medium" => "referral","visit_duration" => 27.0,"visitors" => 294,@@ -520,7 +516,6 @@ defmodule Plausible.Imported.GoogleAnalytics4Test doassert List.first(results) == %{"bounce_rate" => 35.0,- "pageviews" => 838,"visit_duration" => 43.0,"visitors" => 675,"visits" => 712,@@ -529,11 +524,10 @@ defmodule Plausible.Imported.GoogleAnalytics4Test doassert List.last(results) == %{"bounce_rate" => 0.0,- "pageviews" => 1,- "visit_duration" => 27.0,+ "entry_page" => "/znamenitosti-rima-koje-treba-vidjeti",+ "visit_duration" => 40.0,"visitors" => 1,- "visits" => 1,- "entry_page" => "/kad-lisce-pada"+ "visits" => 1}end@@ -543,12 +537,11 @@ defmodule Plausible.Imported.GoogleAnalytics4Test do%{"results" => results} =get(conn, "/api/v1/stats/breakdown", params) |> json_response(200)- assert length(results) == 488+ assert length(results) == 494assert List.first(results) == %{"bounce_rate" => 35.0,"city" => 792_680,- "pageviews" => 1650,"visit_duration" => 39.0,"visitors" => 1233,"visits" => 1273@@ -556,9 +549,8 @@ defmodule Plausible.Imported.GoogleAnalytics4Test doassert List.last(results) == %{"bounce_rate" => 0.0,- "city" => 4_399_605,- "pageviews" => 7,- "visit_duration" => 128.0,+ "city" => 11_951_298,+ "visit_duration" => 271.0,"visitors" => 1,"visits" => 1}@@ -574,7 +566,6 @@ defmodule Plausible.Imported.GoogleAnalytics4Test doassert List.first(results) == %{"bounce_rate" => 38.0,- "pageviews" => 7041,"visit_duration" => 37.0,"visitors" => 5277,"visits" => 5532,@@ -583,7 +574,6 @@ defmodule Plausible.Imported.GoogleAnalytics4Test doassert List.last(results) == %{"bounce_rate" => 37.0,- "pageviews" => 143,"visit_duration" => 60.0,"visitors" => 97,"visits" => 100,@@ -601,7 +591,6 @@ defmodule Plausible.Imported.GoogleAnalytics4Test doassert List.first(results) == %{"bounce_rate" => 33.0,- "pageviews" => 8143,"visit_duration" => 50.0,"visitors" => 4625,"visits" => 4655,@@ -610,7 +599,6 @@ defmodule Plausible.Imported.GoogleAnalytics4Test doassert List.last(results) == %{"bounce_rate" => 0.0,- "pageviews" => 6,"visit_duration" => 0.0,"visitors" => 1,"visits" => 1,@@ -628,7 +616,6 @@ defmodule Plausible.Imported.GoogleAnalytics4Test doassert List.first(results) == %{"bounce_rate" => 34.0,- "pageviews" => 5827,"visit_duration" => 41.0,"visitors" => 4319,"visits" => 4495,@@ -637,7 +624,6 @@ defmodule Plausible.Imported.GoogleAnalytics4Test doassert List.last(results) == %{"bounce_rate" => 0.0,- "pageviews" => 6,"visit_duration" => 0.0,"visitors" => 1,"visits" => 1,@@ -657,7 +643,6 @@ defmodule Plausible.Imported.GoogleAnalytics4Test do"bounce_rate" => 32.0,"os" => "Android","os_version" => "13.0.0",- "pageviews" => 1673,"visit_duration" => 42.0,"visitors" => 1247,"visits" => 1295@@ -665,17 +650,16 @@ defmodule Plausible.Imported.GoogleAnalytics4Test doassert List.last(results) == %{"bounce_rate" => 0.0,- "os" => "iOS",- "os_version" => "15.1",- "pageviews" => 1,- "visit_duration" => 54.0,+ "os" => "Chrome OS",+ "os_version" => "x86_64 15662.76.0",+ "visit_duration" => 16.0,"visitors" => 1,"visits" => 1}enddefp assert_pages(conn, params) do- metrics = "visitors,visits,pageviews,time_on_page,visit_duration,bounce_rate"+ metrics = "visitors,visits,time_on_page,visit_duration,bounce_rate"params =params@@ -686,7 +670,7 @@ defmodule Plausible.Imported.GoogleAnalytics4Test do%{"results" => results} =get(conn, "/api/v1/stats/breakdown", params) |> json_response(200)- assert length(results) == 729+ assert length(results) == 730# The `event:page` breakdown is currently using the `entry_page`# property to allow querying session metrics.@@ -696,7 +680,6 @@ defmodule Plausible.Imported.GoogleAnalytics4Test do# it will allow us to assert on the session metrics as well.assert Enum.at(results, 2) == %{"page" => "/",- "pageviews" => 5537,"time_on_page" => 17.677262055264585,"visitors" => 371,"visits" => 212,@@ -707,13 +690,12 @@ defmodule Plausible.Imported.GoogleAnalytics4Test do# This page was never an entry_page in the imported data, and# therefore the session metrics are returned as `nil`.assert List.last(results) == %{- "page" => "/5-dobrih-razloga-zasto-zapoceti-dan-zobenom-kasom/",- "pageviews" => 2,- "time_on_page" => 10.0,+ "bounce_rate" => 0.0,+ "page" => "/znamenitosti-rima-koje-treba-vidjeti/",+ "time_on_page" => 40.0,+ "visit_duration" => 0.0,"visitors" => 1,- "visits" => 1,- "bounce_rate" => nil,- "visit_duration" => nil+ "visits" => 1}endenddiff --git a/test/plausible/stats/table_decider_test.exs b/test/plausible/stats/table_decider_test.exsindex 7226542c17..b216d02c7d 100644--- a/test/plausible/stats/table_decider_test.exs+++ b/test/plausible/stats/table_decider_test.exs@@ -76,7 +76,7 @@ defmodule Plausible.Stats.TableDeciderTest doquery = make_query(false, [])assert partition_metrics([:time_on_page, :percentage, :total_visitors], query) ==- {[], [], [:time_on_page, :percentage, :total_visitors]}+ {[], [:percentage], [:time_on_page, :total_visitors]}endtest "raises if unknown metric" do@@ -108,11 +108,11 @@ defmodule Plausible.Stats.TableDeciderTest do{[], [:visit_duration, :visits], []}end- test "metrics that can be calculated on either are biased to sessions" do+ test "metrics that can be calculated on either are biased to events" doquery = make_query(true, [])assert partition_metrics([:bounce_rate, :total_revenue, :visitors], query) ==- {[:total_revenue], [:bounce_rate, :visitors], []}+ {[:total_revenue, :visitors], [:bounce_rate], []}endtest "sample_percent is handled with either metrics" dodiff --git a/test/plausible_web/controllers/CSVs/30d-filter-path/pages.csv b/test/plausible_web/controllers/CSVs/30d-filter-path/pages.csvindex 1cef2d8c42..4741db70e8 100644--- a/test/plausible_web/controllers/CSVs/30d-filter-path/pages.csv+++ b/test/plausible_web/controllers/CSVs/30d-filter-path/pages.csv@@ -1,2 +1,2 @@name,visitors,pageviews,bounce_rate,time_on_page-/some-other-page,1,1,,60.0+/some-other-page,1,1,0,60.0diff --git a/test/plausible_web/controllers/CSVs/30d/pages.csv b/test/plausible_web/controllers/CSVs/30d/pages.csvindex e2990caa97..62b246c420 100644--- a/test/plausible_web/controllers/CSVs/30d/pages.csv+++ b/test/plausible_web/controllers/CSVs/30d/pages.csv@@ -1,4 +1,4 @@name,visitors,pageviews,bounce_rate,time_on_page/,4,3,67,/signup,1,1,0,60.0-/some-other-page,1,1,,60.0+/some-other-page,1,1,0,60.0diff --git a/test/plausible_web/controllers/CSVs/6m/pages.csv b/test/plausible_web/controllers/CSVs/6m/pages.csvindex 8051585445..ed895e036f 100644--- a/test/plausible_web/controllers/CSVs/6m/pages.csv+++ b/test/plausible_web/controllers/CSVs/6m/pages.csv@@ -1,4 +1,4 @@name,visitors,pageviews,bounce_rate,time_on_page/,5,4,75,/signup,1,1,0,60.0-/some-other-page,1,1,,60.0+/some-other-page,1,1,0,60.0diff --git a/test/plausible_web/controllers/api/external_stats_controller/breakdown_test.exs b/test/plausible_web/controllers/api/external_stats_controller/breakdown_test.exsindex 8be42e3187..d6b9a305b6 100644--- a/test/plausible_web/controllers/api/external_stats_controller/breakdown_test.exs+++ b/test/plausible_web/controllers/api/external_stats_controller/breakdown_test.exs@@ -890,8 +890,8 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest doassert json_response(conn, 200) == %{"results" => [%{"page" => "/", "pageviews" => 2},- %{"page" => "/plausible.io", "pageviews" => 1},- %{"page" => "/include-me", "pageviews" => 1}+ %{"page" => "/include-me", "pageviews" => 1},+ %{"page" => "/plausible.io", "pageviews" => 1}]}end@@ -1023,7 +1023,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do"period" => "day","date" => "2021-01-01","property" => "visit:exit_page",- "metrics" => "visitors,visits,pageviews,bounce_rate,visit_duration,events",+ "metrics" => "visitors,visits,bounce_rate,visit_duration,events,pageviews","with_imported" => "true"})@@ -1031,21 +1031,21 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do"results" => [%{"bounce_rate" => 0.0,- "events" => 7,"exit_page" => "/b",- "pageviews" => 7,"visit_duration" => 150.0,"visitors" => 3,- "visits" => 4+ "visits" => 4,+ "events" => 7,+ "pageviews" => 7},%{"bounce_rate" => 100.0,- "events" => 1,"exit_page" => "/a",- "pageviews" => 1,"visit_duration" => 0.0,"visitors" => 1,- "visits" => 1+ "visits" => 1,+ "events" => 1,+ "pageviews" => 1}]}@@ -2176,8 +2176,8 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest doassert json_response(conn, 200) == %{"results" => [- %{"page" => "/plausible.io", "bounce_rate" => 100},- %{"page" => "/important-page", "bounce_rate" => 100}+ %{"page" => "/important-page", "bounce_rate" => 100},+ %{"page" => "/plausible.io", "bounce_rate" => 100}]}end@@ -2596,14 +2596,14 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest doassert json_response(conn, 200) == %{"results" => [- %{- "page" => "/B",- "time_on_page" => 90.0- },%{"page" => "/A","time_on_page" => 60.0},+ %{+ "page" => "/B",+ "time_on_page" => 90.0+ },%{"page" => "/C","time_on_page" => nil@@ -3045,13 +3045,13 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest doassert json_response(conn, 200) == %{"results" => [- %{- "entry_page" => "/entry-page-1",- "bounce_rate" => 0- },%{"entry_page" => "/entry-page-2","bounce_rate" => 100+ },+ %{+ "entry_page" => "/entry-page-1",+ "bounce_rate" => 0}]}@@ -3146,6 +3146,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do},%{"page" => "/plausible.io",+ # Breaks for event:page breakdown since visitors is calculated based on entry_page :/"visitors" => 2,"bounce_rate" => 100,"visit_duration" => 0,@@ -3290,8 +3291,6 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest doassert %{"browser" => "Chrome", "events" => 1} = breakdown_and_first.("visit:browser")assert %{"device" => "Desktop", "events" => 1} = breakdown_and_first.("visit:device")- assert %{"entry_page" => "/test", "events" => 1} = breakdown_and_first.("visit:entry_page")- assert %{"exit_page" => "/test", "events" => 1} = breakdown_and_first.("visit:exit_page")assert %{"country" => "EE", "events" => 1} = breakdown_and_first.("visit:country")assert %{"os" => "Mac", "events" => 1} = breakdown_and_first.("visit:os")assert %{"page" => "/test", "events" => 1} = breakdown_and_first.("event:page")diff --git a/test/plausible_web/controllers/api/external_stats_controller/query_test.exs b/test/plausible_web/controllers/api/external_stats_controller/query_test.exsindex cc4aec1ba6..331687e1f0 100644--- a/test/plausible_web/controllers/api/external_stats_controller/query_test.exs+++ b/test/plausible_web/controllers/api/external_stats_controller/query_test.exs@@ -1477,7 +1477,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest doconn =post(conn, "/api/v2/query", %{"site_id" => site.domain,- "metrics" => ["visitors"],+ "metrics" => ["visitors", "percentage"],"date_range" => ["2021-01-01", "2021-01-01"],"dimensions" => [unquote(dimension)]})@@ -1485,9 +1485,9 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest do%{"results" => results} = json_response(conn, 200)assert results == [- %{"dimensions" => [unquote(value1)], "metrics" => [3]},- %{"dimensions" => [unquote(value2)], "metrics" => [2]},- %{"dimensions" => [unquote(blank_value)], "metrics" => [1]}+ %{"dimensions" => [unquote(value1)], "metrics" => [3, 50]},+ %{"dimensions" => [unquote(value2)], "metrics" => [2, 33.3]},+ %{"dimensions" => [unquote(blank_value)], "metrics" => [1, 16.7]}]endend@@ -3463,6 +3463,48 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest do%{"dimensions" => ["Chrome"], "metrics" => [1]}]end++ test "all metrics for breakdown by event prop", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:pageview,+ user_id: 1,+ pathname: "/",+ timestamp: ~N[2021-01-01 00:00:00]+ ),+ build(:pageview,+ user_id: 1,+ pathname: "/plausible.io",+ timestamp: ~N[2021-01-01 00:10:00]+ ),+ build(:pageview, pathname: "/", timestamp: ~N[2021-01-01 00:25:00]),+ build(:pageview,+ pathname: "/plausible.io",+ timestamp: ~N[2021-01-01 00:00:00]+ )+ ])++ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "metrics" => [+ "visitors",+ "visits",+ "pageviews",+ "events",+ "bounce_rate",+ "visit_duration"+ ],+ "date_range" => "all",+ "dimensions" => ["event:page"]+ })++ %{"results" => results} = json_response(conn, 200)++ assert results == [+ %{"dimensions" => ["/"], "metrics" => [2, 2, 2, 2, 50, 300]},+ %{"dimensions" => ["/plausible.io"], "metrics" => [2, 2, 2, 2, 100, 0]}+ ]+ endenddescribe "imported data" do@@ -3589,10 +3631,6 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest doassert %{"dimensions" => ["Desktop"], "metrics" => [1]} =breakdown_and_first.("visit:device")- # :TODO: These should not pass validation - not available on events.- # visit dimension and event-only metric- # assert %{"dimensions" => ["/test"], "metrics" => [1]} = breakdown_and_first.("visit:entry_page")- # assert %{"dimensions" => ["/test"], "metrics" => [1]} = breakdown_and_first.("visit:exit_page")assert %{"dimensions" => ["EE"], "metrics" => [1]} = breakdown_and_first.("visit:country")assert %{"dimensions" => ["Mac"], "metrics" => [1]} = breakdown_and_first.("visit:os")assert %{"dimensions" => ["/test"], "metrics" => [1]} = breakdown_and_first.("event:page")@@ -3865,6 +3903,56 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest doassert meta["warning"] =~ "Imported stats are not included in the results"end++ test "imported country, region and city data",+ %{+ conn: conn,+ site: site+ } do+ site_import = insert(:site_import, site: site)++ populate_stats(site, site_import.id, [+ build(:pageview,+ timestamp: ~N[2021-01-01 00:15:00],+ country_code: "DE",+ subdivision1_code: "DE-BE",+ city_geoname_id: 2_950_159+ ),+ build(:pageview,+ timestamp: ~N[2021-01-01 00:15:00],+ country_code: "DE",+ subdivision1_code: "DE-BE",+ city_geoname_id: 2_950_159+ ),+ build(:pageview,+ timestamp: ~N[2021-01-01 00:15:00],+ country_code: "EE",+ subdivision1_code: "EE-37",+ city_geoname_id: 588_409+ ),+ build(:imported_locations, country: "EE", region: "EE-37", city: 588_409, visitors: 33)+ ])++ for {dimension, stats_value, imports_value} <- [+ {"visit:country", "DE", "EE"},+ {"visit:region", "DE-BE", "EE-37"},+ {"visit:city", 2_950_159, 588_409}+ ] do+ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "metrics" => ["visitors"],+ "date_range" => "all",+ "dimensions" => [dimension],+ "include" => %{"imports" => true}+ })++ assert json_response(conn, 200)["results"] == [+ %{"dimensions" => [imports_value], "metrics" => [34]},+ %{"dimensions" => [stats_value], "metrics" => [2]}+ ]+ end+ endendtest "multiple breakdown timeseries with sources", %{conn: conn, site: site} dodiff --git a/test/plausible_web/controllers/api/stats_controller/conversions_test.exs b/test/plausible_web/controllers/api/stats_controller/conversions_test.exsindex 24e18ed2da..a5bcdb38b2 100644--- a/test/plausible_web/controllers/api/stats_controller/conversions_test.exs+++ b/test/plausible_web/controllers/api/stats_controller/conversions_test.exs@@ -207,6 +207,7 @@ defmodule PlausibleWeb.Api.StatsController.ConversionsTest do]end+ @tag capture_log: truetest "garbage filters don't crash the call", %{conn: conn, site: site} dofilters ="{\"source\":\"Direct / None\",\"screen\":\"Desktop\",\"browser\":\"Chrome\",\"os\":\"Mac\",\"os_version\":\"10.15\",\"country\":\"DE\",\"city\":\"2950159\"}%' AND 2*3*8=6*8 AND 'L9sv'!='L9sv%"diff --git a/test/plausible_web/controllers/api/stats_controller/imported_test.exs b/test/plausible_web/controllers/api/stats_controller/imported_test.exsindex 370eb0ecf6..16e7a35f92 100644--- a/test/plausible_web/controllers/api/stats_controller/imported_test.exs+++ b/test/plausible_web/controllers/api/stats_controller/imported_test.exs@@ -602,15 +602,15 @@ defmodule PlausibleWeb.Api.StatsController.ImportedTest doassert json_response(conn, 200)["results"] == [%{- "name" => "blog",+ "name" => "ad","visitors" => 2,- "bounce_rate" => 50.0,+ "bounce_rate" => 100.0,"visit_duration" => 50.0},%{- "name" => "ad",+ "name" => "blog","visitors" => 2,- "bounce_rate" => 100.0,+ "bounce_rate" => 50.0,"visit_duration" => 50.0}]@@ -708,7 +708,7 @@ defmodule PlausibleWeb.Api.StatsController.ImportedTest doassert json_response(conn, 200)["results"] == [%{- "bounce_rate" => nil,+ "bounce_rate" => 0,"time_on_page" => 60,"visitors" => 3,"pageviews" => 4,diff --git a/test/plausible_web/controllers/api/stats_controller/pages_test.exs b/test/plausible_web/controllers/api/stats_controller/pages_test.exsindex 4af5c5cd0b..6a55d0bf84 100644--- a/test/plausible_web/controllers/api/stats_controller/pages_test.exs+++ b/test/plausible_web/controllers/api/stats_controller/pages_test.exs@@ -340,7 +340,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do"name" => "/blog/other-post","visitors" => 1,"pageviews" => 1,- "bounce_rate" => nil,+ "bounce_rate" => 0,"time_on_page" => nil}]@@ -392,7 +392,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do"name" => "/blog/other-post","visitors" => 1,"pageviews" => 1,- "bounce_rate" => nil,+ "bounce_rate" => 0,"time_on_page" => nil}]@@ -744,7 +744,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do"name" => "/blog/post-2","visitors" => 1,"pageviews" => 1,- "bounce_rate" => nil,+ "bounce_rate" => 0,"time_on_page" => nil}]@@ -789,7 +789,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do"name" => "/blog/(/post-2","visitors" => 1,"pageviews" => 1,- "bounce_rate" => nil,+ "bounce_rate" => 0,"time_on_page" => nil}]@@ -842,7 +842,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do"name" => "/about","visitors" => 1,"pageviews" => 1,- "bounce_rate" => nil,+ "bounce_rate" => 0,"time_on_page" => nil}]@@ -940,7 +940,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do"name" => "/"},%{- "bounce_rate" => nil,+ "bounce_rate" => 0,"time_on_page" => nil,"visitors" => 1,"pageviews" => 1,@@ -1066,7 +1066,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do"visitors" => 2},%{- "bounce_rate" => nil,+ "bounce_rate" => 0,"name" => "/exit-blog","pageviews" => 1,"time_on_page" => nil,@@ -1192,7 +1192,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do"name" => "/"},%{- "bounce_rate" => nil,+ "bounce_rate" => 0,"time_on_page" => 60,"visitors" => 2,"pageviews" => 2,diff --git a/test/plausible_web/controllers/api/stats_controller/sources_test.exs b/test/plausible_web/controllers/api/stats_controller/sources_test.exsindex a76b1da274..ea7b9c15a2 100644--- a/test/plausible_web/controllers/api/stats_controller/sources_test.exs+++ b/test/plausible_web/controllers/api/stats_controller/sources_test.exs@@ -453,6 +453,9 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest doreferrer_source: "DuckDuckGo",referrer: "duckduckgo.com"),+ build(:imported_sources,+ source: "DuckDuckGo"+ ),build(:imported_sources,source: "DuckDuckGo")@@ -467,7 +470,7 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest doconn = get(conn, "/api/stats/#{site.domain}/sources?limit=1&page=2&with_imported=true")assert json_response(conn, 200)["results"] == [- %{"name" => "DuckDuckGo", "visitors" => 2}+ %{"name" => "Google", "visitors" => 2}]end@@ -590,17 +593,17 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do)assert json_response(conn, 200)["results"] == [- %{- "name" => "social",- "visitors" => 1,- "bounce_rate" => 0,- "visit_duration" => 900- },%{"name" => "email","visitors" => 1,"bounce_rate" => 100,"visit_duration" => 0+ },+ %{+ "name" => "social",+ "visitors" => 1,+ "bounce_rate" => 0,+ "visit_duration" => 900}]@@ -612,16 +615,16 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest doassert json_response(conn, 200)["results"] == [%{- "name" => "social",+ "name" => "email","visitors" => 2,"bounce_rate" => 50,- "visit_duration" => 800.0+ "visit_duration" => 50},%{- "name" => "email",+ "name" => "social","visitors" => 2,"bounce_rate" => 50,- "visit_duration" => 50+ "visit_duration" => 800.0}]end