Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.
TypeScript analyticsanalytics-dashboardclickhouseelixirgoogle-analyticsmarketingopen-source-analyticsphoenixplausible-analyticspostgresqlprivacyprivacy-friendlyself-hostedsimple-analyticsstatisticstailwindcssweb-analyticswebsite-analyticswebsite-statswebsite-tracking
Revert "APIv2: Replace breakdown module with QueryBuilder (#4283)" (#4292)
Karl-Aksel Puulmannef5e0e03823d
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 75f271eeb9..065196ce6b 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -16,7 +16,6 @@ All notable changes to this project will be documented in this file.- Increase hourly request limit for API keys in CE from 600 to 1000000 (practically removing the limit) plausible/analytics#4200- Make TCP connections try IPv6 first with IPv4 fallback in CE plausible/analytics#4245- `is` and `is not` filters in dashboard no longer support wildcards. Use contains/does not contain filter instead.-- `bounce_rate` metric now returns 0 instead of null for event:page breakdown when page has never been entry page.### Fixeddiff --git a/lib/plausible/stats/base.ex b/lib/plausible/stats/base.exindex 5354833a1f..3b3dfd2e33 100644--- a/lib/plausible/stats/base.ex+++ b/lib/plausible/stats/base.ex@@ -250,8 +250,6 @@ defmodule Plausible.Stats.Base doenddefp select_session_metric(:percentage, _query), do: %{}- defp select_session_metric(:conversion_rate, _query), do: %{}- defp select_session_metric(:group_conversion_rate, _query), do: %{}def filter_converted_sessions(db_query, site, query) doif Query.has_event_filters?(query) dodiff --git a/lib/plausible/stats/breakdown.ex b/lib/plausible/stats/breakdown.exindex 5c6f8a6aa2..d68e9fcad0 100644--- a/lib/plausible/stats/breakdown.ex+++ b/lib/plausible/stats/breakdown.ex@@ -3,61 +3,344 @@ defmodule Plausible.Stats.Breakdown douse Plausibleuse Plausible.Stats.Fragments- import Plausible.Stats.Base+ import Plausible.Stats.{Base, Imported}import Ecto.Query- alias Plausible.Stats.{Query, QueryOptimizer, QueryResult, SQL}- alias Plausible.Stats.Filters.QueryParser-- def breakdown(site, %Query{dimensions: [dimension]} = query, metrics, pagination, _opts \\ []) do- transformed_metrics = transform_metrics(metrics, dimension)-- query_with_metrics =- Query.set(- query,- metrics: transformed_metrics,- order_by: infer_order_by(transformed_metrics, dimension),- dimensions: transform_dimensions(dimension),- filters: query.filters ++ dimension_filters(dimension),- preloaded_goals: QueryParser.preload_goals_if_needed(site, query.filters, [dimension]),- v2: true+ alias Plausible.Stats.{Query, Util, TableDecider}++ @no_ref "Direct / None"+ @not_set "(not set)"++ @session_metrics [:bounce_rate, :visit_duration]++ @revenue_metrics on_ee(do: Plausible.Stats.Goal.Revenue.revenue_metrics(), else: [])++ @event_metrics [:visits, :visitors, :pageviews, :events, :percentage] ++ @revenue_metrics++ # These metrics can be asked from the `breakdown/5` function,+ # but they are different from regular metrics such as `visitors`,+ # or `bounce_rate` - we cannot currently "select them" directly in+ # the db queries. Instead, we need to artificially append them to+ # the breakdown results later on.+ @computed_metrics [:conversion_rate, :total_visitors]++ def breakdown(site, query, metrics, pagination, opts \\ [])++ def breakdown(+ site,+ %Query{dimensions: ["event:goal"]} = query,+ metrics,+ pagination,+ opts+ ) do+ site = Plausible.Repo.preload(site, :goals)++ {event_goals, pageview_goals} = Enum.split_with(site.goals, & &1.event_name)+ events = Enum.map(event_goals, & &1.event_name)++ event_query =+ query+ |> Query.put_filter([:is, "event:name", events])+ |> Query.set_dimensions(["event:name"])++ if !Keyword.get(opts, :skip_tracing), do: Query.trace(query, metrics)++ no_revenue = {nil, metrics -- @revenue_metrics}++ {revenue_goals, metrics} =+ on_ee do+ if Plausible.Billing.Feature.RevenueGoals.enabled?(site) do+ revenue_goals = Enum.filter(event_goals, &Plausible.Goal.Revenue.revenue?/1)+ metrics = if Enum.empty?(revenue_goals), do: metrics -- @revenue_metrics, else: metrics++ {revenue_goals, metrics}+ else+ no_revenue+ end+ else+ no_revenue+ end++ metrics_to_select = Util.maybe_add_visitors_metric(metrics) -- @computed_metrics++ event_q =+ if Enum.any?(event_goals) do+ site+ |> breakdown_events(event_query, metrics_to_select)+ |> apply_pagination(pagination)+ else+ nil+ end++ page_q =+ if Enum.any?(pageview_goals) do+ page_query = Query.set_dimensions(query, ["event:page"])++ page_exprs = Enum.map(pageview_goals, & &1.page_path)+ page_regexes = Enum.map(page_exprs, &page_regex/1)++ select_columns = metrics_to_select |> select_event_metrics |> mark_revenue_as_nil++ from(e in base_event_query(site, page_query),+ order_by: [desc: fragment("uniq(?)", e.user_id)],+ where:+ fragment(+ "notEmpty(multiMatchAllIndices(?, ?) as indices)",+ e.pathname,+ ^page_regexes+ ) and e.name == "pageview",+ array_join: index in fragment("indices"),+ group_by: index,+ select: %{+ name: fragment("concat('Visit ', ?[?])", ^page_exprs, index)+ }+ )+ |> select_merge(^select_columns)+ |> merge_imported_pageview_goals(site, page_query, page_exprs, metrics_to_select)+ |> apply_pagination(pagination)+ else+ nil+ end++ full_q =+ case {event_q, page_q} do+ {nil, nil} ->+ nil++ {event_q, nil} ->+ event_q++ {nil, page_q} ->+ page_q++ {event_q, page_q} ->+ from(+ e in subquery(union_all(event_q, ^page_q)),+ order_by: [desc: e.visitors]+ )+ |> apply_pagination(pagination)+ end++ if full_q do+ full_q+ |> maybe_add_conversion_rate(site, query, metrics)+ |> ClickhouseRepo.all()+ |> transform_keys(%{name: :goal})+ |> cast_revenue_metrics_to_money(revenue_goals)+ |> Util.keep_requested_metrics(metrics)+ else+ []+ end+ end++ def breakdown(+ site,+ %Query{dimensions: ["event:props:" <> custom_prop]} = query,+ metrics,+ pagination,+ opts+ ) do+ {currency, metrics} =+ on_ee do+ Plausible.Stats.Goal.Revenue.get_revenue_tracking_currency(site, query, metrics)+ else+ {nil, metrics}+ end++ metrics_to_select = Util.maybe_add_visitors_metric(metrics) -- @computed_metrics++ if !Keyword.get(opts, :skip_tracing), do: Query.trace(query, metrics)++ breakdown_events(site, query, metrics_to_select)+ |> maybe_add_conversion_rate(site, query, metrics)+ |> paginate_and_execute(metrics, pagination)+ |> transform_keys(%{breakdown_prop_value: custom_prop})+ |> Enum.map(&cast_revenue_metrics_to_money(&1, currency))+ end++ def breakdown(site, %Query{dimensions: ["event:page"]} = query, metrics, pagination, opts) do+ event_metrics =+ metrics+ |> Util.maybe_add_visitors_metric()+ |> Enum.filter(&(&1 in @event_metrics))++ if !Keyword.get(opts, :skip_tracing), do: Query.trace(query, metrics)++ event_result =+ site+ |> breakdown_events(query, event_metrics)+ |> maybe_add_group_conversion_rate(&breakdown_events/3, site, query, metrics)+ |> paginate_and_execute(metrics, pagination)+ |> maybe_add_time_on_page(site, query, metrics)++ session_metrics = Enum.filter(metrics, &(&1 in @session_metrics))++ entry_page_query =+ case event_result do+ [] ->+ query++ pages ->+ query+ |> Query.remove_filters(["event:page"])+ |> Query.put_filter([:is, "visit:entry_page", Enum.map(pages, & &1[:page])])+ |> Query.set_dimensions(["visit:entry_page"])+ end++ if Enum.any?(event_metrics) && Enum.empty?(event_result) do+ []+ else+ {limit, _page} = pagination++ session_result =+ breakdown_sessions(site, entry_page_query, session_metrics)+ |> paginate_and_execute(session_metrics, {limit, 1})+ |> transform_keys(%{entry_page: :page})++ metrics = metrics ++ [:page]++ zip_results(+ event_result,+ session_result,+ :page,+ metrics)- |> QueryOptimizer.optimize()+ |> Enum.map(&Map.take(&1, metrics))+ end+ end- q = SQL.QueryBuilder.build(query_with_metrics, site)+ def breakdown(site, %Query{dimensions: ["event:name"]} = query, metrics, pagination, opts) do+ if !Keyword.get(opts, :skip_tracing), do: Query.trace(query, metrics)- q- |> apply_pagination(pagination)- |> ClickhouseRepo.all()- |> QueryResult.from(query_with_metrics)- |> build_breakdown_result(query_with_metrics, metrics)- |> maybe_add_time_on_page(site, query_with_metrics, metrics)- |> update_currency_metrics(site, query_with_metrics)+ breakdown_events(site, query, metrics)+ |> paginate_and_execute(metrics, pagination)+ end++ def breakdown(site, query, metrics, pagination, opts) do+ query = maybe_update_breakdown_filters(query)+ if !Keyword.get(opts, :skip_tracing), do: Query.trace(query, metrics)++ metrics_to_select = Util.maybe_add_visitors_metric(metrics) -- @computed_metrics++ case breakdown_table(query, metrics) do+ :session ->+ breakdown_sessions(site, query, metrics_to_select)+ |> maybe_add_group_conversion_rate(&breakdown_sessions/3, site, query, metrics)+ |> paginate_and_execute(metrics, pagination)++ :event ->+ breakdown_events(site, query, metrics_to_select)+ |> maybe_add_group_conversion_rate(&breakdown_events/3, site, query, metrics)+ |> paginate_and_execute(metrics, pagination)+ end+ end++ defp maybe_update_breakdown_filters(%Query{dimensions: [visit_entry_prop]} = query)+ when visit_entry_prop in [+ "visit:source",+ "visit:entry_page",+ "visit:utm_medium",+ "visit:utm_source",+ "visit:utm_campaign",+ "visit:utm_content",+ "visit:utm_term",+ "visit:entry_page",+ "visit:referrer"+ ] do+ update_hostname_filter_prop(query, "visit:entry_page_hostname")+ end++ defp maybe_update_breakdown_filters(%Query{dimensions: ["visit:exit_page"]} = query) do+ update_hostname_filter_prop(query, "visit:exit_page_hostname")+ end++ defp maybe_update_breakdown_filters(query) do+ query+ end++ defp update_hostname_filter_prop(query, visit_prop) do+ case Query.get_filter(query, "event:hostname") do+ nil ->+ query++ [op, "event:hostname", value] ->+ query+ |> Query.put_filter([op, visit_prop, value])+ end+ end++ # Backwards compatibility+ defp breakdown_table(%Query{experimental_reduced_joins?: false}, _), do: :session++ defp breakdown_table(%Query{dimensions: ["visit:entry_page"]}, _metrics), do: :session+ defp breakdown_table(%Query{dimensions: ["visit:entry_page_hostname"]}, _metrics), do: :session+ defp breakdown_table(%Query{dimensions: ["visit:exit_page"]}, _metrics), do: :session+ defp breakdown_table(%Query{dimensions: ["visit:exit_page_hostname"]}, _metrics), do: :session++ defp breakdown_table(%Query{dimensions: [_dimension]} = query, metrics) do+ {_, session_metrics, _} = TableDecider.partition_metrics(metrics, query)++ if not Enum.empty?(session_metrics) do+ :session+ else+ :event+ endend- defp build_breakdown_result(query_result, query, metrics) do- query_result.results- |> Enum.map(fn %{dimensions: dimensions, metrics: entry_metrics} ->- dimension_map =- query.dimensions |> Enum.map(&result_key/1) |> Enum.zip(dimensions) |> Enum.into(%{})+ defp zip_results(event_result, session_result, property, metrics) do+ null_row = Enum.map(metrics, fn metric -> {metric, nil} end) |> Enum.into(%{})- metrics_map = Enum.zip(metrics, entry_metrics) |> Enum.into(%{})+ prop_values =+ Enum.map(event_result ++ session_result, fn row -> row[property] end)+ |> Enum.uniq()- Map.merge(dimension_map, metrics_map)+ Enum.map(prop_values, fn value ->+ event_row = Enum.find(event_result, fn row -> row[property] == value end) || %{}+ session_row = Enum.find(session_result, fn row -> row[property] == value end) || %{}++ null_row+ |> Map.merge(event_row)+ |> Map.merge(session_row)end)+ |> sort_results(metrics)+ end++ defp breakdown_sessions(site, %Query{dimensions: [dimension]} = query, metrics) do+ from(s in query_sessions(site, query),+ order_by: [desc: fragment("uniq(?)", s.user_id)],+ select: ^select_session_metrics(metrics, query)+ )+ |> filter_converted_sessions(site, query)+ |> do_group_by(dimension)+ |> merge_imported(site, query, metrics)+ |> add_percentage_metric(site, query, metrics)+ end++ defp breakdown_events(site, %Query{dimensions: [dimension]} = query, metrics) do+ from(e in base_event_query(site, query),+ order_by: [desc: fragment("uniq(?)", e.user_id)],+ select: %{}+ )+ |> do_group_by(dimension)+ |> select_merge(^select_event_metrics(metrics))+ |> merge_imported(site, query, metrics)+ |> add_percentage_metric(site, query, metrics)end- defp result_key("event:props:" <> custom_property), do: custom_property- defp result_key("event:" <> key), do: key |> String.to_existing_atom()- defp result_key("visit:" <> key), do: key |> String.to_existing_atom()- defp result_key(dimension), do: dimension+ defp paginate_and_execute(_, [], _), do: []++ defp paginate_and_execute(q, metrics, pagination) do+ q+ |> apply_pagination(pagination)+ |> ClickhouseRepo.all()+ |> Util.keep_requested_metrics(metrics)+ enddefp maybe_add_time_on_page(event_results, site, query, metrics) do- if query.dimensions == ["event:page"] and :time_on_page in metrics do+ if :time_on_page in metrics dopages = Enum.map(event_results, & &1[:page])time_on_page_result = breakdown_time_on_page(site, query, pages)- event_results- |> Enum.map(fn row ->+ Enum.map(event_results, fn row ->Map.put(row, :time_on_page, time_on_page_result[row[:page]])end)else@@ -148,102 +431,359 @@ defmodule Plausible.Stats.Breakdown do|> Map.new()end- defp transform_metrics(metrics, dimension) do- metrics =- if is_nil(metric_to_order_by(metrics)) do- metrics ++ [:visitors]- else- metrics- end+ defp do_group_by(+ %Ecto.Query{from: %Ecto.Query.FromExpr{source: {"events" <> _, _}}} = q,+ "event:props:" <> prop+ ) do+ from(+ e in q,+ select_merge: %{+ breakdown_prop_value:+ selected_as(+ fragment(+ "if(not empty(?), ?, '(none)')",+ get_by_key(e, :meta, ^prop),+ get_by_key(e, :meta, ^prop)+ ),+ :breakdown_prop_value+ )+ },+ group_by: selected_as(:breakdown_prop_value),+ order_by: {:asc, selected_as(:breakdown_prop_value)}+ )+ end- Enum.map(metrics, fn metric ->- case {metric, dimension} do- {:conversion_rate, "event:props:" <> _} -> :conversion_rate- {:conversion_rate, "event:goal"} -> :conversion_rate- {:conversion_rate, _} -> :group_conversion_rate- _ -> metric- end- end)+ defp do_group_by(+ %Ecto.Query{from: %Ecto.Query.FromExpr{source: {"events" <> _, _}}} = q,+ "event:name"+ ) do+ from(+ e in q,+ group_by: e.name,+ select_merge: %{name: e.name},+ order_by: {:asc, e.name}+ )+ end++ defp do_group_by(+ %Ecto.Query{from: %Ecto.Query.FromExpr{source: {"events" <> _, _}}} = q,+ "event:page"+ ) do+ from(+ e in q,+ group_by: e.pathname,+ select_merge: %{page: e.pathname},+ order_by: {:asc, e.pathname}+ )end- defp infer_order_by(metrics, "event:goal"), do: [{metric_to_order_by(metrics), :desc}]+ defp do_group_by(q, "visit:source") do+ from(+ s in q,+ group_by: s.source,+ select_merge: %{+ source: fragment("if(empty(?), ?, ?)", s.source, @no_ref, s.source)+ },+ order_by: {:asc, s.source}+ )+ end- defp infer_order_by(metrics, dimension),- do: [{metric_to_order_by(metrics), :desc}, {dimension, :asc}]+ defp do_group_by(q, "visit:country") do+ from(+ s in q,+ where: s.country != "\0\0" and s.country != "ZZ",+ group_by: s.country,+ select_merge: %{country: s.country},+ order_by: {:asc, s.country}+ )+ end- defp metric_to_order_by(metrics) do- Enum.find(metrics, &(&1 != :time_on_page))+ defp do_group_by(q, "visit:region") do+ from(+ s in q,+ where: s.region != "",+ group_by: s.region,+ select_merge: %{region: s.region},+ order_by: {:asc, s.region}+ )end- def transform_dimensions("visit:browser_version"),- do: ["visit:browser", "visit:browser_version"]+ defp do_group_by(q, "visit:city") do+ from(+ s in q,+ where: s.city != 0,+ group_by: s.city,+ select_merge: %{city: s.city},+ order_by: {:asc, s.city}+ )+ end- def transform_dimensions("visit:os_version"), do: ["visit:os", "visit:os_version"]- def transform_dimensions(dimension), do: [dimension]+ defp do_group_by(q, "visit:entry_page") do+ from(+ s in q,+ # Sessions without pageviews don't get entry_page assigned, hence they should get ignored+ where: s.entry_page != "",+ group_by: s.entry_page,+ select_merge: %{entry_page: s.entry_page},+ order_by: {:asc, s.entry_page}+ )+ end- @filter_dimensions_not %{- "visit:city" => [0],- "visit:country" => ["\0\0", "ZZ"],- "visit:region" => [""],- "visit:utm_medium" => [""],- "visit:utm_source" => [""],- "visit:utm_campaign" => [""],- "visit:utm_content" => [""],- "visit:utm_term" => [""],- "visit:entry_page" => [""],- "visit:exit_page" => [""]- }+ defp do_group_by(q, "visit:exit_page") do+ from(+ s in q,+ # Sessions without pageviews don't get entry_page assigned, hence they should get ignored+ where: s.entry_page != "",+ group_by: s.exit_page,+ select_merge: %{exit_page: s.exit_page},+ order_by: {:asc, s.exit_page}+ )+ end- @extra_filter_dimensions Map.keys(@filter_dimensions_not)+ defp do_group_by(q, "visit:referrer") do+ from(+ s in q,+ group_by: s.referrer,+ select_merge: %{+ referrer: fragment("if(empty(?), ?, ?)", s.referrer, @no_ref, s.referrer)+ },+ order_by: {:asc, s.referrer}+ )+ end- defp dimension_filters(dimension) when dimension in @extra_filter_dimensions do- [[:is_not, dimension, Map.get(@filter_dimensions_not, dimension)]]+ defp do_group_by(q, "visit:utm_medium") do+ from(+ s in q,+ where: fragment("not empty(?)", s.utm_medium),+ group_by: s.utm_medium,+ select_merge: %{+ utm_medium: s.utm_medium+ }+ )end- defp dimension_filters(_), do: []+ defp do_group_by(q, "visit:utm_source") do+ from(+ s in q,+ where: fragment("not empty(?)", s.utm_source),+ group_by: s.utm_source,+ select_merge: %{+ utm_source: s.utm_source+ }+ )+ end- defp apply_pagination(q, {limit, page}) do- offset = (page - 1) * limit+ defp do_group_by(q, "visit:utm_campaign") do+ from(+ s in q,+ where: fragment("not empty(?)", s.utm_campaign),+ group_by: s.utm_campaign,+ select_merge: %{+ utm_campaign: s.utm_campaign+ }+ )+ end- q- |> limit(^limit)- |> offset(^offset)+ defp do_group_by(q, "visit:utm_content") do+ from(+ s in q,+ where: fragment("not empty(?)", s.utm_content),+ group_by: s.utm_content,+ select_merge: %{+ utm_content: s.utm_content+ }+ )end- on_ee do- defp update_currency_metrics(results, site, %Query{dimensions: ["event:goal"]}) do- site = Plausible.Repo.preload(site, :goals)+ defp do_group_by(q, "visit:utm_term") do+ from(+ s in q,+ where: fragment("not empty(?)", s.utm_term),+ group_by: s.utm_term,+ select_merge: %{+ utm_term: s.utm_term+ }+ )+ end- {event_goals, _pageview_goals} = Enum.split_with(site.goals, & &1.event_name)- revenue_goals = Enum.filter(event_goals, &Plausible.Goal.Revenue.revenue?/1)+ defp do_group_by(q, "visit:device") do+ from(+ s in q,+ group_by: s.device,+ select_merge: %{+ device: fragment("if(empty(?), ?, ?)", s.device, @not_set, s.device)+ },+ order_by: {:asc, s.device}+ )+ end- if length(revenue_goals) > 0 and Plausible.Billing.Feature.RevenueGoals.enabled?(site) do- Plausible.Stats.Goal.Revenue.cast_revenue_metrics_to_money(results, revenue_goals)- else- remove_revenue_metrics(results)- end- end+ defp do_group_by(q, "visit:os") do+ from(+ s in q,+ group_by: s.os,+ select_merge: %{+ os: fragment("if(empty(?), ?, ?)", s.os, @not_set, s.os)+ },+ order_by: {:asc, s.os}+ )+ end- defp update_currency_metrics(results, site, query) do- {currency, _metrics} =- Plausible.Stats.Goal.Revenue.get_revenue_tracking_currency(site, query, query.metrics)+ defp do_group_by(q, "visit:os_version") do+ from(+ s in q,+ group_by: [s.os, s.os_version],+ select_merge: %{+ os: fragment("if(empty(?), ?, ?)", s.os, @not_set, s.os),+ os_version:+ fragment(+ "if(empty(?), ?, ?)",+ s.os_version,+ @not_set,+ s.os_version+ )+ },+ order_by: {:asc, s.os_version}+ )+ end- if currency do- results- |> Enum.map(&Plausible.Stats.Goal.Revenue.cast_revenue_metrics_to_money(&1, currency))- else- remove_revenue_metrics(results)- end+ defp do_group_by(q, "visit:browser") do+ from(+ s in q,+ group_by: s.browser,+ select_merge: %{+ browser: fragment("if(empty(?), ?, ?)", s.browser, @not_set, s.browser)+ },+ order_by: {:asc, s.browser}+ )+ end++ defp do_group_by(q, "visit:browser_version") do+ from(+ s in q,+ group_by: [s.browser, s.browser_version],+ select_merge: %{+ browser: fragment("if(empty(?), ?, ?)", s.browser, @not_set, s.browser),+ browser_version:+ fragment("if(empty(?), ?, ?)", s.browser_version, @not_set, s.browser_version)+ },+ order_by: {:asc, s.browser_version}+ )+ end++ defp group_by_field_names("event:props:" <> _prop), do: [:name]+ defp group_by_field_names("visit:os_version"), do: [:os, :os_version]+ defp group_by_field_names("visit:browser_version"), do: [:browser, :browser_version]++ defp group_by_field_names(property), do: [Plausible.Stats.Filters.without_prefix(property)]++ defp on_matches_group_by(fields) do+ Enum.reduce(fields, nil, &fields_equal/2)+ end++ defp outer_order_by(fields) do+ Enum.map(fields, fn field_name -> {:asc, dynamic([q], field(q, ^field_name))} end)+ end++ defp fields_equal(field_name, nil),+ do: dynamic([a, b], field(a, ^field_name) == field(b, ^field_name))++ defp fields_equal(field_name, condition),+ do: dynamic([a, b], field(a, ^field_name) == field(b, ^field_name) and ^condition)++ defp sort_results(results, metrics) do+ Enum.sort_by(+ results,+ fn entry ->+ case entry[sorting_key(metrics)] do+ nil -> 0+ n -> n+ end+ end,+ :desc+ )+ end++ # This function injects a conversion_rate metric into+ # a breakdown query. It is calculated as X / Y, where:+ #+ # * X is the number of conversions for a breakdown+ # result (conversion = number of visitors who+ # completed the filtered goal with the filtered+ # custom properties).+ #+ # * Y is the number of all visitors for this breakdown+ # result without the `event:goal` and `event:props:*`+ # filters.+ defp maybe_add_group_conversion_rate(+ q,+ breakdown_fn,+ site,+ %Query{dimensions: [dimension]} = query,+ metrics+ ) do+ if :conversion_rate in metrics do+ breakdown_total_visitors_query =+ query |> Query.remove_filters(["event:goal", "event:props"])++ breakdown_total_visitors_q =+ breakdown_fn.(site, breakdown_total_visitors_query, [:visitors])++ from(e in subquery(q),+ left_join: c in subquery(breakdown_total_visitors_q),+ on: ^on_matches_group_by(group_by_field_names(dimension)),+ select_merge: %{+ total_visitors: c.visitors,+ conversion_rate:+ fragment(+ "if(? > 0, round(? / ? * 100, 1), 0)",+ c.visitors,+ e.visitors,+ c.visitors+ )+ },+ order_by: [desc: e.visitors],+ order_by: ^outer_order_by(group_by_field_names(dimension))+ )+ else+ qend- else- defp update_currency_metrics(results, _site, _query), do: remove_revenue_metrics(results)end- defp remove_revenue_metrics(results) do+ # When querying custom event goals and pageviewgoals together, UNION ALL is used+ # so the same fields must be present on both sides of the union. This change to the+ # query will ensure that we don't unnecessarily read revenue column for pageview goals+ defp mark_revenue_as_nil(select_columns) do+ select_columns+ |> Map.replace(:total_revenue, nil)+ |> Map.replace(:average_revenue, nil)+ end++ defp sorting_key(metrics) do+ if Enum.member?(metrics, :visitors), do: :visitors, else: List.first(metrics)+ end++ defp transform_keys(results, keys_to_replace) doEnum.map(results, fn map ->- map- |> Map.delete(:total_revenue)- |> Map.delete(:average_revenue)+ Enum.map(map, fn {key, val} ->+ {Map.get(keys_to_replace, key, key), val}+ end)+ |> Enum.into(%{})end)end++ defp apply_pagination(q, {limit, page}) do+ offset = (page - 1) * limit++ q+ |> limit(^limit)+ |> offset(^offset)+ end++ on_ee do+ defp cast_revenue_metrics_to_money(results, revenue_goals) do+ Plausible.Stats.Goal.Revenue.cast_revenue_metrics_to_money(results, revenue_goals)+ end+ else+ defp cast_revenue_metrics_to_money(results, _revenue_goals), do: results+ endenddiff --git a/lib/plausible/stats/filters/query_parser.ex b/lib/plausible/stats/filters/query_parser.exindex 3f802dcb64..0184478905 100644--- a/lib/plausible/stats/filters/query_parser.ex+++ b/lib/plausible/stats/filters/query_parser.ex@@ -4,7 +4,6 @@ defmodule Plausible.Stats.Filters.QueryParser doalias Plausible.Stats.TableDecideralias Plausible.Stats.Filtersalias Plausible.Stats.Query- alias Plausible.Stats.Metricsdef parse(site, params, now \\ nil) when is_map(params) dowith {:ok, metrics} <- parse_metrics(Map.get(params, "metrics", [])),@@ -45,12 +44,17 @@ defmodule Plausible.Stats.Filters.QueryParser dodefp parse_metrics(_invalid_metrics), do: {:error, "Invalid metrics passed"}- defp parse_metric(metric_str) do- case Metrics.from_string(metric_str) do- {:ok, metric} -> {:ok, metric}- _ -> {:error, "Unknown metric '#{inspect(metric_str)}'"}- end- end+ defp parse_metric("time_on_page"), do: {:ok, :time_on_page}+ defp parse_metric("conversion_rate"), do: {:ok, :conversion_rate}+ defp parse_metric("group_conversion_rate"), do: {:ok, :group_conversion_rate}+ defp parse_metric("visitors"), do: {:ok, :visitors}+ defp parse_metric("pageviews"), do: {:ok, :pageviews}+ defp parse_metric("events"), do: {:ok, :events}+ defp parse_metric("visits"), do: {:ok, :visits}+ defp parse_metric("bounce_rate"), do: {:ok, :bounce_rate}+ defp parse_metric("visit_duration"), do: {:ok, :visit_duration}+ defp parse_metric("views_per_visit"), do: {:ok, :views_per_visit}+ defp parse_metric(unknown_metric), do: {:error, "Unknown metric '#{inspect(unknown_metric)}'"}def parse_filters(filters) when is_list(filters) doparse_list(filters, &parse_filter/1)@@ -280,7 +284,7 @@ defmodule Plausible.Stats.Filters.QueryParser doendend- def preload_goals_if_needed(site, filters, dimensions) do+ defp preload_goals_if_needed(site, filters, dimensions) dogoal_filters? =Enum.any?(filters, fn [_, filter_key | _rest] -> filter_key == "event:goal" end)diff --git a/lib/plausible/stats/filters/where_builder.ex b/lib/plausible/stats/filters/where_builder.exindex 257d4257a0..e04396d999 100644--- a/lib/plausible/stats/filters/where_builder.ex+++ b/lib/plausible/stats/filters/where_builder.ex@@ -10,8 +10,6 @@ defmodule Plausible.Stats.Filters.WhereBuilder douse Plausible.Stats.Fragments- require Logger-@sessions_only_visit_fields [:entry_page,:exit_page,@@ -152,15 +150,6 @@ defmodule Plausible.Stats.Filters.WhereBuilder dotrueend- defp add_filter(table, _query, filter) do- Logger.info("Unable to process garbage filter. No results are returned",- table: table,- filter: filter- )-- false- end-defp filter_custom_prop(prop_name, column_name, [:is, _, values]) donone_value_included = Enum.member?(values, "(none)")diff --git a/lib/plausible/stats/imported/base.ex b/lib/plausible/stats/imported/base.exindex 5b774d9ea9..a446817a37 100644--- a/lib/plausible/stats/imported/base.ex+++ b/lib/plausible/stats/imported/base.ex@@ -54,12 +54,10 @@ defmodule Plausible.Stats.Imported.Base dodef property_to_table_mappings(), do: @property_to_table_mappingsdef query_imported(site, query) do- [table] =- query- |> transform_filters()- |> decide_tables()-- query_imported(table, site, query)+ query+ |> transform_filters()+ |> decide_table()+ |> query_imported(site, query)enddef query_imported(table, site, query) do@@ -77,13 +75,13 @@ defmodule Plausible.Stats.Imported.Base do|> apply_filter(query)end- def decide_tables(query) do+ def decide_table(query) doquery = transform_filters(query)if custom_prop_query?(query) dodo_decide_custom_prop_table(query)else- do_decide_tables(query)+ do_decide_table(query)endend@@ -94,17 +92,19 @@ defmodule Plausible.Stats.Imported.Base do[:is, "event:name", ["pageview"]] -> true_ -> falseend)- |> Enum.flat_map(fn- [op, "event:goal", clauses] ->- clauses- |> Enum.group_by(&elem(&1, 0), &elem(&1, 1))- |> Enum.map(fn- {:event, names} -> [op, "event:name", names]- {:page, pages} -> [op, "event:page", pages]- end)-- filter ->- [filter]+ |> Enum.flat_map(fn filter ->+ case filter do+ [op, "event:goal", events] ->+ events+ |> Enum.group_by(&elem(&1, 0), &elem(&1, 1))+ |> Enum.map(fn+ {:event, names} -> [op, "event:name", names]+ {:page, pages} -> [op, "event:page", pages]+ end)++ filter ->+ [filter]+ endend)struct!(query, filters: new_filters)@@ -136,10 +136,10 @@ defmodule Plausible.Stats.Imported.Base dodo_decide_custom_prop_table(query, custom_prop_filter)_ ->- []+ nilendelse- []+ nilendend@@ -158,27 +158,23 @@ defmodule Plausible.Stats.Imported.Base doend)if has_required_name_filter? and not has_unsupported_filters? do- ["imported_custom_events"]+ "imported_custom_events"else- []+ nilendend- defp do_decide_tables(%Query{filters: [], dimensions: []}), do: ["imported_visitors"]+ defp do_decide_table(%Query{filters: [], dimensions: []}), do: "imported_visitors"- defp do_decide_tables(%Query{filters: [], dimensions: ["event:goal"]}) do- ["imported_pages", "imported_custom_events"]+ defp do_decide_table(%Query{filters: [], dimensions: ["event:goal"]}) do+ "imported_custom_events"end- defp do_decide_tables(%Query{filters: [], dimensions: [dimension]}) do- if Map.has_key?(@property_to_table_mappings, dimension) do- [@property_to_table_mappings[dimension]]- else- []- end+ defp do_decide_table(%Query{filters: [], dimensions: [dimension]}) do+ @property_to_table_mappings[dimension]end- defp do_decide_tables(%Query{filters: filters, dimensions: ["event:goal"]}) do+ defp do_decide_table(%Query{filters: filters, dimensions: ["event:goal"]}) dofilter_props = Enum.map(filters, &Enum.at(&1, 1))any_event_name_filters? = "event:name" in filter_props@@ -186,28 +182,27 @@ defmodule Plausible.Stats.Imported.Base doany_other_filters? = Enum.any?(filter_props, &(&1 not in ["event:page", "event:name"]))cond do- any_other_filters? -> []- any_event_name_filters? and not any_page_filters? -> ["imported_custom_events"]- any_page_filters? and not any_event_name_filters? -> ["imported_pages"]- true -> []+ any_other_filters? -> nil+ any_event_name_filters? and not any_page_filters? -> "imported_custom_events"+ any_page_filters? and not any_event_name_filters? -> "imported_pages"+ true -> nilendend- defp do_decide_tables(%Query{filters: filters, dimensions: dimensions}) do+ defp do_decide_table(%Query{filters: filters, dimensions: dimensions}) dotable_candidates =filters|> Enum.map(fn [_, filter_key | _] -> filter_key end)|> Enum.concat(dimensions)|> Enum.map(fn"visit:screen" -> "visit:device"- dimension -> dimension+ prop -> propend)|> Enum.map(&@property_to_table_mappings[&1])case Enum.uniq(table_candidates) do- [nil] -> []- [candidate] -> [candidate]- _ -> []+ [candidate] -> candidate+ _ -> nilendenddiff --git a/lib/plausible/stats/imported/imported.ex b/lib/plausible/stats/imported/imported.exindex 57cd6737bb..6542ab44f3 100644--- a/lib/plausible/stats/imported/imported.ex+++ b/lib/plausible/stats/imported/imported.ex@@ -4,11 +4,10 @@ defmodule Plausible.Stats.Imported doimport Ecto.Queryimport Plausible.Stats.Fragments- import Plausible.Stats.Util, only: [shortname: 2]+ alias Plausible.Stats.Basealias Plausible.Stats.Importedalias Plausible.Stats.Query- alias Plausible.Stats.SQL.QueryBuilder@no_ref "Direct / None"@not_set "(not set)"@@ -16,7 +15,7 @@ defmodule Plausible.Stats.Imported do@property_to_table_mappings Imported.Base.property_to_table_mappings()- @imported_dimensions Map.keys(@property_to_table_mappings) +++ @imported_properties Map.keys(@property_to_table_mappings) ++Plausible.Imported.imported_custom_props()@goals_with_url Plausible.Imported.goals_with_url()@@ -38,7 +37,7 @@ defmodule Plausible.Stats.Imported do(see `@goals_with_url` and `@goals_with_path`)."""def schema_supports_query?(query) do- length(Imported.Base.decide_tables(query)) > 0+ not is_nil(Imported.Base.decide_table(query))enddef merge_imported_country_suggestions(native_q, _site, %Plausible.Stats.Query{@@ -268,6 +267,78 @@ defmodule Plausible.Stats.Imported dodef merge_imported(q, _, %Query{include_imported: false}, _), do: q+ # Note: Only called for APIv2, old APIs use merge_imported_pageview_goals+ def merge_imported(q, site, %Query{dimensions: ["event:goal"]} = query, metrics)+ when query.v2 do+ {events, page_regexes} = Filters.Utils.split_goals_query_expressions(query.preloaded_goals)++ events_q =+ "imported_custom_events"+ |> Imported.Base.query_imported(site, query)+ |> where([i], i.visitors > 0)+ |> select_merge([i], %{+ dim0: selected_as(fragment("-indexOf(?, ?)", ^events, i.name), :dim0)+ })+ |> select_imported_metrics(metrics)+ |> group_by([], selected_as(:dim0))+ |> where([], selected_as(:dim0) != 0)++ pages_q =+ "imported_pages"+ |> Imported.Base.query_imported(site, query)+ |> where([i], i.visitors > 0)+ |> where(+ [i],+ fragment("notEmpty(multiMatchAllIndices(?, ?) as indices)", i.page, ^page_regexes)+ )+ |> join(:array, index in fragment("indices"))+ |> group_by([_i, index], index)+ |> select_merge([_i, index], %{+ dim0: type(fragment("?", index), :integer)+ })+ |> select_imported_metrics(metrics)++ q+ |> naive_dimension_join(events_q, metrics)+ |> naive_dimension_join(pages_q, metrics)+ end++ def merge_imported(q, site, %Query{dimensions: [dimension]} = query, metrics)+ when dimension in @imported_properties do+ dim = Plausible.Stats.Filters.without_prefix(dimension)++ imported_q =+ site+ |> Imported.Base.query_imported(query)+ |> where([i], i.visitors > 0)+ |> group_imported_by(dim, query)+ |> select_imported_metrics(metrics)++ join_on =+ case dim do+ _ when dim in [:url, :path] and not query.v2 ->+ dynamic([s, i], s.breakdown_prop_value == i.breakdown_prop_value)++ :os_version when not query.v2 ->+ dynamic([s, i], s.os == i.os and s.os_version == i.os_version)++ :browser_version when not query.v2 ->+ dynamic([s, i], s.browser == i.browser and s.browser_version == i.browser_version)++ dim ->+ dynamic([s, i], field(s, ^shortname(query, dim)) == field(i, ^shortname(query, dim)))+ end++ from(s in Ecto.Query.subquery(q),+ full_join: i in subquery(imported_q),+ on: ^join_on,+ select: %{}+ )+ |> select_joined_dimension(dim, query)+ |> select_joined_metrics(metrics)+ |> apply_order_by(query, metrics)+ end+def merge_imported(q, site, %Query{dimensions: []} = query, metrics) doimported_q =site@@ -282,82 +353,51 @@ defmodule Plausible.Stats.Imported do|> select_joined_metrics(metrics)end- def merge_imported(q, site, %Query{dimensions: ["event:goal"]} = query, metrics) do- {events, page_regexes} = Filters.Utils.split_goals_query_expressions(query.preloaded_goals)+ def merge_imported(q, _, _, _), do: q- Imported.Base.decide_tables(query)- |> Enum.map(fn- "imported_custom_events" ->- Imported.Base.query_imported("imported_custom_events", site, query)- |> where([i], i.visitors > 0)- |> select_merge([i], %{- dim0:- selected_as(- fragment("-indexOf(?, ?)", type(^events, {:array, :string}), i.name),- :dim0- )- })- |> select_imported_metrics(metrics)- |> group_by([], selected_as(:dim0))- |> where([], selected_as(:dim0) != 0)+ def merge_imported_pageview_goals(q, _, %Query{include_imported: false}, _, _), do: q- "imported_pages" ->- Imported.Base.query_imported("imported_pages", site, query)+ def merge_imported_pageview_goals(q, site, query, page_exprs, metrics) do+ if Imported.Base.decide_table(query) == "imported_pages" do+ page_regexes = Enum.map(page_exprs, &Base.page_regex/1)++ imported_q =+ "imported_pages"+ |> Imported.Base.query_imported(site, query)|> where([i], i.visitors > 0)|> where([i],- fragment(- "notEmpty(multiMatchAllIndices(?, ?) as indices)",- i.page,- type(^page_regexes, {:array, :string})- )+ fragment("notEmpty(multiMatchAllIndices(?, ?) as indices)", i.page, ^page_regexes))|> join(:array, index in fragment("indices"))|> group_by([_i, index], index)|> select_merge([_i, index], %{- dim0: selected_as(type(fragment("?", index), :integer), :dim0)+ name: fragment("concat('Visit ', ?[?])", ^page_exprs, index)})|> select_imported_metrics(metrics)- end)- |> Enum.reduce(q, fn imports_q, q ->- naive_dimension_join(q, imports_q, metrics)- end)- end-- def merge_imported(q, site, %Query{dimensions: dimensions} = query, metrics) do- if merge_imported_dimensions?(dimensions) do- imported_q =- site- |> Imported.Base.query_imported(query)- |> where([i], i.visitors > 0)- |> group_imported_by(query)- |> select_imported_metrics(metrics)- from(s in subquery(q),+ from(s in Ecto.Query.subquery(q),full_join: i in subquery(imported_q),- on: ^QueryBuilder.build_group_by_join(query),+ on: s.name == i.name,select: %{})- |> select_joined_dimensions(query)+ |> select_joined_dimension(:name, query)|> select_joined_metrics(metrics)elseqendend- def merge_imported(q, _, _, _), do: q-- defp merge_imported_dimensions?(dimensions) do- dimensions in [["visit:browser", "visit:browser_version"], ["visit:os", "visit:os_version"]] or- (length(dimensions) == 1 and hd(dimensions) in @imported_dimensions)- end-def total_imported_visitors(site, query) dosite|> Imported.Base.query_imported(query)|> select_merge([i], %{total_visitors: fragment("sum(?)", i.visitors)})end+ # :TRICKY: Handle backwards compatibility with old breakdown module+ defp shortname(query, _dim) when query.v2, do: :dim0+ defp shortname(_query, dim), do: dim+defp select_imported_metrics(q, []), do: qdefp select_imported_metrics(q, [:visitors | rest]) do@@ -552,175 +592,190 @@ defmodule Plausible.Stats.Imported do|> select_imported_metrics(rest)end- defp group_imported_by(q, query) do- Enum.reduce(query.dimensions, q, fn dimension, q ->- dim = Plausible.Stats.Filters.without_prefix(dimension)-- group_imported_by(q, dim, shortname(query, dimension))- end)- end-- defp group_imported_by(q, dim, key) when dim in [:source, :referrer] do+ defp group_imported_by(q, dim, query) when dim in [:source, :referrer] doq|> group_by([i], field(i, ^dim))|> select_merge([i], %{- ^key =>- selected_as(- fragment(- "if(empty(?), ?, ?)",- field(i, ^dim),- @no_ref,- field(i, ^dim)- ),- ^key+ ^shortname(query, dim) =>+ fragment(+ "if(empty(?), ?, ?)",+ field(i, ^dim),+ @no_ref,+ field(i, ^dim))})end- defp group_imported_by(q, dim, key)+ defp group_imported_by(q, dim, query)when dim in [:utm_source, :utm_medium, :utm_campaign, :utm_term, :utm_content] doq|> group_by([i], field(i, ^dim))|> where([i], fragment("not empty(?)", field(i, ^dim)))- |> select_merge([i], %{^key => selected_as(field(i, ^dim), ^key)})+ |> select_merge([i], %{^shortname(query, dim) => field(i, ^dim)})end- defp group_imported_by(q, :page, key) do+ defp group_imported_by(q, :page, query) doq|> group_by([i], i.page)- |> select_merge([i], %{^key => selected_as(i.page, ^key), time_on_page: sum(i.time_on_page)})+ |> select_merge([i], %{^shortname(query, :page) => i.page, time_on_page: sum(i.time_on_page)})end- defp group_imported_by(q, :country, key) do+ defp group_imported_by(q, :country, query) doq|> group_by([i], i.country)|> where([i], i.country != "ZZ")- |> select_merge([i], %{^key => selected_as(i.country, ^key)})+ |> select_merge([i], %{^shortname(query, :country) => i.country})end- defp group_imported_by(q, :region, key) do+ defp group_imported_by(q, :region, query) doq|> group_by([i], i.region)|> where([i], i.region != "")- |> select_merge([i], %{^key => selected_as(i.region, ^key)})+ |> select_merge([i], %{^shortname(query, :region) => i.region})end- defp group_imported_by(q, :city, key) do+ defp group_imported_by(q, :city, query) doq|> group_by([i], i.city)|> where([i], i.city != 0 and not is_nil(i.city))- |> select_merge([i], %{^key => selected_as(i.city, ^key)})+ |> select_merge([i], %{^shortname(query, :city) => i.city})end- defp group_imported_by(q, dim, key) when dim in [:device, :browser] do+ defp group_imported_by(q, dim, query) when dim in [:device, :browser] doq|> group_by([i], field(i, ^dim))|> select_merge([i], %{- ^key =>- selected_as(- fragment("if(empty(?), ?, ?)", field(i, ^dim), @not_set, field(i, ^dim)),- ^key- )+ ^shortname(query, dim) =>+ fragment("if(empty(?), ?, ?)", field(i, ^dim), @not_set, field(i, ^dim))})end- defp group_imported_by(q, :browser_version, key) do+ defp group_imported_by(q, :browser_version, query) doq- |> group_by([i], [i.browser_version])+ |> group_by([i], [i.browser, i.browser_version])|> select_merge([i], %{- ^key =>- selected_as(- fragment(- "if(empty(?), ?, ?)",- i.browser_version,- @not_set,- i.browser_version- ),- ^key+ ^shortname(query, :browser) =>+ fragment("if(empty(?), ?, ?)", i.browser, @not_set, i.browser),+ ^shortname(query, :browser_version) =>+ fragment(+ "if(empty(?), ?, ?)",+ i.browser_version,+ @not_set,+ i.browser_version)})end- defp group_imported_by(q, :os, key) do+ defp group_imported_by(q, :os, query) doq|> group_by([i], i.operating_system)|> select_merge([i], %{- ^key =>- selected_as(- fragment("if(empty(?), ?, ?)", i.operating_system, @not_set, i.operating_system),- ^key- )+ ^shortname(query, :os) =>+ fragment("if(empty(?), ?, ?)", i.operating_system, @not_set, i.operating_system)})end- defp group_imported_by(q, :os_version, key) do+ defp group_imported_by(q, :os_version, query) doq- |> group_by([i], [i.operating_system_version])+ |> group_by([i], [i.operating_system, i.operating_system_version])|> select_merge([i], %{- ^key =>- selected_as(- fragment(- "if(empty(?), ?, ?)",- i.operating_system_version,- @not_set,- i.operating_system_version- ),- ^key+ ^shortname(query, :os) =>+ fragment("if(empty(?), ?, ?)", i.operating_system, @not_set, i.operating_system),+ ^shortname(query, :os_version) =>+ fragment(+ "if(empty(?), ?, ?)",+ i.operating_system_version,+ @not_set,+ i.operating_system_version)})end- defp group_imported_by(q, dim, key) when dim in [:entry_page, :exit_page] do+ defp group_imported_by(q, dim, query) when dim in [:entry_page, :exit_page] doq|> group_by([i], field(i, ^dim))- |> select_merge([i], %{^key => selected_as(field(i, ^dim), ^key)})+ |> select_merge([i], %{^shortname(query, dim) => field(i, ^dim)})end- defp group_imported_by(q, :name, key) do+ defp group_imported_by(q, :name, query) doq|> group_by([i], i.name)- |> select_merge([i], %{^key => selected_as(i.name, ^key)})+ |> select_merge([i], %{^shortname(query, :name) => i.name})+ end++ defp group_imported_by(q, :url, query) when query.v2 do+ q+ |> group_by([i], i.link_url)+ |> select_merge([i], %{+ ^shortname(query, :url) => fragment("if(not empty(?), ?, ?)", i.link_url, i.link_url, @none)+ })end- defp group_imported_by(q, :url, key) do+ defp group_imported_by(q, :url, _query) doq|> group_by([i], i.link_url)|> select_merge([i], %{- ^key => selected_as(fragment("if(not empty(?), ?, ?)", i.link_url, i.link_url, @none), ^key)+ breakdown_prop_value: fragment("if(not empty(?), ?, ?)", i.link_url, i.link_url, @none)})end- defp group_imported_by(q, :path, key) do+ defp group_imported_by(q, :path, query) when query.v2 doq|> group_by([i], i.path)|> select_merge([i], %{- ^key => selected_as(fragment("if(not empty(?), ?, ?)", i.path, i.path, @none), ^key)+ ^shortname(query, :path) => fragment("if(not empty(?), ?, ?)", i.path, i.path, @none)})end- defp select_joined_dimensions(q, query) do- Enum.reduce(query.dimensions, q, fn dimension, q ->- select_joined_dimension(q, dimension, shortname(query, dimension))- end)+ defp group_imported_by(q, :path, _query) do+ q+ |> group_by([i], i.path)+ |> select_merge([i], %{+ breakdown_prop_value: fragment("if(not empty(?), ?, ?)", i.path, i.path, @none)+ })+ end++ defp select_joined_dimension(q, :city, query) do+ select_merge(q, [s, i], %{+ ^shortname(query, :city) => fragment("greatest(?,?)", i.city, s.city)+ })+ end++ defp select_joined_dimension(q, :os_version, query) when not query.v2 do+ select_merge(q, [s, i], %{+ os: fragment("if(empty(?), ?, ?)", s.os, i.os, s.os),+ os_version: fragment("if(empty(?), ?, ?)", s.os_version, i.os_version, s.os_version)+ })+ end++ defp select_joined_dimension(q, :browser_version, query) when not query.v2 do+ select_merge(q, [s, i], %{+ browser: fragment("if(empty(?), ?, ?)", s.browser, i.browser, s.browser),+ browser_version:+ fragment("if(empty(?), ?, ?)", s.browser_version, i.browser_version, s.browser_version)+ })end- defp select_joined_dimension(q, "visit:city", key) do+ defp select_joined_dimension(q, dim, query) when dim in [:url, :path] and not query.v2 doselect_merge(q, [s, i], %{- ^key => selected_as(fragment("greatest(?,?)", field(i, ^key), field(s, ^key)), ^key)+ breakdown_prop_value:+ fragment(+ "if(empty(?), ?, ?)",+ s.breakdown_prop_value,+ i.breakdown_prop_value,+ s.breakdown_prop_value+ )})end- defp select_joined_dimension(q, _dimension, key) do+ defp select_joined_dimension(q, dim, query) doselect_merge(q, [s, i], %{- ^key =>- selected_as(- fragment(- "if(empty(?), ?, ?)",- field(s, ^key),- field(i, ^key),- field(s, ^key)- ),- ^key+ ^shortname(query, dim) =>+ fragment(+ "if(empty(?), ?, ?)",+ field(s, ^shortname(query, dim)),+ field(i, ^shortname(query, dim)),+ field(s, ^shortname(query, dim)))})end@@ -827,12 +882,21 @@ defmodule Plausible.Stats.Imported do|> select_joined_metrics(rest)end+ defp apply_order_by(q, %Query{v2: true}, _), do: q++ defp apply_order_by(q, query, [:visitors | rest]) do+ order_by(q, [s, i], desc: s.visitors + i.visitors)+ |> apply_order_by(query, rest)+ end++ defp apply_order_by(q, _query, _), do: q+defp naive_dimension_join(q1, q2, metrics) do- from(a in subquery(q1),+ from(a in Ecto.Query.subquery(q1),full_join: b in subquery(q2),on: a.dim0 == b.dim0,select: %{- dim0: selected_as(fragment("if(? != 0, ?, ?)", a.dim0, a.dim0, b.dim0), :dim0)+ dim0: fragment("coalesce(?, ?)", a.dim0, b.dim0)})|> select_joined_metrics(metrics)diff --git a/lib/plausible/stats/metrics.ex b/lib/plausible/stats/metrics.exindex 29319d07a4..8ed71dd2e3 100644--- a/lib/plausible/stats/metrics.ex+++ b/lib/plausible/stats/metrics.ex@@ -17,8 +17,7 @@ defmodule Plausible.Stats.Metrics do:events,:conversion_rate,:group_conversion_rate,- :time_on_page,- :percentage+ :time_on_page] ++ on_ee(do: Plausible.Stats.Goal.Revenue.revenue_metrics(), else: [])@metric_mappings Enum.into(@all_metrics, %{}, fn metric -> {to_string(metric), metric} end)@@ -28,8 +27,4 @@ defmodule Plausible.Stats.Metrics dodef from_string!(str) doMap.fetch!(@metric_mappings, str)end-- def from_string(str) do- Map.fetch(@metric_mappings, str)- endenddiff --git a/lib/plausible/stats/query.ex b/lib/plausible/stats/query.exindex fa6947c2d3..86fc13220c 100644--- a/lib/plausible/stats/query.ex+++ b/lib/plausible/stats/query.ex@@ -225,12 +225,6 @@ defmodule Plausible.Stats.Query dostruct!(query, filters: Filters.parse(params["filters"]))end- def set(query, keywords) do- query- |> struct!(keywords)- |> refresh_imported_opts()- end-@spec set_dimensions(t(), list(String.t())) :: t()def set_dimensions(query, dimensions) doquerydiff --git a/lib/plausible/stats/query_optimizer.ex b/lib/plausible/stats/query_optimizer.exindex 25146d950d..c1c38e5fb1 100644--- a/lib/plausible/stats/query_optimizer.ex+++ b/lib/plausible/stats/query_optimizer.ex@@ -3,7 +3,6 @@ defmodule Plausible.Stats.QueryOptimizer doMethods to manipulate Query for business logic reasons before building an ecto query."""- use Plausiblealias Plausible.Stats.{Query, TableDecider, Util}@doc """@@ -149,17 +148,8 @@ defmodule Plausible.Stats.QueryOptimizer dodimension -> dimensionend)- filters =- if "event:page" in query.dimensions do- query.filters- |> Enum.map(fn- [op, "event:page" | rest] -> [op, "visit:entry_page" | rest]- filter -> filter- end)- else- query.filters- end-- Query.set(query, filters: filters, metrics: session_metrics, dimensions: dimensions)+ query+ |> Query.set_metrics(session_metrics)+ |> Query.set_dimensions(dimensions)endenddiff --git a/lib/plausible/stats/sql/expression.ex b/lib/plausible/stats/sql/expression.exindex d3f38a1428..e74348daed 100644--- a/lib/plausible/stats/sql/expression.ex+++ b/lib/plausible/stats/sql/expression.ex@@ -11,121 +11,84 @@ defmodule Plausible.Stats.SQL.Expression do@no_ref "Direct / None"@not_set "(not set)"- defmacrop field_or_blank_value(expr, empty_value, select_alias) do+ defmacrop field_or_blank_value(expr, empty_value) doquote dodynamic([t],- selected_as(- fragment("if(empty(?), ?, ?)", unquote(expr), unquote(empty_value), unquote(expr)),- ^unquote(select_alias)- )+ fragment("if(empty(?), ?, ?)", unquote(expr), unquote(empty_value), unquote(expr)))endend- def dimension("time:hour", query, select_alias) do- dynamic(- [t],- selected_as(- fragment("toStartOfHour(toTimeZone(?, ?))", t.timestamp, ^query.timezone),- ^select_alias- )- )+ def dimension("time:hour", query) do+ dynamic([t], fragment("toStartOfHour(toTimeZone(?, ?))", t.timestamp, ^query.timezone))end- def dimension("time:day", query, select_alias) do- dynamic(- [t],- selected_as(- fragment("toDate(toTimeZone(?, ?))", t.timestamp, ^query.timezone),- ^select_alias- )- )+ def dimension("time:day", query) do+ dynamic([t], fragment("toDate(toTimeZone(?, ?))", t.timestamp, ^query.timezone))end- def dimension("time:month", query, select_alias) do- dynamic(- [t],- selected_as(- fragment("toStartOfMonth(toTimeZone(?, ?))", t.timestamp, ^query.timezone),- ^select_alias- )- )+ def dimension("time:month", query) do+ dynamic([t], fragment("toStartOfMonth(toTimeZone(?, ?))", t.timestamp, ^query.timezone))end- def dimension("event:name", _query, select_alias),- do: dynamic([t], selected_as(t.name, ^select_alias))-- def dimension("event:page", _query, select_alias),- do: dynamic([t], selected_as(t.pathname, ^select_alias))-- def dimension("event:hostname", _query, select_alias),- do: dynamic([t], selected_as(t.hostname, ^select_alias))+ def dimension("event:name", _query), do: dynamic([t], t.name)+ def dimension("event:page", _query), do: dynamic([t], t.pathname)+ def dimension("event:hostname", _query), do: dynamic([t], t.hostname)- def dimension("event:props:" <> property_name, _query, select_alias) do+ def dimension("event:props:" <> property_name, _query) dodynamic([t],- selected_as(- fragment(- "if(not empty(?), ?, '(none)')",- get_by_key(t, :meta, ^property_name),- get_by_key(t, :meta, ^property_name)- ),- ^select_alias+ fragment(+ "if(not empty(?), ?, '(none)')",+ get_by_key(t, :meta, ^property_name),+ get_by_key(t, :meta, ^property_name)))end- def dimension("visit:entry_page", _query, select_alias),- do: dynamic([t], selected_as(t.entry_page, ^select_alias))-- def dimension("visit:exit_page", _query, select_alias),- do: dynamic([t], selected_as(t.exit_page, ^select_alias))-- def dimension("visit:utm_medium", _query, select_alias),- do: field_or_blank_value(t.utm_medium, @not_set, select_alias)-- def dimension("visit:utm_source", _query, select_alias),- do: field_or_blank_value(t.utm_source, @not_set, select_alias)+ def dimension("visit:entry_page", _query), do: dynamic([t], t.entry_page)+ def dimension("visit:exit_page", _query), do: dynamic([t], t.exit_page)- def dimension("visit:utm_campaign", _query, select_alias),- do: field_or_blank_value(t.utm_campaign, @not_set, select_alias)+ def dimension("visit:utm_medium", _query),+ do: field_or_blank_value(t.utm_medium, @not_set)- def dimension("visit:utm_content", _query, select_alias),- do: field_or_blank_value(t.utm_content, @not_set, select_alias)+ def dimension("visit:utm_source", _query),+ do: field_or_blank_value(t.utm_source, @not_set)- def dimension("visit:utm_term", _query, select_alias),- do: field_or_blank_value(t.utm_term, @not_set, select_alias)+ def dimension("visit:utm_campaign", _query),+ do: field_or_blank_value(t.utm_campaign, @not_set)- def dimension("visit:source", _query, select_alias),- do: field_or_blank_value(t.source, @no_ref, select_alias)+ def dimension("visit:utm_content", _query),+ do: field_or_blank_value(t.utm_content, @not_set)- def dimension("visit:referrer", _query, select_alias),- do: field_or_blank_value(t.referrer, @no_ref, select_alias)+ def dimension("visit:utm_term", _query),+ do: field_or_blank_value(t.utm_term, @not_set)- def dimension("visit:device", _query, select_alias),- do: field_or_blank_value(t.device, @not_set, select_alias)+ def dimension("visit:source", _query),+ do: field_or_blank_value(t.source, @no_ref)- def dimension("visit:os", _query, select_alias),- do: field_or_blank_value(t.os, @not_set, select_alias)+ def dimension("visit:referrer", _query),+ do: field_or_blank_value(t.referrer, @no_ref)- def dimension("visit:os_version", _query, select_alias),- do: field_or_blank_value(t.os_version, @not_set, select_alias)+ def dimension("visit:device", _query),+ do: field_or_blank_value(t.device, @not_set)- def dimension("visit:browser", _query, select_alias),- do: field_or_blank_value(t.browser, @not_set, select_alias)+ def dimension("visit:os", _query), do: field_or_blank_value(t.os, @not_set)- def dimension("visit:browser_version", _query, select_alias),- do: field_or_blank_value(t.browser_version, @not_set, select_alias)+ def dimension("visit:os_version", _query),+ do: field_or_blank_value(t.os_version, @not_set)- def dimension("visit:country", _query, select_alias),- do: dynamic([t], selected_as(t.country, ^select_alias))+ def dimension("visit:browser", _query),+ do: field_or_blank_value(t.browser, @not_set)- def dimension("visit:region", _query, select_alias),- do: dynamic([t], selected_as(t.region, ^select_alias))+ def dimension("visit:browser_version", _query),+ do: field_or_blank_value(t.browser_version, @not_set)- def dimension("visit:city", _query, select_alias),- do: dynamic([t], selected_as(t.city, ^select_alias))+ # :TODO: Locations also set extra filters+ def dimension("visit:country", _query), do: dynamic([t], t.country)+ def dimension("visit:region", _query), do: dynamic([t], t.region)+ def dimension("visit:city", _query), do: dynamic([t], t.city)defmacro event_goal_join(events, page_regexes) doquote dodiff --git a/lib/plausible/stats/sql/query_builder.ex b/lib/plausible/stats/sql/query_builder.exindex 7e5e49a887..40db95f28f 100644--- a/lib/plausible/stats/sql/query_builder.ex+++ b/lib/plausible/stats/sql/query_builder.ex@@ -7,7 +7,7 @@ defmodule Plausible.Stats.SQL.QueryBuilder doimport Plausible.Stats.Importedimport Plausible.Stats.Util- alias Plausible.Stats.{Base, Query, QueryOptimizer, TableDecider, Filters}+ alias Plausible.Stats.{Base, Query, QueryOptimizer, TableDecider, Filters, Metrics}alias Plausible.Stats.SQL.Expressionrequire Plausible.Stats.SQL.Expression@@ -44,7 +44,6 @@ defmodule Plausible.Stats.SQL.QueryBuilder do|> merge_imported(site, events_query, events_query.metrics)|> maybe_add_global_conversion_rate(site, events_query)|> maybe_add_group_conversion_rate(site, events_query)- |> Base.add_percentage_metric(site, events_query, events_query.metrics)enddefp join_sessions_if_needed(q, site, query) do@@ -85,9 +84,6 @@ defmodule Plausible.Stats.SQL.QueryBuilder do|> join_events_if_needed(site, sessions_query)|> build_group_by(sessions_query)|> merge_imported(site, sessions_query, sessions_query.metrics)- |> maybe_add_global_conversion_rate(site, sessions_query)- |> maybe_add_group_conversion_rate(site, sessions_query)- |> Base.add_percentage_metric(site, sessions_query, sessions_query.metrics)enddef join_events_if_needed(q, site, query) do@@ -127,23 +123,36 @@ defmodule Plausible.Stats.SQL.QueryBuilder do^shortname(query, dimension) => fragment("?", goal)},group_by: goal,- where: goal != 0 and (e.name == "pageview" or goal < 0)+ where: goal != 0)enddefp dimension_group_by(q, query, dimension) do- key = shortname(query, dimension)-q- |> select_merge(^%{key => Expression.dimension(dimension, query, key)})- |> group_by([], selected_as(^key))+ |> select_merge(^%{shortname(query, dimension) => Expression.dimension(dimension, query)})+ |> group_by(^Expression.dimension(dimension, query))+ end++ defp build_order_by(q, query, mode) do+ Enum.reduce(query.order_by || [], q, &build_order_by(&2, query, &1, mode))end- defp build_order_by(q, query) do- Enum.reduce(query.order_by || [], q, &build_order_by(&2, query, &1))+ def build_order_by(q, query, {metric_or_dimension, order_direction}, :inner) do+ order_by(+ q,+ [t],+ ^{+ order_direction,+ if(+ Metrics.metric?(metric_or_dimension),+ do: dynamic([], selected_as(^shortname(query, metric_or_dimension))),+ else: Expression.dimension(metric_or_dimension, query)+ )+ }+ )end- def build_order_by(q, query, {metric_or_dimension, order_direction}) do+ def build_order_by(q, query, {metric_or_dimension, order_direction}, :outer) doorder_by(q,[t],@@ -253,10 +262,10 @@ defmodule Plausible.Stats.SQL.QueryBuilder dodefp join_query_results({nil, _}, {nil, _}), do: nildefp join_query_results({events_q, events_query}, {nil, _}),- do: events_q |> build_order_by(events_query)+ do: events_q |> build_order_by(events_query, :inner)- defp join_query_results({nil, events_query}, {sessions_q, _}),- do: sessions_q |> build_order_by(events_query)+ defp join_query_results({nil, _}, {sessions_q, sessions_query}),+ do: sessions_q |> build_order_by(sessions_query, :inner)defp join_query_results({events_q, events_query}, {sessions_q, sessions_query}) dojoin(subquery(events_q), :left, [e], s in subquery(sessions_q),@@ -265,12 +274,12 @@ defmodule Plausible.Stats.SQL.QueryBuilder do|> select_join_fields(events_query, events_query.dimensions, e)|> select_join_fields(events_query, events_query.metrics, e)|> select_join_fields(sessions_query, List.delete(sessions_query.metrics, :sample_percent), s)- |> build_order_by(events_query)+ |> build_order_by(events_query, :outer)end- def build_group_by_join(%Query{dimensions: []}), do: true+ defp build_group_by_join(%Query{dimensions: []}), do: true- def build_group_by_join(query) do+ defp build_group_by_join(query) doquery.dimensions|> Enum.map(fn dim ->dynamic([e, s], field(e, ^shortname(query, dim)) == field(s, ^shortname(query, dim)))diff --git a/lib/plausible/stats/table_decider.ex b/lib/plausible/stats/table_decider.exindex 36302613c0..c0176d417f 100644--- a/lib/plausible/stats/table_decider.ex+++ b/lib/plausible/stats/table_decider.ex@@ -29,42 +29,34 @@ defmodule Plausible.Stats.TableDecider do|> filter_keys()|> partition(query, &filters_partitioner/2)- %{event: event_only_dimensions, session: session_only_dimensions} =- partition(query.dimensions, query, &filters_partitioner/2)-cond do# Only one table needs to be queried- empty?(event_only_metrics) && empty?(event_only_filters) && empty?(event_only_dimensions) ->+ empty?(event_only_metrics) && empty?(event_only_filters) ->{[], session_only_metrics ++ either_metrics ++ sample_percent, other_metrics}- empty?(session_only_metrics) && empty?(session_only_filters) &&- empty?(session_only_dimensions) ->+ empty?(session_only_metrics) && empty?(session_only_filters) ->{event_only_metrics ++ either_metrics ++ sample_percent, [], other_metrics}- # Filters and/or dimensions on both events and sessions, but only one kind of metric- empty?(event_only_metrics) && empty?(event_only_dimensions) ->+ # Filters on both events and sessions, but only one kind of metric+ empty?(event_only_metrics) ->{[], session_only_metrics ++ either_metrics ++ sample_percent, other_metrics}- empty?(session_only_metrics) && empty?(session_only_dimensions) ->+ empty?(session_only_metrics) ->{event_only_metrics ++ either_metrics ++ sample_percent, [], other_metrics}- # Default: prefer events+ # Default: prefer sessionstrue ->- {event_only_metrics ++ either_metrics ++ sample_percent,- session_only_metrics ++ sample_percent, other_metrics}+ {event_only_metrics ++ sample_percent,+ session_only_metrics ++ either_metrics ++ sample_percent, other_metrics}endenddefp filter_keys(query) doquery.filters|> Enum.map(fn [_, filter_key | _rest] -> filter_key end)+ |> Enum.concat(query.dimensions)end- defp metric_partitioner(%Query{v2: true}, :conversion_rate), do: :either- defp metric_partitioner(%Query{v2: true}, :group_conversion_rate), do: :either- defp metric_partitioner(%Query{v2: true}, :visitors), do: :either- defp metric_partitioner(%Query{v2: true}, :visits), do: :either-defp metric_partitioner(_, :conversion_rate), do: :eventdefp metric_partitioner(_, :group_conversion_rate), do: :eventdefp metric_partitioner(_, :average_revenue), do: :event@@ -84,7 +76,7 @@ defmodule Plausible.Stats.TableDecider do# Calculated metrics - handled on callsite separately from other metrics.defp metric_partitioner(_, :time_on_page), do: :otherdefp metric_partitioner(_, :total_visitors), do: :other- defp metric_partitioner(_, :percentage), do: :either+ defp metric_partitioner(_, :percentage), do: :other# Sample percentage is included in both tables if queried.defp metric_partitioner(_, :sample_percent), do: :sample_percentdiff --git a/test/plausible/imported/csv_importer_test.exs b/test/plausible/imported/csv_importer_test.exsindex 3bb6f8ca08..18087bddf7 100644--- a/test/plausible/imported/csv_importer_test.exs+++ b/test/plausible/imported/csv_importer_test.exs@@ -622,7 +622,7 @@ defmodule Plausible.Imported.CSVImporterTest docase params_or_site do%Plausible.Site{} = site ->common_params.(site)- |> Map.put("metrics", "visitors,visits,visit_duration,bounce_rate")+ |> Map.put("metrics", "visitors,visits,pageviews,visit_duration,bounce_rate")|> Map.put("limit", 1000)|> Map.put("property", by)@@ -669,7 +669,7 @@ defmodule Plausible.Imported.CSVImporterTest doassert exported["pageviews"] == imported["pageviews"]assert exported["bounce_rate"] == imported["bounce_rate"]assert_in_delta exported["visitors"], imported["visitors"], 1- assert_in_delta exported["visits"], imported["visits"], 1+ assert exported["visits"] == imported["visits"]assert_in_delta exported["visit_duration"], imported["visit_duration"], 1end)@@ -810,7 +810,7 @@ defmodule Plausible.Imported.CSVImporterTest do_no_diff = 0endend)- ) == [0.0, 0.0, 0.0, 0.0, 0.0]+ ) == [0.0, 0.0, 0.0, 0.0, 0.03614457831325302]# NOTE: city breakdown's visit duration difference is up to 14%assert summary(field(exported_cities, "visit_duration")) == [0, 0.0, 0.0, 1.0, 1718]@@ -829,7 +829,7 @@ defmodule Plausible.Imported.CSVImporterTest do_no_diff = 0endend)- ) == [0, 0.0, 0.0, 0.0, 0.0]+ ) == [0, 0.0, 0.0, 0.0, 0.1428571428571429]# NOTE: city breakdown's visitors relative difference is up to 27%assert summary(field(exported_cities, "visitors")) == [1, 1.0, 1.0, 2.0, 22]diff --git a/test/plausible/imported/google_analytics4_test.exs b/test/plausible/imported/google_analytics4_test.exsindex 728378787e..7253877c4c 100644--- a/test/plausible/imported/google_analytics4_test.exs+++ b/test/plausible/imported/google_analytics4_test.exs@@ -117,7 +117,7 @@ defmodule Plausible.Imported.GoogleAnalytics4Test dobreakdown_params =common_params- |> Map.put("metrics", "visitors,visits,visit_duration,bounce_rate")+ |> Map.put("metrics", "visitors,visits,pageviews,visit_duration,bounce_rate")|> Map.put("limit", 1000)%{key: api_key} = insert(:api_key, user: user)@@ -467,6 +467,7 @@ defmodule Plausible.Imported.GoogleAnalytics4Test doassert List.first(results) == %{"bounce_rate" => 35.0,+ "pageviews" => 6229,"visit_duration" => 40.0,"visitors" => 4671,"visits" => 4917,@@ -474,11 +475,12 @@ defmodule Plausible.Imported.GoogleAnalytics4Test do}assert List.last(results) == %{- "bounce_rate" => 0.0,- "source" => "yahoo",- "visit_duration" => 41.0,+ "bounce_rate" => 100.0,+ "pageviews" => 1,+ "visit_duration" => 0.0,"visitors" => 1,- "visits" => 1+ "visits" => 1,+ "source" => "petalsearch.com"}end@@ -491,6 +493,7 @@ defmodule Plausible.Imported.GoogleAnalytics4Test doassert [%{"bounce_rate" => 35.0,+ "pageviews" => 6399,"utm_medium" => "organic","visit_duration" => 40.0,"visitors" => 4787,@@ -498,6 +501,7 @@ defmodule Plausible.Imported.GoogleAnalytics4Test do},%{"bounce_rate" => 58.0,+ "pageviews" => 491,"utm_medium" => "referral","visit_duration" => 27.0,"visitors" => 294,@@ -516,6 +520,7 @@ defmodule Plausible.Imported.GoogleAnalytics4Test doassert List.first(results) == %{"bounce_rate" => 35.0,+ "pageviews" => 838,"visit_duration" => 43.0,"visitors" => 675,"visits" => 712,@@ -524,10 +529,11 @@ defmodule Plausible.Imported.GoogleAnalytics4Test doassert List.last(results) == %{"bounce_rate" => 0.0,- "entry_page" => "/znamenitosti-rima-koje-treba-vidjeti",- "visit_duration" => 40.0,+ "pageviews" => 1,+ "visit_duration" => 27.0,"visitors" => 1,- "visits" => 1+ "visits" => 1,+ "entry_page" => "/kad-lisce-pada"}end@@ -537,11 +543,12 @@ defmodule Plausible.Imported.GoogleAnalytics4Test do%{"results" => results} =get(conn, "/api/v1/stats/breakdown", params) |> json_response(200)- assert length(results) == 494+ assert length(results) == 488assert List.first(results) == %{"bounce_rate" => 35.0,"city" => 792_680,+ "pageviews" => 1650,"visit_duration" => 39.0,"visitors" => 1233,"visits" => 1273@@ -549,8 +556,9 @@ defmodule Plausible.Imported.GoogleAnalytics4Test doassert List.last(results) == %{"bounce_rate" => 0.0,- "city" => 11_951_298,- "visit_duration" => 271.0,+ "city" => 4_399_605,+ "pageviews" => 7,+ "visit_duration" => 128.0,"visitors" => 1,"visits" => 1}@@ -566,6 +574,7 @@ defmodule Plausible.Imported.GoogleAnalytics4Test doassert List.first(results) == %{"bounce_rate" => 38.0,+ "pageviews" => 7041,"visit_duration" => 37.0,"visitors" => 5277,"visits" => 5532,@@ -574,6 +583,7 @@ defmodule Plausible.Imported.GoogleAnalytics4Test doassert List.last(results) == %{"bounce_rate" => 37.0,+ "pageviews" => 143,"visit_duration" => 60.0,"visitors" => 97,"visits" => 100,@@ -591,6 +601,7 @@ defmodule Plausible.Imported.GoogleAnalytics4Test doassert List.first(results) == %{"bounce_rate" => 33.0,+ "pageviews" => 8143,"visit_duration" => 50.0,"visitors" => 4625,"visits" => 4655,@@ -599,6 +610,7 @@ defmodule Plausible.Imported.GoogleAnalytics4Test doassert List.last(results) == %{"bounce_rate" => 0.0,+ "pageviews" => 6,"visit_duration" => 0.0,"visitors" => 1,"visits" => 1,@@ -616,6 +628,7 @@ defmodule Plausible.Imported.GoogleAnalytics4Test doassert List.first(results) == %{"bounce_rate" => 34.0,+ "pageviews" => 5827,"visit_duration" => 41.0,"visitors" => 4319,"visits" => 4495,@@ -624,6 +637,7 @@ defmodule Plausible.Imported.GoogleAnalytics4Test doassert List.last(results) == %{"bounce_rate" => 0.0,+ "pageviews" => 6,"visit_duration" => 0.0,"visitors" => 1,"visits" => 1,@@ -643,6 +657,7 @@ defmodule Plausible.Imported.GoogleAnalytics4Test do"bounce_rate" => 32.0,"os" => "Android","os_version" => "13.0.0",+ "pageviews" => 1673,"visit_duration" => 42.0,"visitors" => 1247,"visits" => 1295@@ -650,16 +665,17 @@ defmodule Plausible.Imported.GoogleAnalytics4Test doassert List.last(results) == %{"bounce_rate" => 0.0,- "os" => "Chrome OS",- "os_version" => "x86_64 15662.76.0",- "visit_duration" => 16.0,+ "os" => "iOS",+ "os_version" => "15.1",+ "pageviews" => 1,+ "visit_duration" => 54.0,"visitors" => 1,"visits" => 1}enddefp assert_pages(conn, params) do- metrics = "visitors,visits,time_on_page,visit_duration,bounce_rate"+ metrics = "visitors,visits,pageviews,time_on_page,visit_duration,bounce_rate"params =params@@ -670,7 +686,7 @@ defmodule Plausible.Imported.GoogleAnalytics4Test do%{"results" => results} =get(conn, "/api/v1/stats/breakdown", params) |> json_response(200)- assert length(results) == 730+ assert length(results) == 729# The `event:page` breakdown is currently using the `entry_page`# property to allow querying session metrics.@@ -680,6 +696,7 @@ defmodule Plausible.Imported.GoogleAnalytics4Test do# it will allow us to assert on the session metrics as well.assert Enum.at(results, 2) == %{"page" => "/",+ "pageviews" => 5537,"time_on_page" => 17.677262055264585,"visitors" => 371,"visits" => 212,@@ -690,12 +707,13 @@ defmodule Plausible.Imported.GoogleAnalytics4Test do# This page was never an entry_page in the imported data, and# therefore the session metrics are returned as `nil`.assert List.last(results) == %{- "bounce_rate" => 0.0,- "page" => "/znamenitosti-rima-koje-treba-vidjeti/",- "time_on_page" => 40.0,- "visit_duration" => 0.0,+ "page" => "/5-dobrih-razloga-zasto-zapoceti-dan-zobenom-kasom/",+ "pageviews" => 2,+ "time_on_page" => 10.0,"visitors" => 1,- "visits" => 1+ "visits" => 1,+ "bounce_rate" => nil,+ "visit_duration" => nil}endenddiff --git a/test/plausible/stats/table_decider_test.exs b/test/plausible/stats/table_decider_test.exsindex b216d02c7d..7226542c17 100644--- a/test/plausible/stats/table_decider_test.exs+++ b/test/plausible/stats/table_decider_test.exs@@ -76,7 +76,7 @@ defmodule Plausible.Stats.TableDeciderTest doquery = make_query(false, [])assert partition_metrics([:time_on_page, :percentage, :total_visitors], query) ==- {[], [:percentage], [:time_on_page, :total_visitors]}+ {[], [], [:time_on_page, :percentage, :total_visitors]}endtest "raises if unknown metric" do@@ -108,11 +108,11 @@ defmodule Plausible.Stats.TableDeciderTest do{[], [:visit_duration, :visits], []}end- test "metrics that can be calculated on either are biased to events" do+ test "metrics that can be calculated on either are biased to sessions" doquery = make_query(true, [])assert partition_metrics([:bounce_rate, :total_revenue, :visitors], query) ==- {[:total_revenue, :visitors], [:bounce_rate], []}+ {[:total_revenue], [:bounce_rate, :visitors], []}endtest "sample_percent is handled with either metrics" dodiff --git a/test/plausible_web/controllers/CSVs/30d-filter-path/pages.csv b/test/plausible_web/controllers/CSVs/30d-filter-path/pages.csvindex 4741db70e8..1cef2d8c42 100644--- a/test/plausible_web/controllers/CSVs/30d-filter-path/pages.csv+++ b/test/plausible_web/controllers/CSVs/30d-filter-path/pages.csv@@ -1,2 +1,2 @@name,visitors,pageviews,bounce_rate,time_on_page-/some-other-page,1,1,0,60.0+/some-other-page,1,1,,60.0diff --git a/test/plausible_web/controllers/CSVs/30d/pages.csv b/test/plausible_web/controllers/CSVs/30d/pages.csvindex 62b246c420..e2990caa97 100644--- a/test/plausible_web/controllers/CSVs/30d/pages.csv+++ b/test/plausible_web/controllers/CSVs/30d/pages.csv@@ -1,4 +1,4 @@name,visitors,pageviews,bounce_rate,time_on_page/,4,3,67,/signup,1,1,0,60.0-/some-other-page,1,1,0,60.0+/some-other-page,1,1,,60.0diff --git a/test/plausible_web/controllers/CSVs/6m/pages.csv b/test/plausible_web/controllers/CSVs/6m/pages.csvindex ed895e036f..8051585445 100644--- a/test/plausible_web/controllers/CSVs/6m/pages.csv+++ b/test/plausible_web/controllers/CSVs/6m/pages.csv@@ -1,4 +1,4 @@name,visitors,pageviews,bounce_rate,time_on_page/,5,4,75,/signup,1,1,0,60.0-/some-other-page,1,1,0,60.0+/some-other-page,1,1,,60.0diff --git a/test/plausible_web/controllers/api/external_stats_controller/breakdown_test.exs b/test/plausible_web/controllers/api/external_stats_controller/breakdown_test.exsindex d62cdf0610..8be42e3187 100644--- a/test/plausible_web/controllers/api/external_stats_controller/breakdown_test.exs+++ b/test/plausible_web/controllers/api/external_stats_controller/breakdown_test.exs@@ -890,8 +890,8 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest doassert json_response(conn, 200) == %{"results" => [%{"page" => "/", "pageviews" => 2},- %{"page" => "/include-me", "pageviews" => 1},- %{"page" => "/plausible.io", "pageviews" => 1}+ %{"page" => "/plausible.io", "pageviews" => 1},+ %{"page" => "/include-me", "pageviews" => 1}]}end@@ -1023,7 +1023,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do"period" => "day","date" => "2021-01-01","property" => "visit:exit_page",- "metrics" => "visitors,visits,bounce_rate,visit_duration",+ "metrics" => "visitors,visits,pageviews,bounce_rate,visit_duration,events","with_imported" => "true"})@@ -1031,14 +1031,18 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do"results" => [%{"bounce_rate" => 0.0,+ "events" => 7,"exit_page" => "/b",+ "pageviews" => 7,"visit_duration" => 150.0,"visitors" => 3,"visits" => 4},%{"bounce_rate" => 100.0,+ "events" => 1,"exit_page" => "/a",+ "pageviews" => 1,"visit_duration" => 0.0,"visitors" => 1,"visits" => 1@@ -2172,8 +2176,8 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest doassert json_response(conn, 200) == %{"results" => [- %{"page" => "/important-page", "bounce_rate" => 100},- %{"page" => "/plausible.io", "bounce_rate" => 100}+ %{"page" => "/plausible.io", "bounce_rate" => 100},+ %{"page" => "/important-page", "bounce_rate" => 100}]}end@@ -2592,14 +2596,14 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest doassert json_response(conn, 200) == %{"results" => [- %{- "page" => "/A",- "time_on_page" => 60.0- },%{"page" => "/B","time_on_page" => 90.0},+ %{+ "page" => "/A",+ "time_on_page" => 60.0+ },%{"page" => "/C","time_on_page" => nil@@ -3041,13 +3045,13 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest doassert json_response(conn, 200) == %{"results" => [- %{- "entry_page" => "/entry-page-2",- "bounce_rate" => 100- },%{"entry_page" => "/entry-page-1","bounce_rate" => 0+ },+ %{+ "entry_page" => "/entry-page-2",+ "bounce_rate" => 100}]}@@ -3142,7 +3146,6 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest do},%{"page" => "/plausible.io",- # Breaks for event:page breakdown since visitors is calculated based on entry_page :/"visitors" => 2,"bounce_rate" => 100,"visit_duration" => 0,@@ -3287,6 +3290,8 @@ defmodule PlausibleWeb.Api.ExternalStatsController.BreakdownTest doassert %{"browser" => "Chrome", "events" => 1} = breakdown_and_first.("visit:browser")assert %{"device" => "Desktop", "events" => 1} = breakdown_and_first.("visit:device")+ assert %{"entry_page" => "/test", "events" => 1} = breakdown_and_first.("visit:entry_page")+ assert %{"exit_page" => "/test", "events" => 1} = breakdown_and_first.("visit:exit_page")assert %{"country" => "EE", "events" => 1} = breakdown_and_first.("visit:country")assert %{"os" => "Mac", "events" => 1} = breakdown_and_first.("visit:os")assert %{"page" => "/test", "events" => 1} = breakdown_and_first.("event:page")diff --git a/test/plausible_web/controllers/api/external_stats_controller/query_test.exs b/test/plausible_web/controllers/api/external_stats_controller/query_test.exsindex 331687e1f0..cc4aec1ba6 100644--- a/test/plausible_web/controllers/api/external_stats_controller/query_test.exs+++ b/test/plausible_web/controllers/api/external_stats_controller/query_test.exs@@ -1477,7 +1477,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest doconn =post(conn, "/api/v2/query", %{"site_id" => site.domain,- "metrics" => ["visitors", "percentage"],+ "metrics" => ["visitors"],"date_range" => ["2021-01-01", "2021-01-01"],"dimensions" => [unquote(dimension)]})@@ -1485,9 +1485,9 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest do%{"results" => results} = json_response(conn, 200)assert results == [- %{"dimensions" => [unquote(value1)], "metrics" => [3, 50]},- %{"dimensions" => [unquote(value2)], "metrics" => [2, 33.3]},- %{"dimensions" => [unquote(blank_value)], "metrics" => [1, 16.7]}+ %{"dimensions" => [unquote(value1)], "metrics" => [3]},+ %{"dimensions" => [unquote(value2)], "metrics" => [2]},+ %{"dimensions" => [unquote(blank_value)], "metrics" => [1]}]endend@@ -3463,48 +3463,6 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest do%{"dimensions" => ["Chrome"], "metrics" => [1]}]end-- test "all metrics for breakdown by event prop", %{conn: conn, site: site} do- populate_stats(site, [- build(:pageview,- user_id: 1,- pathname: "/",- timestamp: ~N[2021-01-01 00:00:00]- ),- build(:pageview,- user_id: 1,- pathname: "/plausible.io",- timestamp: ~N[2021-01-01 00:10:00]- ),- build(:pageview, pathname: "/", timestamp: ~N[2021-01-01 00:25:00]),- build(:pageview,- pathname: "/plausible.io",- timestamp: ~N[2021-01-01 00:00:00]- )- ])-- conn =- post(conn, "/api/v2/query", %{- "site_id" => site.domain,- "metrics" => [- "visitors",- "visits",- "pageviews",- "events",- "bounce_rate",- "visit_duration"- ],- "date_range" => "all",- "dimensions" => ["event:page"]- })-- %{"results" => results} = json_response(conn, 200)-- assert results == [- %{"dimensions" => ["/"], "metrics" => [2, 2, 2, 2, 50, 300]},- %{"dimensions" => ["/plausible.io"], "metrics" => [2, 2, 2, 2, 100, 0]}- ]- endenddescribe "imported data" do@@ -3631,6 +3589,10 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest doassert %{"dimensions" => ["Desktop"], "metrics" => [1]} =breakdown_and_first.("visit:device")+ # :TODO: These should not pass validation - not available on events.+ # visit dimension and event-only metric+ # assert %{"dimensions" => ["/test"], "metrics" => [1]} = breakdown_and_first.("visit:entry_page")+ # assert %{"dimensions" => ["/test"], "metrics" => [1]} = breakdown_and_first.("visit:exit_page")assert %{"dimensions" => ["EE"], "metrics" => [1]} = breakdown_and_first.("visit:country")assert %{"dimensions" => ["Mac"], "metrics" => [1]} = breakdown_and_first.("visit:os")assert %{"dimensions" => ["/test"], "metrics" => [1]} = breakdown_and_first.("event:page")@@ -3903,56 +3865,6 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest doassert meta["warning"] =~ "Imported stats are not included in the results"end-- test "imported country, region and city data",- %{- conn: conn,- site: site- } do- site_import = insert(:site_import, site: site)-- populate_stats(site, site_import.id, [- build(:pageview,- timestamp: ~N[2021-01-01 00:15:00],- country_code: "DE",- subdivision1_code: "DE-BE",- city_geoname_id: 2_950_159- ),- build(:pageview,- timestamp: ~N[2021-01-01 00:15:00],- country_code: "DE",- subdivision1_code: "DE-BE",- city_geoname_id: 2_950_159- ),- build(:pageview,- timestamp: ~N[2021-01-01 00:15:00],- country_code: "EE",- subdivision1_code: "EE-37",- city_geoname_id: 588_409- ),- build(:imported_locations, country: "EE", region: "EE-37", city: 588_409, visitors: 33)- ])-- for {dimension, stats_value, imports_value} <- [- {"visit:country", "DE", "EE"},- {"visit:region", "DE-BE", "EE-37"},- {"visit:city", 2_950_159, 588_409}- ] do- conn =- post(conn, "/api/v2/query", %{- "site_id" => site.domain,- "metrics" => ["visitors"],- "date_range" => "all",- "dimensions" => [dimension],- "include" => %{"imports" => true}- })-- assert json_response(conn, 200)["results"] == [- %{"dimensions" => [imports_value], "metrics" => [34]},- %{"dimensions" => [stats_value], "metrics" => [2]}- ]- end- endendtest "multiple breakdown timeseries with sources", %{conn: conn, site: site} dodiff --git a/test/plausible_web/controllers/api/stats_controller/conversions_test.exs b/test/plausible_web/controllers/api/stats_controller/conversions_test.exsindex a5bcdb38b2..24e18ed2da 100644--- a/test/plausible_web/controllers/api/stats_controller/conversions_test.exs+++ b/test/plausible_web/controllers/api/stats_controller/conversions_test.exs@@ -207,7 +207,6 @@ defmodule PlausibleWeb.Api.StatsController.ConversionsTest do]end- @tag capture_log: truetest "garbage filters don't crash the call", %{conn: conn, site: site} dofilters ="{\"source\":\"Direct / None\",\"screen\":\"Desktop\",\"browser\":\"Chrome\",\"os\":\"Mac\",\"os_version\":\"10.15\",\"country\":\"DE\",\"city\":\"2950159\"}%' AND 2*3*8=6*8 AND 'L9sv'!='L9sv%"diff --git a/test/plausible_web/controllers/api/stats_controller/imported_test.exs b/test/plausible_web/controllers/api/stats_controller/imported_test.exsindex 16e7a35f92..370eb0ecf6 100644--- a/test/plausible_web/controllers/api/stats_controller/imported_test.exs+++ b/test/plausible_web/controllers/api/stats_controller/imported_test.exs@@ -602,15 +602,15 @@ defmodule PlausibleWeb.Api.StatsController.ImportedTest doassert json_response(conn, 200)["results"] == [%{- "name" => "ad",+ "name" => "blog","visitors" => 2,- "bounce_rate" => 100.0,+ "bounce_rate" => 50.0,"visit_duration" => 50.0},%{- "name" => "blog",+ "name" => "ad","visitors" => 2,- "bounce_rate" => 50.0,+ "bounce_rate" => 100.0,"visit_duration" => 50.0}]@@ -708,7 +708,7 @@ defmodule PlausibleWeb.Api.StatsController.ImportedTest doassert json_response(conn, 200)["results"] == [%{- "bounce_rate" => 0,+ "bounce_rate" => nil,"time_on_page" => 60,"visitors" => 3,"pageviews" => 4,diff --git a/test/plausible_web/controllers/api/stats_controller/pages_test.exs b/test/plausible_web/controllers/api/stats_controller/pages_test.exsindex 6a55d0bf84..4af5c5cd0b 100644--- a/test/plausible_web/controllers/api/stats_controller/pages_test.exs+++ b/test/plausible_web/controllers/api/stats_controller/pages_test.exs@@ -340,7 +340,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do"name" => "/blog/other-post","visitors" => 1,"pageviews" => 1,- "bounce_rate" => 0,+ "bounce_rate" => nil,"time_on_page" => nil}]@@ -392,7 +392,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do"name" => "/blog/other-post","visitors" => 1,"pageviews" => 1,- "bounce_rate" => 0,+ "bounce_rate" => nil,"time_on_page" => nil}]@@ -744,7 +744,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do"name" => "/blog/post-2","visitors" => 1,"pageviews" => 1,- "bounce_rate" => 0,+ "bounce_rate" => nil,"time_on_page" => nil}]@@ -789,7 +789,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do"name" => "/blog/(/post-2","visitors" => 1,"pageviews" => 1,- "bounce_rate" => 0,+ "bounce_rate" => nil,"time_on_page" => nil}]@@ -842,7 +842,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do"name" => "/about","visitors" => 1,"pageviews" => 1,- "bounce_rate" => 0,+ "bounce_rate" => nil,"time_on_page" => nil}]@@ -940,7 +940,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do"name" => "/"},%{- "bounce_rate" => 0,+ "bounce_rate" => nil,"time_on_page" => nil,"visitors" => 1,"pageviews" => 1,@@ -1066,7 +1066,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do"visitors" => 2},%{- "bounce_rate" => 0,+ "bounce_rate" => nil,"name" => "/exit-blog","pageviews" => 1,"time_on_page" => nil,@@ -1192,7 +1192,7 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do"name" => "/"},%{- "bounce_rate" => 0,+ "bounce_rate" => nil,"time_on_page" => 60,"visitors" => 2,"pageviews" => 2,diff --git a/test/plausible_web/controllers/api/stats_controller/sources_test.exs b/test/plausible_web/controllers/api/stats_controller/sources_test.exsindex ea7b9c15a2..a76b1da274 100644--- a/test/plausible_web/controllers/api/stats_controller/sources_test.exs+++ b/test/plausible_web/controllers/api/stats_controller/sources_test.exs@@ -453,9 +453,6 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest doreferrer_source: "DuckDuckGo",referrer: "duckduckgo.com"),- build(:imported_sources,- source: "DuckDuckGo"- ),build(:imported_sources,source: "DuckDuckGo")@@ -470,7 +467,7 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest doconn = get(conn, "/api/stats/#{site.domain}/sources?limit=1&page=2&with_imported=true")assert json_response(conn, 200)["results"] == [- %{"name" => "Google", "visitors" => 2}+ %{"name" => "DuckDuckGo", "visitors" => 2}]end@@ -593,17 +590,17 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest do)assert json_response(conn, 200)["results"] == [- %{- "name" => "email",- "visitors" => 1,- "bounce_rate" => 100,- "visit_duration" => 0- },%{"name" => "social","visitors" => 1,"bounce_rate" => 0,"visit_duration" => 900+ },+ %{+ "name" => "email",+ "visitors" => 1,+ "bounce_rate" => 100,+ "visit_duration" => 0}]@@ -615,16 +612,16 @@ defmodule PlausibleWeb.Api.StatsController.SourcesTest doassert json_response(conn, 200)["results"] == [%{- "name" => "email",+ "name" => "social","visitors" => 2,"bounce_rate" => 50,- "visit_duration" => 50+ "visit_duration" => 800.0},%{- "name" => "social",+ "name" => "email","visitors" => 2,"bounce_rate" => 50,- "visit_duration" => 800.0+ "visit_duration" => 50}]end