Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.
TypeScript analyticsanalytics-dashboardclickhouseelixirgoogle-analyticsmarketingopen-source-analyticsphoenixplausible-analyticspostgresqlprivacyprivacy-friendlyself-hostedsimple-analyticsstatisticstailwindcssweb-analyticswebsite-analyticswebsite-statswebsite-tracking
APIv2: Aggregates, timeseries, conversion_rate, hostname (#4251)
Karl-Aksel Puulmann2eeaf7a15287
diff --git a/lib/plausible/stats/aggregate.ex b/lib/plausible/stats/aggregate.exindex 82d1c8a180..9a2e65bab6 100644--- a/lib/plausible/stats/aggregate.ex+++ b/lib/plausible/stats/aggregate.ex@@ -15,10 +15,7 @@ defmodule Plausible.Stats.Aggregate doQuery.trace(query, metrics)- query_with_metrics = %Plausible.Stats.Query{- query- | metrics: Util.maybe_add_visitors_metric(metrics)- }+ query_with_metrics = %Query{query | metrics: metrics}q = Plausible.Stats.SQL.QueryBuilder.build(query_with_metrics, site)diff --git a/lib/plausible/stats/base.ex b/lib/plausible/stats/base.exindex bd01257fb7..3b3dfd2e33 100644--- a/lib/plausible/stats/base.ex+++ b/lib/plausible/stats/base.ex@@ -121,6 +121,7 @@ defmodule Plausible.Stats.Base dodefp select_event_metric(:percentage), do: %{}defp select_event_metric(:conversion_rate), do: %{}+ defp select_event_metric(:group_conversion_rate), do: %{}defp select_event_metric(:total_visitors), do: %{}defp select_event_metric(unknown), do: raise("Unknown metric: #{unknown}")@@ -344,9 +345,9 @@ defmodule Plausible.Stats.Base do# only if it's included in the base query - otherwise the total will be based on# a different data set, making the metric inaccurate. This is why we're using an# explicit `include_imported` argument here.- defp total_visitors_subquery(site, query, include_imported)+ def total_visitors_subquery(site, query, include_imported)- defp total_visitors_subquery(site, query, true = _include_imported) do+ def total_visitors_subquery(site, query, true = _include_imported) dodynamic([e],selected_as(@@ -357,7 +358,7 @@ defmodule Plausible.Stats.Base do)end- defp total_visitors_subquery(site, query, false = _include_imported) do+ def total_visitors_subquery(site, query, false = _include_imported) dodynamic([e], selected_as(subquery(total_visitors(site, query)), :__total_visitors))enddiff --git a/lib/plausible/stats/filters/filters.ex b/lib/plausible/stats/filters/filters.exindex dc13c7dcd9..b6760965ed 100644--- a/lib/plausible/stats/filters/filters.ex+++ b/lib/plausible/stats/filters/filters.ex@@ -87,6 +87,6 @@ defmodule Plausible.Stats.Filters doproperty|> String.split(":")|> List.last()- |> String.to_atom()+ |> String.to_existing_atom()endenddiff --git a/lib/plausible/stats/filters/query_parser.ex b/lib/plausible/stats/filters/query_parser.exindex 5023261899..0184478905 100644--- a/lib/plausible/stats/filters/query_parser.ex+++ b/lib/plausible/stats/filters/query_parser.ex@@ -5,13 +5,15 @@ defmodule Plausible.Stats.Filters.QueryParser doalias Plausible.Stats.Filtersalias Plausible.Stats.Query- def parse(site, params) when is_map(params) do+ def parse(site, params, now \\ nil) when is_map(params) dowith {:ok, metrics} <- parse_metrics(Map.get(params, "metrics", [])),{:ok, filters} <- parse_filters(Map.get(params, "filters", [])),- {:ok, date_range} <- parse_date_range(site, Map.get(params, "date_range")),+ {:ok, date_range} <-+ parse_date_range(site, Map.get(params, "date_range"), now || today(site)),{:ok, dimensions} <- parse_dimensions(Map.get(params, "dimensions", [])),{:ok, order_by} <- parse_order_by(Map.get(params, "order_by")),{:ok, include} <- parse_include(Map.get(params, "include", %{})),+ preloaded_goals <- preload_goals_if_needed(site, filters, dimensions),query = %{metrics: metrics,filters: filters,@@ -19,10 +21,11 @@ defmodule Plausible.Stats.Filters.QueryParser dodimensions: dimensions,order_by: order_by,timezone: site.timezone,- imported_data_requested: Map.get(include, :imports, false)+ imported_data_requested: Map.get(include, :imports, false),+ preloaded_goals: preloaded_goals},:ok <- validate_order_by(query),- :ok <- validate_goal_filters(site, query),+ :ok <- validate_goal_filters(query),:ok <- validate_custom_props_access(site, query),:ok <- validate_metrics(query) do{:ok, query}@@ -43,12 +46,14 @@ defmodule Plausible.Stats.Filters.QueryParser dodefp parse_metric("time_on_page"), do: {:ok, :time_on_page}defp parse_metric("conversion_rate"), do: {:ok, :conversion_rate}+ defp parse_metric("group_conversion_rate"), do: {:ok, :group_conversion_rate}defp parse_metric("visitors"), do: {:ok, :visitors}defp parse_metric("pageviews"), do: {:ok, :pageviews}defp parse_metric("events"), do: {:ok, :events}defp parse_metric("visits"), do: {:ok, :visits}defp parse_metric("bounce_rate"), do: {:ok, :bounce_rate}defp parse_metric("visit_duration"), do: {:ok, :visit_duration}+ defp parse_metric("views_per_visit"), do: {:ok, :views_per_visit}defp parse_metric(unknown_metric), do: {:error, "Unknown metric '#{inspect(unknown_metric)}'"}def parse_filters(filters) when is_list(filters) do@@ -84,7 +89,7 @@ defmodule Plausible.Stats.Filters.QueryParser dodo: parse_clauses_list(filter)defp parse_clauses_list([_operation, filter_key, list] = filter) when is_list(list) do- all_strings? = Enum.all?(list, &is_bitstring/1)+ all_strings? = Enum.all?(list, &is_binary/1)cond dofilter_key == "event:goal" && all_strings? -> {:ok, [Filters.Utils.wrap_goal_value(list)]}@@ -95,27 +100,62 @@ defmodule Plausible.Stats.Filters.QueryParser dodefp parse_clauses_list(filter), do: {:error, "Invalid filter '#{inspect(filter)}'"}- defp parse_date_range(site, "day") do- today = DateTime.now!(site.timezone) |> DateTime.to_date()- {:ok, Date.range(today, today)}+ defp parse_date_range(_site, "day", date) do+ {:ok, Date.range(date, date)}end- defp parse_date_range(_site, "7d"), do: {:ok, "7d"}- defp parse_date_range(_site, "30d"), do: {:ok, "30d"}- defp parse_date_range(_site, "month"), do: {:ok, "month"}- defp parse_date_range(_site, "6mo"), do: {:ok, "6mo"}- defp parse_date_range(_site, "12mo"), do: {:ok, "12mo"}- defp parse_date_range(_site, "year"), do: {:ok, "year"}+ defp parse_date_range(_site, "7d", last) do+ first = last |> Date.add(-6)+ {:ok, Date.range(first, last)}+ end++ defp parse_date_range(_site, "30d", last) do+ first = last |> Date.add(-30)+ {:ok, Date.range(first, last)}+ end++ defp parse_date_range(_site, "month", today) do+ last = today |> Date.end_of_month()+ first = last |> Date.beginning_of_month()+ {:ok, Date.range(first, last)}+ end++ defp parse_date_range(_site, "6mo", today) do+ last = today |> Date.end_of_month()++ first =+ last+ |> Timex.shift(months: -5)+ |> Date.beginning_of_month()++ {:ok, Date.range(first, last)}+ end- defp parse_date_range(site, "all") do- today = DateTime.now!(site.timezone) |> DateTime.to_date()+ defp parse_date_range(_site, "12mo", today) do+ last = today |> Date.end_of_month()++ first =+ last+ |> Timex.shift(months: -11)+ |> Date.beginning_of_month()++ {:ok, Date.range(first, last)}+ end++ defp parse_date_range(_site, "year", today) do+ last = today |> Timex.end_of_year()+ first = last |> Timex.beginning_of_year()+ {:ok, Date.range(first, last)}+ end++ defp parse_date_range(site, "all", today) dostart_date = Plausible.Sites.stats_start_date(site) || today{:ok, Date.range(start_date, today)}end- defp parse_date_range(_site, [from_date_string, to_date_string])- when is_bitstring(from_date_string) and is_bitstring(to_date_string) do+ defp parse_date_range(_site, [from_date_string, to_date_string], _date)+ when is_binary(from_date_string) and is_binary(to_date_string) dowith {:ok, from_date} <- Date.from_iso8601(from_date_string),{:ok, to_date} <- Date.from_iso8601(to_date_string) do{:ok, Date.range(from_date, to_date)}@@ -124,7 +164,10 @@ defmodule Plausible.Stats.Filters.QueryParser doendend- defp parse_date_range(_site, unknown), do: {:error, "Invalid date range '#{inspect(unknown)}'"}+ defp parse_date_range(_site, unknown, _),+ do: {:error, "Invalid date_range '#{inspect(unknown)}'"}++ defp today(site), do: DateTime.now!(site.timezone) |> DateTime.to_date()defp parse_dimensions(dimensions) when is_list(dimensions) doif length(dimensions) == length(Enum.uniq(dimensions)) do@@ -178,10 +221,9 @@ defmodule Plausible.Stats.Filters.QueryParser doenddefp parse_time("time"), do: {:ok, "time"}+ defp parse_time("time:hour"), do: {:ok, "time:hour"}defp parse_time("time:day"), do: {:ok, "time:day"}- defp parse_time("time:week"), do: {:ok, "time:week"}defp parse_time("time:month"), do: {:ok, "time:month"}- defp parse_time("time:year"), do: {:ok, "time:year"}defp parse_time(_), do: :errordefp parse_order_direction([_, "asc"]), do: {:ok, :asc}@@ -242,7 +284,22 @@ defmodule Plausible.Stats.Filters.QueryParser doendend- defp validate_goal_filters(site, query) do+ defp preload_goals_if_needed(site, filters, dimensions) do+ goal_filters? =+ Enum.any?(filters, fn [_, filter_key | _rest] -> filter_key == "event:goal" end)++ if goal_filters? or Enum.member?(dimensions, "event:goal") do+ Plausible.Goals.for_site(site)+ |> Enum.map(fn+ %{page_path: path} when is_binary(path) -> {:page, path}+ %{event_name: event_name} -> {:event, event_name}+ end)+ else+ []+ end+ end++ defp validate_goal_filters(query) dogoal_filter_clauses =Enum.flat_map(query.filters, fn[_operation, "event:goal", clauses] -> clauses@@ -250,14 +307,7 @@ defmodule Plausible.Stats.Filters.QueryParser doend)if length(goal_filter_clauses) > 0 do- configured_goals =- Plausible.Goals.for_site(site)- |> Enum.map(fn- %{page_path: path} when is_binary(path) -> {:page, path}- %{event_name: event_name} -> {:event, event_name}- end)-- validate_list(goal_filter_clauses, &validate_goal_filter(&1, configured_goals))+ validate_list(goal_filter_clauses, &validate_goal_filter(&1, query.preloaded_goals))else:okend@@ -298,14 +348,12 @@ defmodule Plausible.Stats.Filters.QueryParser doenddefp validate_metrics(query) do- validate_list(query.metrics, &validate_metric(&1, query))-with :ok <- validate_list(query.metrics, &validate_metric(&1, query)) dovalidate_no_metrics_filters_conflict(query)endend- defp validate_metric(:conversion_rate = metric, query) do+ defp validate_metric(metric, query) when metric in [:conversion_rate, :group_conversion_rate] doif Enum.member?(query.dimensions, "event:goal") ornot is_nil(Query.get_filter(query, "event:goal")) do:ok@@ -314,20 +362,42 @@ defmodule Plausible.Stats.Filters.QueryParser doendend+ defp validate_metric(:views_per_visit = metric, query) do+ cond do+ not is_nil(Query.get_filter(query, "event:page")) ->+ {:error, "Metric `#{metric}` cannot be queried with a filter on `event:page`"}++ length(query.dimensions) > 0 ->+ {:error, "Metric `#{metric}` cannot be queried with `dimensions`"}++ true ->+ :ok+ end+ end+defp validate_metric(_, _), do: :okdefp validate_no_metrics_filters_conflict(query) do{_event_metrics, sessions_metrics, _other_metrics} =TableDecider.partition_metrics(query.metrics, query)- if Enum.empty?(sessions_metrics) or not TableDecider.event_filters?(query) do+ if Enum.empty?(sessions_metrics) or+ not event_dimensions_not_allowing_session_metrics?(query.dimensions) do:okelse{:error,- "Session metric(s) `#{sessions_metrics |> Enum.join(", ")}` cannot be queried along with event filters or dimensions"}+ "Session metric(s) `#{sessions_metrics |> Enum.join(", ")}` cannot be queried along with event dimensions"}endend+ def event_dimensions_not_allowing_session_metrics?(dimensions) do+ Enum.any?(dimensions, fn+ "event:page" -> false+ "event:" <> _ -> true+ _ -> false+ end)+ end+defp parse_list(list, parser_function) doEnum.reduce_while(list, {:ok, []}, fn value, {:ok, results} ->case parser_function.(value) dodiff --git a/lib/plausible/stats/filters/utils.ex b/lib/plausible/stats/filters/utils.exindex 396b62e13e..96606c1d02 100644--- a/lib/plausible/stats/filters/utils.ex+++ b/lib/plausible/stats/filters/utils.ex@@ -66,4 +66,18 @@ defmodule Plausible.Stats.Filters.Utils dodef unwrap_goal_value(goals) when is_list(goals), do: Enum.map(goals, &unwrap_goal_value/1)def unwrap_goal_value({:page, page}), do: "Visit " <> pagedef unwrap_goal_value({:event, event}), do: event++ def split_goals(goals) do+ Enum.split_with(goals, fn {type, _} -> type == :event end)+ end++ def split_goals_query_expressions(goals) do+ {event_goals, pageview_goals} = split_goals(goals)+ events = Enum.map(event_goals, fn {_, event} -> event end)++ page_regexes =+ Enum.map(pageview_goals, fn {_, path} -> Plausible.Stats.Base.page_regex(path) end)++ {events, page_regexes}+ endenddiff --git a/lib/plausible/stats/imported/imported.ex b/lib/plausible/stats/imported/imported.exindex 5c1d77cc34..6542ab44f3 100644--- a/lib/plausible/stats/imported/imported.ex+++ b/lib/plausible/stats/imported/imported.ex@@ -1,4 +1,5 @@defmodule Plausible.Stats.Imported do+ alias Plausible.Stats.Filtersuse Plausible.ClickhouseRepoimport Ecto.Query@@ -266,6 +267,42 @@ defmodule Plausible.Stats.Imported dodef merge_imported(q, _, %Query{include_imported: false}, _), do: q+ # Note: Only called for APIv2, old APIs use merge_imported_pageview_goals+ def merge_imported(q, site, %Query{dimensions: ["event:goal"]} = query, metrics)+ when query.v2 do+ {events, page_regexes} = Filters.Utils.split_goals_query_expressions(query.preloaded_goals)++ events_q =+ "imported_custom_events"+ |> Imported.Base.query_imported(site, query)+ |> where([i], i.visitors > 0)+ |> select_merge([i], %{+ dim0: selected_as(fragment("-indexOf(?, ?)", ^events, i.name), :dim0)+ })+ |> select_imported_metrics(metrics)+ |> group_by([], selected_as(:dim0))+ |> where([], selected_as(:dim0) != 0)++ pages_q =+ "imported_pages"+ |> Imported.Base.query_imported(site, query)+ |> where([i], i.visitors > 0)+ |> where(+ [i],+ fragment("notEmpty(multiMatchAllIndices(?, ?) as indices)", i.page, ^page_regexes)+ )+ |> join(:array, index in fragment("indices"))+ |> group_by([_i, index], index)+ |> select_merge([_i, index], %{+ dim0: type(fragment("?", index), :integer)+ })+ |> select_imported_metrics(metrics)++ q+ |> naive_dimension_join(events_q, metrics)+ |> naive_dimension_join(pages_q, metrics)+ end+def merge_imported(q, site, %Query{dimensions: [dimension]} = query, metrics)when dimension in @imported_properties dodim = Plausible.Stats.Filters.without_prefix(dimension)@@ -289,7 +326,7 @@ defmodule Plausible.Stats.Imported dodynamic([s, i], s.browser == i.browser and s.browser_version == i.browser_version)dim ->- dynamic([s, i], field(s, ^dim) == field(i, ^dim))+ dynamic([s, i], field(s, ^shortname(query, dim)) == field(i, ^shortname(query, dim)))endfrom(s in Ecto.Query.subquery(q),@@ -299,7 +336,7 @@ defmodule Plausible.Stats.Imported do)|> select_joined_dimension(dim, query)|> select_joined_metrics(metrics)- |> apply_order_by(metrics)+ |> apply_order_by(query, metrics)enddef merge_imported(q, site, %Query{dimensions: []} = query, metrics) do@@ -357,6 +394,10 @@ defmodule Plausible.Stats.Imported do|> select_merge([i], %{total_visitors: fragment("sum(?)", i.visitors)})end+ # :TRICKY: Handle backwards compatibility with old breakdown module+ defp shortname(query, _dim) when query.v2, do: :dim0+ defp shortname(_query, dim), do: dim+defp select_imported_metrics(q, []), do: qdefp select_imported_metrics(q, [:visitors | rest]) do@@ -551,63 +592,71 @@ defmodule Plausible.Stats.Imported do|> select_imported_metrics(rest)end- defp group_imported_by(q, dim, _query) when dim in [:source, :referrer] do+ defp group_imported_by(q, dim, query) when dim in [:source, :referrer] doq|> group_by([i], field(i, ^dim))|> select_merge([i], %{- ^dim => fragment("if(empty(?), ?, ?)", field(i, ^dim), @no_ref, field(i, ^dim))+ ^shortname(query, dim) =>+ fragment(+ "if(empty(?), ?, ?)",+ field(i, ^dim),+ @no_ref,+ field(i, ^dim)+ )})end- defp group_imported_by(q, dim, _query)+ defp group_imported_by(q, dim, query)when dim in [:utm_source, :utm_medium, :utm_campaign, :utm_term, :utm_content] doq|> group_by([i], field(i, ^dim))|> where([i], fragment("not empty(?)", field(i, ^dim)))- |> select_merge([i], %{^dim => field(i, ^dim)})+ |> select_merge([i], %{^shortname(query, dim) => field(i, ^dim)})end- defp group_imported_by(q, :page, _query) do+ defp group_imported_by(q, :page, query) doq|> group_by([i], i.page)- |> select_merge([i], %{page: i.page, time_on_page: sum(i.time_on_page)})+ |> select_merge([i], %{^shortname(query, :page) => i.page, time_on_page: sum(i.time_on_page)})end- defp group_imported_by(q, :country, _query) do+ defp group_imported_by(q, :country, query) doq|> group_by([i], i.country)|> where([i], i.country != "ZZ")- |> select_merge([i], %{country: i.country})+ |> select_merge([i], %{^shortname(query, :country) => i.country})end- defp group_imported_by(q, :region, _query) do+ defp group_imported_by(q, :region, query) doq|> group_by([i], i.region)|> where([i], i.region != "")- |> select_merge([i], %{region: i.region})+ |> select_merge([i], %{^shortname(query, :region) => i.region})end- defp group_imported_by(q, :city, _query) do+ defp group_imported_by(q, :city, query) doq|> group_by([i], i.city)|> where([i], i.city != 0 and not is_nil(i.city))- |> select_merge([i], %{city: i.city})+ |> select_merge([i], %{^shortname(query, :city) => i.city})end- defp group_imported_by(q, dim, _query) when dim in [:device, :browser] do+ defp group_imported_by(q, dim, query) when dim in [:device, :browser] doq|> group_by([i], field(i, ^dim))|> select_merge([i], %{- ^dim => fragment("if(empty(?), ?, ?)", field(i, ^dim), @not_set, field(i, ^dim))+ ^shortname(query, dim) =>+ fragment("if(empty(?), ?, ?)", field(i, ^dim), @not_set, field(i, ^dim))})end- defp group_imported_by(q, :browser_version, _query) do+ defp group_imported_by(q, :browser_version, query) doq|> group_by([i], [i.browser, i.browser_version])|> select_merge([i], %{- browser: fragment("if(empty(?), ?, ?)", i.browser, @not_set, i.browser),- browser_version:+ ^shortname(query, :browser) =>+ fragment("if(empty(?), ?, ?)", i.browser, @not_set, i.browser),+ ^shortname(query, :browser_version) =>fragment("if(empty(?), ?, ?)",i.browser_version,@@ -617,20 +666,22 @@ defmodule Plausible.Stats.Imported do})end- defp group_imported_by(q, :os, _query) do+ defp group_imported_by(q, :os, query) doq|> group_by([i], i.operating_system)|> select_merge([i], %{- os: fragment("if(empty(?), ?, ?)", i.operating_system, @not_set, i.operating_system)+ ^shortname(query, :os) =>+ fragment("if(empty(?), ?, ?)", i.operating_system, @not_set, i.operating_system)})end- defp group_imported_by(q, :os_version, _query) do+ defp group_imported_by(q, :os_version, query) doq|> group_by([i], [i.operating_system, i.operating_system_version])|> select_merge([i], %{- os: fragment("if(empty(?), ?, ?)", i.operating_system, @not_set, i.operating_system),- os_version:+ ^shortname(query, :os) =>+ fragment("if(empty(?), ?, ?)", i.operating_system, @not_set, i.operating_system),+ ^shortname(query, :os_version) =>fragment("if(empty(?), ?, ?)",i.operating_system_version,@@ -640,23 +691,23 @@ defmodule Plausible.Stats.Imported do})end- defp group_imported_by(q, dim, _query) when dim in [:entry_page, :exit_page] do+ defp group_imported_by(q, dim, query) when dim in [:entry_page, :exit_page] doq|> group_by([i], field(i, ^dim))- |> select_merge([i], %{^dim => field(i, ^dim)})+ |> select_merge([i], %{^shortname(query, dim) => field(i, ^dim)})end- defp group_imported_by(q, :name, _query) do+ defp group_imported_by(q, :name, query) doq|> group_by([i], i.name)- |> select_merge([i], %{name: i.name})+ |> select_merge([i], %{^shortname(query, :name) => i.name})enddefp group_imported_by(q, :url, query) when query.v2 doq|> group_by([i], i.link_url)|> select_merge([i], %{- url: fragment("if(not empty(?), ?, ?)", i.link_url, i.link_url, @none)+ ^shortname(query, :url) => fragment("if(not empty(?), ?, ?)", i.link_url, i.link_url, @none)})end@@ -672,7 +723,7 @@ defmodule Plausible.Stats.Imported doq|> group_by([i], i.path)|> select_merge([i], %{- path: fragment("if(not empty(?), ?, ?)", i.path, i.path, @none)+ ^shortname(query, :path) => fragment("if(not empty(?), ?, ?)", i.path, i.path, @none)})end@@ -684,9 +735,9 @@ defmodule Plausible.Stats.Imported do})end- defp select_joined_dimension(q, :city, _query) do+ defp select_joined_dimension(q, :city, query) doselect_merge(q, [s, i], %{- city: fragment("greatest(?,?)", i.city, s.city)+ ^shortname(query, :city) => fragment("greatest(?,?)", i.city, s.city)})end@@ -717,9 +768,15 @@ defmodule Plausible.Stats.Imported do})end- defp select_joined_dimension(q, dim, _query) do+ defp select_joined_dimension(q, dim, query) doselect_merge(q, [s, i], %{- ^dim => fragment("if(empty(?), ?, ?)", field(s, ^dim), field(i, ^dim), field(s, ^dim))+ ^shortname(query, dim) =>+ fragment(+ "if(empty(?), ?, ?)",+ field(s, ^shortname(query, dim)),+ field(i, ^shortname(query, dim)),+ field(s, ^shortname(query, dim))+ )})end@@ -825,10 +882,23 @@ defmodule Plausible.Stats.Imported do|> select_joined_metrics(rest)end- defp apply_order_by(q, [:visitors | rest]) do+ defp apply_order_by(q, %Query{v2: true}, _), do: q++ defp apply_order_by(q, query, [:visitors | rest]) doorder_by(q, [s, i], desc: s.visitors + i.visitors)- |> apply_order_by(rest)+ |> apply_order_by(query, rest)end- defp apply_order_by(q, _), do: q+ defp apply_order_by(q, _query, _), do: q++ defp naive_dimension_join(q1, q2, metrics) do+ from(a in Ecto.Query.subquery(q1),+ full_join: b in subquery(q2),+ on: a.dim0 == b.dim0,+ select: %{+ dim0: fragment("coalesce(?, ?)", a.dim0, b.dim0)+ }+ )+ |> select_joined_metrics(metrics)+ endenddiff --git a/lib/plausible/stats/metrics.ex b/lib/plausible/stats/metrics.exindex 733c7d6e1c..8ed71dd2e3 100644--- a/lib/plausible/stats/metrics.ex+++ b/lib/plausible/stats/metrics.ex@@ -16,6 +16,7 @@ defmodule Plausible.Stats.Metrics do:visit_duration,:events,:conversion_rate,+ :group_conversion_rate,:time_on_page] ++ on_ee(do: Plausible.Stats.Goal.Revenue.revenue_metrics(), else: [])diff --git a/lib/plausible/stats/query.ex b/lib/plausible/stats/query.exindex ae25c6536b..86fc13220c 100644--- a/lib/plausible/stats/query.ex+++ b/lib/plausible/stats/query.ex@@ -15,9 +15,10 @@ defmodule Plausible.Stats.Query doexperimental_reduced_joins?: false,latest_import_end_date: nil,metrics: [],- order_by: [],+ order_by: nil,timezone: nil,- v2: false+ v2: false,+ preloaded_goals: []require OpenTelemetry.Tracer, as: Traceralias Plausible.Stats.{Filters, Interval, Imported}@@ -231,6 +232,18 @@ defmodule Plausible.Stats.Query do|> refresh_imported_opts()end+ def set_metrics(query, metrics) do+ query+ |> struct!(metrics: metrics)+ |> refresh_imported_opts()+ end++ def set_order_by(query, order_by) do+ query+ |> struct!(order_by: order_by)+ |> refresh_imported_opts()+ end+def put_filter(query, filter) doquery|> struct!(filters: query.filters ++ [filter])diff --git a/lib/plausible/stats/query_optimizer.ex b/lib/plausible/stats/query_optimizer.exindex 936fa2758f..c1c38e5fb1 100644--- a/lib/plausible/stats/query_optimizer.ex+++ b/lib/plausible/stats/query_optimizer.ex@@ -1,28 +1,155 @@defmodule Plausible.Stats.QueryOptimizer do- @moduledoc false+ @moduledoc """+ Methods to manipulate Query for business logic reasons before building an ecto query.+ """- alias Plausible.Stats.Query+ alias Plausible.Stats.{Query, TableDecider, Util}+ @doc """+ This module manipulates an existing query, updating it according to business logic.++ For example, it:+ 1. Figures out what the right granularity to group by time is+ 2. Adds a missing order_by clause to a query+ 3. Updating "time" dimension in order_by to the right granularity++ """def optimize(query) doEnum.reduce(pipeline(), query, fn step, acc -> step.(acc) end)end+ @doc """+ Splits a query into event and sessions subcomponents as not all metrics can be+ queried from a single table.++ event:page dimension is treated in a special way, doing a breakdown of visit:entry_page+ for sessions.+ """+ def split(query) do+ {event_metrics, sessions_metrics, _other_metrics} =+ query.metrics+ |> Util.maybe_add_visitors_metric()+ |> TableDecider.partition_metrics(query)++ {+ Query.set_metrics(query, event_metrics),+ split_sessions_query(query, sessions_metrics)+ }+ end+defp pipeline() do[+ &update_group_by_time/1,&add_missing_order_by/1,- &update_group_by_time/1+ &update_time_in_order_by/1,+ &extend_hostname_filters_to_visit/1]enddefp add_missing_order_by(%Query{order_by: nil} = query) do- %Query{query | order_by: [{hd(query.metrics), :desc}]}+ order_by =+ case time_dimension(query) do+ nil -> [{hd(query.metrics), :desc}]+ time_dimension -> [{time_dimension, :asc}, {hd(query.metrics), :desc}]+ end++ %Query{query | order_by: order_by}enddefp add_missing_order_by(query), do: query- defp update_group_by_time(%Query{dimensions: ["time" | rest]} = query) do- %Query{query | dimensions: ["time:month" | rest]}+ defp update_group_by_time(+ %Query{+ date_range: %Date.Range{first: first, last: last}+ } = query+ ) do+ dimensions =+ query.dimensions+ |> Enum.map(fn+ "time" -> resolve_time_dimension(first, last)+ entry -> entry+ end)++ %Query{query | dimensions: dimensions}enddefp update_group_by_time(query), do: query++ defp resolve_time_dimension(first, last) do+ cond do+ Timex.diff(last, first, :hours) <= 48 -> "time:hour"+ Timex.diff(last, first, :days) <= 40 -> "time:day"+ true -> "time:month"+ end+ end++ defp update_time_in_order_by(query) do+ order_by =+ query.order_by+ |> Enum.map(fn+ {"time", direction} -> {time_dimension(query), direction}+ entry -> entry+ end)++ %Query{query | order_by: order_by}+ end++ @dimensions_hostname_map %{+ "visit:source" => "visit:entry_page_hostname",+ "visit:entry_page" => "visit:entry_page_hostname",+ "visit:utm_medium" => "visit:entry_page_hostname",+ "visit:utm_source" => "visit:entry_page_hostname",+ "visit:utm_campaign" => "visit:entry_page_hostname",+ "visit:utm_content" => "visit:entry_page_hostname",+ "visit:utm_term" => "visit:entry_page_hostname",+ "visit:referrer" => "visit:entry_page_hostname",+ "visit:exit_page" => "visit:exit_page_hostname"+ }++ # To avoid showing referrers across hostnames when event:hostname+ # filter is present for breakdowns, add entry/exit page hostname+ # filters+ defp extend_hostname_filters_to_visit(query) do+ hostname_filters =+ query.filters+ |> Enum.filter(fn [_operation, filter_key | _rest] -> filter_key == "event:hostname" end)++ if length(hostname_filters) > 0 do+ extra_filters =+ query.dimensions+ |> Enum.flat_map(&hostname_filters_for_dimension(&1, hostname_filters))++ %Query{query | filters: query.filters ++ extra_filters}+ else+ query+ end+ end++ defp hostname_filters_for_dimension(dimension, hostname_filters) do+ if Map.has_key?(@dimensions_hostname_map, dimension) do+ filter_key = Map.get(@dimensions_hostname_map, dimension)++ hostname_filters+ |> Enum.map(fn [operation, _filter_key | rest] -> [operation, filter_key | rest] end)+ else+ []+ end+ end++ defp time_dimension(query) do+ Enum.find(query.dimensions, &String.starts_with?(&1, "time"))+ end++ defp split_sessions_query(query, session_metrics) do+ dimensions =+ query.dimensions+ |> Enum.map(fn+ "event:page" -> "visit:entry_page"+ dimension -> dimension+ end)++ query+ |> Query.set_metrics(session_metrics)+ |> Query.set_dimensions(dimensions)+ endenddiff --git a/lib/plausible/stats/query_result.ex b/lib/plausible/stats/query_result.exindex d7b72bf80d..b9f3744448 100644--- a/lib/plausible/stats/query_result.ex+++ b/lib/plausible/stats/query_result.ex@@ -1,7 +1,7 @@defmodule Plausible.Stats.QueryResult do@moduledoc false- alias Plausible.Stats.SQL.QueryBuilder+ alias Plausible.Stats.Utilalias Plausible.Stats.Filtersalias Plausible.Stats.Query@@ -15,7 +15,7 @@ defmodule Plausible.Stats.QueryResult doresults|> Enum.map(fn entry ->%{- dimensions: Enum.map(query.dimensions, &Map.get(entry, QueryBuilder.shortname(&1))),+ dimensions: Enum.map(query.dimensions, &dimension_label(&1, entry, query)),metrics: Enum.map(query.metrics, &Map.get(entry, &1))}end)@@ -44,6 +44,22 @@ defmodule Plausible.Stats.QueryResult dodefp meta(_), do: %{}+ defp dimension_label("event:goal", entry, query) do+ {events, paths} = Filters.Utils.split_goals(query.preloaded_goals)++ goal_index = Map.get(entry, Util.shortname(query, "event:goal"))++ # Closely coupled logic with Plausible.Stats.SQL.Expression.event_goal_join/2+ cond do+ goal_index < 0 -> Enum.at(events, -goal_index - 1) |> Filters.Utils.unwrap_goal_value()+ goal_index > 0 -> Enum.at(paths, goal_index - 1) |> Filters.Utils.unwrap_goal_value()+ end+ end++ defp dimension_label(dimension, entry, query) do+ Map.get(entry, Util.shortname(query, dimension))+ end+defp serializable_filter([operation, "event:goal", clauses]) do[operation, "event:goal", Enum.map(clauses, &Filters.Utils.unwrap_goal_value/1)]enddiff --git a/lib/plausible/stats/sql/expression.ex b/lib/plausible/stats/sql/expression.exindex cdd1cffe44..e74348daed 100644--- a/lib/plausible/stats/sql/expression.ex+++ b/lib/plausible/stats/sql/expression.ex@@ -1,5 +1,8 @@defmodule Plausible.Stats.SQL.Expression do- @moduledoc false+ @moduledoc """+ This module is responsible for generating SQL/Ecto expressions+ for dimensions used in query select, group_by and order_by.+ """import Ecto.Query@@ -86,4 +89,21 @@ defmodule Plausible.Stats.SQL.Expression dodef dimension("visit:country", _query), do: dynamic([t], t.country)def dimension("visit:region", _query), do: dynamic([t], t.region)def dimension("visit:city", _query), do: dynamic([t], t.city)++ defmacro event_goal_join(events, page_regexes) do+ quote do+ fragment(+ """+ arrayPushFront(+ CAST(multiMatchAllIndices(?, ?) AS Array(Int64)),+ -indexOf(?, ?)+ )+ """,+ e.pathname,+ type(^unquote(page_regexes), {:array, :string}),+ type(^unquote(events), {:array, :string}),+ e.name+ )+ end+ endenddiff --git a/lib/plausible/stats/sql/query_builder.ex b/lib/plausible/stats/sql/query_builder.exindex a966be4b12..40db95f28f 100644--- a/lib/plausible/stats/sql/query_builder.ex+++ b/lib/plausible/stats/sql/query_builder.ex@@ -5,47 +5,45 @@ defmodule Plausible.Stats.SQL.QueryBuilder doimport Ecto.Queryimport Plausible.Stats.Imported+ import Plausible.Stats.Util- alias Plausible.Stats.{Base, Query, TableDecider, Util, Filters, Metrics}+ alias Plausible.Stats.{Base, Query, QueryOptimizer, TableDecider, Filters, Metrics}alias Plausible.Stats.SQL.Expression+ require Plausible.Stats.SQL.Expression+def build(query, site) do- {event_metrics, sessions_metrics, _other_metrics} =- query.metrics- |> Util.maybe_add_visitors_metric()- |> TableDecider.partition_metrics(query)+ {event_query, sessions_query} = QueryOptimizer.split(query)++ event_q = build_events_query(site, event_query)+ sessions_q = build_sessions_query(site, sessions_query)join_query_results(- build_events_query(site, query, event_metrics),- event_metrics,- build_sessions_query(site, query, sessions_metrics),- sessions_metrics,- query+ {event_q, event_query},+ {sessions_q, sessions_query})end- def shortname(metric) when is_atom(metric), do: metric- def shortname(dimension), do: Plausible.Stats.Filters.without_prefix(dimension)+ defp build_events_query(_site, %Query{metrics: []}), do: nil- defp build_events_query(_, _, []), do: nil-- defp build_events_query(site, query, event_metrics) do+ defp build_events_query(site, events_query) doq =from(e in "events_v2",- where: ^Filters.WhereBuilder.build(:events, site, query),- select: ^Base.select_event_metrics(event_metrics)+ where: ^Filters.WhereBuilder.build(:events, site, events_query),+ select: ^Base.select_event_metrics(events_query.metrics))on_ee do- q = Plausible.Stats.Sampling.add_query_hint(q, query)+ q = Plausible.Stats.Sampling.add_query_hint(q, events_query)endq- |> join_sessions_if_needed(site, query)- |> build_group_by(query)- |> merge_imported(site, query, event_metrics)- |> Base.maybe_add_conversion_rate(site, query, event_metrics)+ |> join_sessions_if_needed(site, events_query)+ |> build_group_by(events_query)+ |> merge_imported(site, events_query, events_query.metrics)+ |> maybe_add_global_conversion_rate(site, events_query)+ |> maybe_add_group_conversion_rate(site, events_query)enddefp join_sessions_if_needed(q, site, query) do@@ -68,24 +66,24 @@ defmodule Plausible.Stats.SQL.QueryBuilder doendend- def build_sessions_query(_, _, []), do: nil+ defp build_sessions_query(_site, %Query{metrics: []}), do: nil- def build_sessions_query(site, query, session_metrics) do+ defp build_sessions_query(site, sessions_query) doq =from(e in "sessions_v2",- where: ^Filters.WhereBuilder.build(:sessions, site, query),- select: ^Base.select_session_metrics(session_metrics, query)+ where: ^Filters.WhereBuilder.build(:sessions, site, sessions_query),+ select: ^Base.select_session_metrics(sessions_query.metrics, sessions_query))on_ee do- q = Plausible.Stats.Sampling.add_query_hint(q, query)+ q = Plausible.Stats.Sampling.add_query_hint(q, sessions_query)endq- |> join_events_if_needed(site, query)- |> build_group_by(query)- |> merge_imported(site, query, session_metrics)+ |> join_events_if_needed(site, sessions_query)+ |> build_group_by(sessions_query)+ |> merge_imported(site, sessions_query, sessions_query.metrics)enddef join_events_if_needed(q, site, query) do@@ -113,15 +111,30 @@ defmodule Plausible.Stats.SQL.QueryBuilder doenddefp build_group_by(q, query) do- Enum.reduce(query.dimensions, q, fn dimension, q ->- q- |> select_merge(^%{shortname(dimension) => Expression.dimension(dimension, query)})- |> group_by(^Expression.dimension(dimension, query))- end)+ Enum.reduce(query.dimensions, q, &dimension_group_by(&2, query, &1))+ end++ defp dimension_group_by(q, query, "event:goal" = dimension) do+ {events, page_regexes} = Filters.Utils.split_goals_query_expressions(query.preloaded_goals)++ from(e in q,+ array_join: goal in Expression.event_goal_join(events, page_regexes),+ select_merge: %{+ ^shortname(query, dimension) => fragment("?", goal)+ },+ group_by: goal,+ where: goal != 0+ )+ end++ defp dimension_group_by(q, query, dimension) do+ q+ |> select_merge(^%{shortname(query, dimension) => Expression.dimension(dimension, query)})+ |> group_by(^Expression.dimension(dimension, query))enddefp build_order_by(q, query, mode) do- Enum.reduce(query.order_by, q, &build_order_by(&2, query, &1, mode))+ Enum.reduce(query.order_by || [], q, &build_order_by(&2, query, &1, mode))enddef build_order_by(q, query, {metric_or_dimension, order_direction}, :inner) do@@ -132,36 +145,36 @@ defmodule Plausible.Stats.SQL.QueryBuilder doorder_direction,if(Metrics.metric?(metric_or_dimension),- do: dynamic([], selected_as(^shortname(metric_or_dimension))),+ do: dynamic([], selected_as(^shortname(query, metric_or_dimension))),else: Expression.dimension(metric_or_dimension, query))})end- def build_order_by(q, _query, {metric_or_dimension, order_direction}, :outer) do+ def build_order_by(q, query, {metric_or_dimension, order_direction}, :outer) doorder_by(q,[t],^{order_direction,- dynamic([], selected_as(^shortname(metric_or_dimension)))+ dynamic([], selected_as(^shortname(query, metric_or_dimension)))})end- defmacrop select_join_fields(q, list, table_name) do+ defmacrop select_join_fields(q, query, list, table_name) doquote doEnum.reduce(unquote(list), unquote(q), fn metric_or_dimension, q ->select_merge(q,^%{- shortname(metric_or_dimension) =>+ shortname(unquote(query), metric_or_dimension) =>dynamic([e, s],selected_as(- field(unquote(table_name), ^shortname(metric_or_dimension)),- ^shortname(metric_or_dimension)+ field(unquote(table_name), ^shortname(unquote(query), metric_or_dimension)),+ ^shortname(unquote(query), metric_or_dimension)))}@@ -170,22 +183,98 @@ defmodule Plausible.Stats.SQL.QueryBuilder doendend- defp join_query_results(nil, _, nil, _, _query), do: nil+ # Adds conversion_rate metric to query, calculated as+ # X / Y where Y is the same breakdown value without goal or props+ # filters.+ def maybe_add_global_conversion_rate(q, site, query) do+ if :conversion_rate in query.metrics do+ total_query =+ query+ |> Query.remove_filters(["event:goal", "event:props"])+ |> Query.set_dimensions([])++ q+ |> select_merge(+ ^%{+ total_visitors: Base.total_visitors_subquery(site, total_query, query.include_imported)+ }+ )+ |> select_merge([e], %{+ conversion_rate:+ selected_as(+ fragment(+ "if(? > 0, round(? / ? * 100, 1), 0)",+ selected_as(:__total_visitors),+ selected_as(:visitors),+ selected_as(:__total_visitors)+ ),+ :conversion_rate+ )+ })+ else+ q+ end+ end++ # This function injects a group_conversion_rate metric into+ # a dimensional query. It is calculated as X / Y, where:+ #+ # * X is the number of conversions for a set of dimensions+ # result (conversion = number of visitors who+ # completed the filtered goal with the filtered+ # custom properties).+ #+ # * Y is the number of all visitors for this set of dimensions+ # result without the `event:goal` and `event:props:*`+ # filters.+ def maybe_add_group_conversion_rate(q, site, query) do+ if :group_conversion_rate in query.metrics do+ group_totals_query =+ query+ |> Query.remove_filters(["event:goal", "event:props"])+ |> Query.set_metrics([:visitors])+ |> Query.set_order_by([])++ from(e in subquery(q),+ left_join: c in subquery(build(group_totals_query, site)),+ on: ^build_group_by_join(query),+ select_merge: %{+ total_visitors: c.visitors,+ group_conversion_rate:+ selected_as(+ fragment(+ "if(? > 0, round(? / ? * 100, 1), 0)",+ c.visitors,+ e.visitors,+ c.visitors+ ),+ :group_conversion_rate+ )+ }+ )+ |> select_join_fields(query, query.dimensions, e)+ |> select_join_fields(query, List.delete(query.metrics, :group_conversion_rate), e)+ else+ q+ end+ end++ defp join_query_results({nil, _}, {nil, _}), do: nil- defp join_query_results(events_q, _, nil, _, query),- do: events_q |> build_order_by(query, :inner)+ defp join_query_results({events_q, events_query}, {nil, _}),+ do: events_q |> build_order_by(events_query, :inner)- defp join_query_results(nil, _, sessions_q, _, query),- do: sessions_q |> build_order_by(query, :inner)+ defp join_query_results({nil, _}, {sessions_q, sessions_query}),+ do: sessions_q |> build_order_by(sessions_query, :inner)- defp join_query_results(events_q, event_metrics, sessions_q, sessions_metrics, query) do+ defp join_query_results({events_q, events_query}, {sessions_q, sessions_query}) dojoin(subquery(events_q), :left, [e], s in subquery(sessions_q),- on: ^build_group_by_join(query)+ on: ^build_group_by_join(events_query))- |> select_join_fields(query.dimensions, e)- |> select_join_fields(event_metrics, e)- |> select_join_fields(List.delete(sessions_metrics, :sample_percent), s)- |> build_order_by(query, :outer)+ |> select_join_fields(events_query, events_query.dimensions, e)+ |> select_join_fields(events_query, events_query.metrics, e)+ |> select_join_fields(sessions_query, List.delete(sessions_query.metrics, :sample_percent), s)+ |> build_order_by(events_query, :outer)enddefp build_group_by_join(%Query{dimensions: []}), do: true@@ -193,7 +282,7 @@ defmodule Plausible.Stats.SQL.QueryBuilder dodefp build_group_by_join(query) doquery.dimensions|> Enum.map(fn dim ->- dynamic([e, s], field(e, ^shortname(dim)) == field(s, ^shortname(dim)))+ dynamic([e, s], field(e, ^shortname(query, dim)) == field(s, ^shortname(query, dim)))end)|> Enum.reduce(fn condition, acc -> dynamic([], ^acc and ^condition) end)enddiff --git a/lib/plausible/stats/table_decider.ex b/lib/plausible/stats/table_decider.exindex 629744228b..c0176d417f 100644--- a/lib/plausible/stats/table_decider.ex+++ b/lib/plausible/stats/table_decider.ex@@ -14,12 +14,6 @@ defmodule Plausible.Stats.TableDecider do|> Enum.any?(&(filters_partitioner(query, &1) == :session))end- def event_filters?(query) do- query- |> filter_keys()- |> Enum.any?(&(filters_partitioner(query, &1) == :event))- end-def partition_metrics(metrics, query) do%{event: event_only_metrics,@@ -64,6 +58,7 @@ defmodule Plausible.Stats.TableDecider doenddefp metric_partitioner(_, :conversion_rate), do: :event+ defp metric_partitioner(_, :group_conversion_rate), do: :eventdefp metric_partitioner(_, :average_revenue), do: :eventdefp metric_partitioner(_, :total_revenue), do: :eventdefp metric_partitioner(_, :pageviews), do: :eventdiff --git a/lib/plausible/stats/util.ex b/lib/plausible/stats/util.exindex 85c36888f8..60bdd7a931 100644--- a/lib/plausible/stats/util.ex+++ b/lib/plausible/stats/util.ex@@ -41,7 +41,8 @@ defmodule Plausible.Stats.Util dofor any of the other metrics to be calculated."""def maybe_add_visitors_metric(metrics) do- needed? = Enum.any?([:conversion_rate, :time_on_page], &(&1 in metrics))+ needed? =+ Enum.any?([:conversion_rate, :group_conversion_rate, :time_on_page], &(&1 in metrics))if needed? and :visitors not in metrics dometrics ++ [:visitors]@@ -49,4 +50,12 @@ defmodule Plausible.Stats.Util dometricsendend++ def shortname(_query, metric) when is_atom(metric), do: metric+ def shortname(_query, "time:" <> _), do: :time++ def shortname(query, dimension) do+ index = Enum.find_index(query.dimensions, &(&1 == dimension))+ :"dim#{index}"+ endenddiff --git a/test/plausible/stats/query_optimizer_test.exs b/test/plausible/stats/query_optimizer_test.exsnew file mode 100644index 0000000000..aee73b88c0--- /dev/null+++ b/test/plausible/stats/query_optimizer_test.exs@@ -0,0 +1,147 @@+defmodule Plausible.Stats.QueryOptimizerTest do+ use Plausible.DataCase, async: true++ alias Plausible.Stats.{Query, QueryOptimizer}++ @default_params %{metrics: [:visitors]}++ def perform(params) do+ params = Map.merge(@default_params, params) |> Map.to_list()+ struct!(Query, params) |> QueryOptimizer.optimize()+ end++ describe "add_missing_order_by" do+ test "does nothing if order_by passed" do+ assert perform(%{order_by: [visitors: :desc]}).order_by == [{:visitors, :desc}]+ end++ test "adds first metric to order_by if order_by not specified" do+ assert perform(%{metrics: [:pageviews, :visitors]}).order_by == [{:pageviews, :desc}]++ assert perform(%{metrics: [:pageviews, :visitors], dimensions: ["event:page"]}).order_by ==+ [{:pageviews, :desc}]+ end++ test "adds time and first metric to order_by if order_by not specified" do+ assert perform(%{+ date_range: Date.range(~N[2022-01-01 00:00:00], ~N[2022-02-01 00:00:00]),+ metrics: [:pageviews, :visitors],+ dimensions: ["time", "event:page"]+ }).order_by ==+ [{"time:day", :asc}, {:pageviews, :desc}]+ end+ end++ describe "update_group_by_time" do+ test "does nothing if `time` dimension not passed" do+ assert perform(%{+ date_range: Date.range(~N[2022-01-01 00:00:00], ~N[2022-01-05 00:00:00]),+ dimensions: ["time:month"]+ }).dimensions == ["time:month"]+ end++ test "updating time dimension" do+ assert perform(%{+ date_range: Date.range(~N[2022-01-01 00:00:00], ~N[2022-01-01 05:00:00]),+ dimensions: ["time"]+ }).dimensions == ["time:hour"]++ assert perform(%{+ date_range: Date.range(~N[2022-01-01 00:00:00], ~N[2022-01-02 00:00:00]),+ dimensions: ["time"]+ }).dimensions == ["time:hour"]++ assert perform(%{+ date_range: Date.range(~N[2022-01-01 00:00:00], ~N[2022-01-02 16:00:00]),+ dimensions: ["time"]+ }).dimensions == ["time:hour"]++ assert perform(%{+ date_range: Date.range(~D[2022-01-01], ~D[2022-01-04]),+ dimensions: ["time"]+ }).dimensions == ["time:day"]++ assert perform(%{+ date_range: Date.range(~D[2022-01-01], ~D[2022-01-10]),+ dimensions: ["time"]+ }).dimensions == ["time:day"]++ assert perform(%{+ date_range: Date.range(~D[2022-01-01], ~D[2022-01-16]),+ dimensions: ["time"]+ }).dimensions == ["time:day"]++ assert perform(%{+ date_range: Date.range(~D[2022-01-01], ~D[2022-02-16]),+ dimensions: ["time"]+ }).dimensions == ["time:month"]++ assert perform(%{+ date_range: Date.range(~D[2022-01-01], ~D[2022-03-16]),+ dimensions: ["time"]+ }).dimensions == ["time:month"]++ assert perform(%{+ date_range: Date.range(~D[2022-01-01], ~D[2022-03-16]),+ dimensions: ["time"]+ }).dimensions == ["time:month"]++ assert perform(%{+ date_range: Date.range(~D[2022-01-01], ~D[2023-11-16]),+ dimensions: ["time"]+ }).dimensions == ["time:month"]++ assert perform(%{+ date_range: Date.range(~D[2022-01-01], ~D[2024-01-16]),+ dimensions: ["time"]+ }).dimensions == ["time:month"]++ assert perform(%{+ date_range: Date.range(~D[2022-01-01], ~D[2026-01-01]),+ dimensions: ["time"]+ }).dimensions == ["time:month"]+ end+ end++ describe "update_time_in_order_by" do+ test "updates explicit time dimension in order_by" do+ assert perform(%{+ date_range: Date.range(~N[2022-01-01 00:00:00], ~N[2022-01-01 05:00:00]),+ dimensions: ["time:hour"],+ order_by: [{"time", :asc}]+ }).order_by == [{"time:hour", :asc}]+ end+ end++ describe "extend_hostname_filters_to_visit" do+ test "updates filters it filtering by event:hostname and visit:referrer and visit:exit_page dimensions" do+ assert perform(%{+ date_range: Date.range(~N[2022-01-01 00:00:00], ~N[2022-01-01 05:00:00]),+ filters: [+ [:is, "event:hostname", ["example.com"]],+ [:matches, "event:hostname", ["*.com"]]+ ],+ dimensions: ["visit:referrer", "visit:exit_page"]+ }).filters == [+ [:is, "event:hostname", ["example.com"]],+ [:matches, "event:hostname", ["*.com"]],+ [:is, "visit:entry_page_hostname", ["example.com"]],+ [:matches, "visit:entry_page_hostname", ["*.com"]],+ [:is, "visit:exit_page_hostname", ["example.com"]],+ [:matches, "visit:exit_page_hostname", ["*.com"]]+ ]+ end++ test "does not update filters if not needed" do+ assert perform(%{+ date_range: Date.range(~N[2022-01-01 00:00:00], ~N[2022-01-01 05:00:00]),+ filters: [+ [:is, "event:hostname", ["example.com"]]+ ],+ dimensions: ["time", "event:hostname"]+ }).filters == [+ [:is, "event:hostname", ["example.com"]]+ ]+ end+ end+enddiff --git a/test/plausible/stats/query_parser_test.exs b/test/plausible/stats/query_parser_test.exsindex 481f0362dc..bf3f3bedb4 100644--- a/test/plausible/stats/query_parser_test.exs+++ b/test/plausible/stats/query_parser_test.exs@@ -6,17 +6,32 @@ defmodule Plausible.Stats.Filters.QueryParserTest dosetup [:create_user, :create_new_site]- @date_range Date.range(Timex.today(), Timex.today())+ @today ~D[2021-05-05]+ @date_range Date.range(@today, @today)def check_success(params, site, expected_result) do- assert parse(site, params) == {:ok, expected_result}+ assert parse(site, params, @today) == {:ok, expected_result}enddef check_error(params, site, expected_error_message) do- {:error, message} = parse(site, params)+ {:error, message} = parse(site, params, @today)assert message =~ expected_error_messageend+ def check_date_range(date_range, site, expected_date_range) do+ %{"metrics" => ["visitors", "events"], "date_range" => date_range}+ |> check_success(site, %{+ metrics: [:visitors, :events],+ date_range: expected_date_range,+ filters: [],+ dimensions: [],+ order_by: nil,+ timezone: site.timezone,+ imported_data_requested: false,+ preloaded_goals: []+ })+ end+test "parsing empty map fails", %{site: site} do%{}|> check_error(site, "No valid metrics passed")@@ -32,7 +47,8 @@ defmodule Plausible.Stats.Filters.QueryParserTest dodimensions: [],order_by: nil,timezone: site.timezone,- imported_data_requested: false+ imported_data_requested: false,+ preloaded_goals: []})end@@ -69,7 +85,8 @@ defmodule Plausible.Stats.Filters.QueryParserTest dodimensions: [],order_by: nil,timezone: site.timezone,- imported_data_requested: false+ imported_data_requested: false,+ preloaded_goals: []})end@@ -98,7 +115,8 @@ defmodule Plausible.Stats.Filters.QueryParserTest dodimensions: [],order_by: nil,timezone: site.timezone,- imported_data_requested: false+ imported_data_requested: false,+ preloaded_goals: []})end@@ -142,7 +160,8 @@ defmodule Plausible.Stats.Filters.QueryParserTest dodimensions: [],order_by: nil,timezone: site.timezone,- imported_data_requested: false+ imported_data_requested: false,+ preloaded_goals: []})end@@ -165,7 +184,8 @@ defmodule Plausible.Stats.Filters.QueryParserTest dodimensions: [],order_by: nil,timezone: site.timezone,- imported_data_requested: false+ imported_data_requested: false,+ preloaded_goals: []})endend@@ -189,7 +209,8 @@ defmodule Plausible.Stats.Filters.QueryParserTest dodimensions: [],order_by: nil,timezone: site.timezone,- imported_data_requested: false+ imported_data_requested: false,+ preloaded_goals: []})endend@@ -240,7 +261,8 @@ defmodule Plausible.Stats.Filters.QueryParserTest dodimensions: [],order_by: nil,timezone: site.timezone,- imported_data_requested: true+ imported_data_requested: true,+ preloaded_goals: []})end@@ -275,7 +297,8 @@ defmodule Plausible.Stats.Filters.QueryParserTest dodimensions: [],order_by: nil,timezone: site.timezone,- imported_data_requested: false+ imported_data_requested: false,+ preloaded_goals: [{:page, "/thank-you"}, {:event, "Signup"}]})end@@ -303,6 +326,42 @@ defmodule Plausible.Stats.Filters.QueryParserTest doenddescribe "date range validation" do+ test "parsing shortcut options", %{site: site} do+ check_date_range("day", site, Date.range(~D[2021-05-05], ~D[2021-05-05]))+ check_date_range("7d", site, Date.range(~D[2021-04-29], ~D[2021-05-05]))+ check_date_range("30d", site, Date.range(~D[2021-04-05], ~D[2021-05-05]))+ check_date_range("month", site, Date.range(~D[2021-05-01], ~D[2021-05-31]))+ check_date_range("6mo", site, Date.range(~D[2020-12-01], ~D[2021-05-31]))+ check_date_range("12mo", site, Date.range(~D[2020-06-01], ~D[2021-05-31]))+ check_date_range("year", site, Date.range(~D[2021-01-01], ~D[2021-12-31]))+ end++ test "parsing `all` with previous data", %{site: site} do+ site = Map.put(site, :stats_start_date, ~D[2020-01-01])+ check_date_range("all", site, Date.range(~D[2020-01-01], ~D[2021-05-05]))+ end++ test "parsing `all` with no previous data", %{site: site} do+ site = Map.put(site, :stats_start_date, nil)++ check_date_range("all", site, Date.range(~D[2021-05-05], ~D[2021-05-05]))+ end++ test "parsing custom date range", %{site: site} do+ check_date_range(+ ["2021-05-05", "2021-05-05"],+ site,+ Date.range(~D[2021-05-05], ~D[2021-05-05])+ )+ end++ test "parsing invalid custom date range", %{site: site} do+ %{"date_range" => "foo", "metrics" => ["visitors"]}+ |> check_error(site, ~r/Invalid date_range '\"foo\"'/)++ %{"date_range" => ["21415-00", "eee"], "metrics" => ["visitors"]}+ |> check_error(site, ~r/Invalid date_range /)+ endenddescribe "dimensions validation" do@@ -320,7 +379,8 @@ defmodule Plausible.Stats.Filters.QueryParserTest dodimensions: ["event:#{unquote(dimension)}"],order_by: nil,timezone: site.timezone,- imported_data_requested: false+ imported_data_requested: false,+ preloaded_goals: []})endend@@ -339,7 +399,8 @@ defmodule Plausible.Stats.Filters.QueryParserTest dodimensions: ["visit:#{unquote(dimension)}"],order_by: nil,timezone: site.timezone,- imported_data_requested: false+ imported_data_requested: false,+ preloaded_goals: []})endend@@ -357,7 +418,8 @@ defmodule Plausible.Stats.Filters.QueryParserTest dodimensions: ["event:props:foobar"],order_by: nil,timezone: site.timezone,- imported_data_requested: false+ imported_data_requested: false,+ preloaded_goals: []})end@@ -412,7 +474,8 @@ defmodule Plausible.Stats.Filters.QueryParserTest dodimensions: [],order_by: [{:events, :desc}, {:visitors, :asc}],timezone: site.timezone,- imported_data_requested: false+ imported_data_requested: false,+ preloaded_goals: []})end@@ -430,7 +493,8 @@ defmodule Plausible.Stats.Filters.QueryParserTest dodimensions: ["event:name"],order_by: [{"event:name", :desc}],timezone: site.timezone,- imported_data_requested: false+ imported_data_requested: false,+ preloaded_goals: []})end@@ -525,7 +589,8 @@ defmodule Plausible.Stats.Filters.QueryParserTest dodimensions: [],order_by: nil,timezone: site.timezone,- imported_data_requested: false+ imported_data_requested: false,+ preloaded_goals: [event: "Signup"]})end@@ -544,11 +609,58 @@ defmodule Plausible.Stats.Filters.QueryParserTest dodimensions: ["event:goal"],order_by: nil,timezone: site.timezone,- imported_data_requested: false+ imported_data_requested: false,+ preloaded_goals: [event: "Signup"]})endend+ describe "views_per_visit metric" do+ test "succeeds with normal filters", %{site: site} do+ insert(:goal, %{site: site, event_name: "Signup"})++ %{+ "metrics" => ["views_per_visit"],+ "date_range" => "all",+ "filters" => [["is", "event:goal", ["Signup"]]]+ }+ |> check_success(site, %{+ metrics: [:views_per_visit],+ date_range: @date_range,+ filters: [[:is, "event:goal", [event: "Signup"]]],+ dimensions: [],+ order_by: nil,+ timezone: site.timezone,+ imported_data_requested: false,+ preloaded_goals: [event: "Signup"]+ })+ end++ test "fails validation if event:page filter specified", %{site: site} do+ %{+ "metrics" => ["views_per_visit"],+ "date_range" => "all",+ "filters" => [["is", "event:page", ["/"]]]+ }+ |> check_error(+ site,+ ~r/Metric `views_per_visit` cannot be queried with a filter on `event:page`/+ )+ end++ test "fails validation with dimensions", %{site: site} do+ %{+ "metrics" => ["views_per_visit"],+ "date_range" => "all",+ "dimensions" => ["event:name"]+ }+ |> check_error(+ site,+ ~r/Metric `views_per_visit` cannot be queried with `dimensions`/+ )+ end+ end+describe "session metrics" dotest "single session metric succeeds", %{site: site} do%{@@ -563,7 +675,8 @@ defmodule Plausible.Stats.Filters.QueryParserTest dodimensions: ["visit:device"],order_by: nil,timezone: site.timezone,- imported_data_requested: false+ imported_data_requested: false,+ preloaded_goals: []})end@@ -575,20 +688,44 @@ defmodule Plausible.Stats.Filters.QueryParserTest do}|> check_error(site,- "Session metric(s) `bounce_rate` cannot be queried along with event filters or dimensions"+ "Session metric(s) `bounce_rate` cannot be queried along with event dimensions")end- test "fails if using session metric with event filter", %{site: site} do+ test "does not fail if using session metric with event:page dimension", %{site: site} do+ %{+ "metrics" => ["bounce_rate"],+ "date_range" => "all",+ "dimensions" => ["event:page"]+ }+ |> check_success(site, %{+ metrics: [:bounce_rate],+ date_range: @date_range,+ filters: [],+ dimensions: ["event:page"],+ order_by: nil,+ timezone: site.timezone,+ imported_data_requested: false,+ preloaded_goals: []+ })+ end++ test "does not fail if using session metric with event filter", %{site: site} do%{"metrics" => ["bounce_rate"],"date_range" => "all","filters" => [["is", "event:props:foo", ["(none)"]]]}- |> check_error(- site,- "Session metric(s) `bounce_rate` cannot be queried along with event filters or dimensions"- )+ |> check_success(site, %{+ metrics: [:bounce_rate],+ date_range: @date_range,+ filters: [[:is, "event:props:foo", ["(none)"]]],+ dimensions: [],+ order_by: nil,+ timezone: site.timezone,+ imported_data_requested: false,+ preloaded_goals: []+ })endendenddiff --git a/test/plausible_web/controllers/api/external_stats_controller/query_test.exs b/test/plausible_web/controllers/api/external_stats_controller/query_test.exsindex fa6ab457f0..cc4aec1ba6 100644--- a/test/plausible_web/controllers/api/external_stats_controller/query_test.exs+++ b/test/plausible_web/controllers/api/external_stats_controller/query_test.exs@@ -141,7 +141,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest do"dimensions" => ["event:name"]})- assert json_response(conn, 400)["error"] =~ "Invalid date range"+ assert json_response(conn, 400)["error"] =~ "Invalid date_range"endtest "fails when an invalid metric is provided", %{conn: conn, site: site} do@@ -156,7 +156,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest doassert json_response(conn, 400)["error"] =~ "Unknown metric '\"baa\"'"end- test "session metrics cannot be used with event:name property", %{conn: conn, site: site} do+ test "session metrics cannot be used with event:name dimension", %{conn: conn, site: site} doconn =post(conn, "/api/v2/query", %{"site_id" => site.domain,@@ -166,10 +166,10 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest do})assert json_response(conn, 400)["error"] =~- "Session metric(s) `bounce_rate` cannot be queried along with event filters or dimensions"+ "Session metric(s) `bounce_rate` cannot be queried along with event dimensions"end- test "session metrics cannot be used with event:props:* property", %{conn: conn, site: site} do+ test "session metrics cannot be used with event:props:* dimension", %{conn: conn, site: site} doconn =post(conn, "/api/v2/query", %{"site_id" => site.domain,@@ -179,23 +179,1103 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest do})assert json_response(conn, 400)["error"] =~- "Session metric(s) `bounce_rate` cannot be queried along with event filters or dimensions"+ "Session metric(s) `bounce_rate` cannot be queried along with event dimensions"end- test "session metrics cannot be used with event:name filter", %{conn: conn, site: site} do+ test "validates that metric views_per_visit cannot be used with event:page filter", %{+ conn: conn,+ site: site+ } do+ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "date_range" => "all",+ "metrics" => ["views_per_visit"],+ "filters" => [["is", "event:page", ["/something"]]]+ })++ assert json_response(conn, 400) == %{+ "error" =>+ "Metric `views_per_visit` cannot be queried with a filter on `event:page`"+ }+ end++ test "validates that metric views_per_visit cannot be used together with dimensions", %{+ conn: conn,+ site: site+ } do+ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "date_range" => "all",+ "metrics" => ["views_per_visit"],+ "dimensions" => ["event:name"]+ })++ assert json_response(conn, 400) == %{+ "error" => "Metric `views_per_visit` cannot be queried with `dimensions`"+ }+ end++ test "validates a metric can't be asked multiple times", %{+ conn: conn,+ site: site+ } do+ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "date_range" => "all",+ "metrics" => ["views_per_visit", "visitors", "visitors"]+ })++ assert json_response(conn, 400) == %{+ "error" => "Metrics cannot be queried multiple times"+ }+ end+ end++ test "aggregates a single metric", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:pageview, user_id: @user_id, timestamp: ~N[2021-01-01 00:00:00]),+ build(:pageview, user_id: @user_id, timestamp: ~N[2021-01-01 00:25:00]),+ build(:pageview, timestamp: ~N[2021-01-01 00:00:00])+ ])++ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "metrics" => ["pageviews"],+ "date_range" => "all"+ })++ assert json_response(conn, 200)["results"] == [%{"metrics" => [3], "dimensions" => []}]+ end++ test "aggregate views_per_visit rounds to two decimal places", %{+ conn: conn,+ site: site+ } do+ populate_stats(site, [+ build(:pageview, user_id: @user_id, timestamp: ~N[2021-01-01 00:00:00]),+ build(:pageview, user_id: @user_id, timestamp: ~N[2021-01-01 00:25:00]),+ build(:pageview, user_id: 456, timestamp: ~N[2021-01-01 00:00:00]),+ build(:pageview, user_id: 456, timestamp: ~N[2021-01-01 00:25:00]),+ build(:pageview, timestamp: ~N[2021-01-01 00:00:00])+ ])++ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "metrics" => ["views_per_visit"],+ "date_range" => "all"+ })++ assert json_response(conn, 200)["results"] == [%{"metrics" => [1.67], "dimensions" => []}]+ end++ test "aggregates all metrics in a single query", %{+ conn: conn,+ site: site+ } do+ populate_stats(site, [+ build(:pageview, user_id: @user_id, timestamp: ~N[2021-01-01 00:00:00]),+ build(:pageview, user_id: @user_id, timestamp: ~N[2021-01-01 00:25:00]),+ build(:pageview, timestamp: ~N[2021-01-01 00:00:00])+ ])++ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "date_range" => "all",+ "metrics" => [+ "pageviews",+ "visits",+ "views_per_visit",+ "visitors",+ "bounce_rate",+ "visit_duration"+ ]+ })++ assert json_response(conn, 200)["results"] == [+ %{"metrics" => [3, 2, 1.5, 2, 50, 750], "dimensions" => []}+ ]+ end++ describe "aggregation with filters" do+ test "can filter by source", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:pageview,+ referrer_source: "Google",+ user_id: @user_id,+ timestamp: ~N[2021-01-01 00:00:00]+ ),+ build(:pageview,+ user_id: @user_id,+ timestamp: ~N[2021-01-01 00:25:00]+ ),+ build(:pageview, timestamp: ~N[2021-01-01 00:00:00])+ ])++ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "date_range" => "all",+ "metrics" => ["pageviews", "visitors", "bounce_rate", "visit_duration"],+ "filters" => [["is", "visit:source", ["Google"]]]+ })++ assert json_response(conn, 200)["results"] == [+ %{"metrics" => [2, 1, 0, 1500], "dimensions" => []}+ ]+ end++ test "can filter by no source/referrer", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:pageview,+ user_id: @user_id,+ timestamp: ~N[2021-01-01 00:00:00]+ ),+ build(:pageview,+ user_id: @user_id,+ timestamp: ~N[2021-01-01 00:25:00]+ ),+ build(:pageview,+ referrer_source: "Google",+ timestamp: ~N[2021-01-01 00:00:00]+ )+ ])++ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "date_range" => "all",+ "metrics" => ["pageviews", "visitors", "bounce_rate", "visit_duration"],+ "filters" => [["is", "visit:source", ["Direct / None"]]]+ })++ assert json_response(conn, 200)["results"] == [+ %{"metrics" => [2, 1, 0, 1500], "dimensions" => []}+ ]+ end++ test "can filter by referrer", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:pageview,+ referrer: "https://facebook.com",+ user_id: @user_id,+ timestamp: ~N[2021-01-01 00:00:00]+ ),+ build(:pageview,+ user_id: @user_id,+ timestamp: ~N[2021-01-01 00:25:00]+ ),+ build(:pageview, timestamp: ~N[2021-01-01 00:00:00])+ ])++ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "date_range" => "all",+ "metrics" => ["pageviews", "visitors", "bounce_rate", "visit_duration"],+ "filters" => [["is", "visit:referrer", ["https://facebook.com"]]]+ })++ assert json_response(conn, 200)["results"] == [+ %{"metrics" => [2, 1, 0, 1500], "dimensions" => []}+ ]+ end++ test "wildcard referrer filter with special regex characters", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:pageview, referrer: "https://a.com"),+ build(:pageview, referrer: "https://a.com"),+ build(:pageview, referrer: "https://ab.com")+ ])++ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "date_range" => "all",+ "metrics" => ["visitors"],+ "filters" => [+ ["matches", "visit:referrer", ["**a.com**"]]+ ]+ })++ assert json_response(conn, 200)["results"] == [%{"metrics" => [2], "dimensions" => []}]+ end++ test "can filter by utm_medium", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:pageview,+ utm_medium: "social",+ user_id: @user_id,+ timestamp: ~N[2021-01-01 00:00:00]+ ),+ build(:pageview,+ user_id: @user_id,+ timestamp: ~N[2021-01-01 00:25:00]+ ),+ build(:pageview, timestamp: ~N[2021-01-01 00:00:00])+ ])++ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "date_range" => "all",+ "metrics" => ["pageviews", "visitors", "bounce_rate", "visit_duration"],+ "filters" => [["is", "visit:utm_medium", ["social"]]]+ })++ assert json_response(conn, 200)["results"] == [+ %{"metrics" => [2, 1, 0, 1500], "dimensions" => []}+ ]+ end++ test "can filter by utm_source", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:pageview,+ utm_source: "Twitter",+ user_id: @user_id,+ timestamp: ~N[2021-01-01 00:00:00]+ ),+ build(:pageview,+ user_id: @user_id,+ timestamp: ~N[2021-01-01 00:25:00]+ ),+ build(:pageview, timestamp: ~N[2021-01-01 00:00:00])+ ])++ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "date_range" => "all",+ "metrics" => ["pageviews", "visitors", "bounce_rate", "visit_duration"],+ "filters" => [["is", "visit:utm_source", ["Twitter"]]]+ })++ assert json_response(conn, 200)["results"] == [+ %{"metrics" => [2, 1, 0, 1500], "dimensions" => []}+ ]+ end++ test "can filter by utm_campaign", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:pageview,+ utm_campaign: "profile",+ user_id: @user_id,+ timestamp: ~N[2021-01-01 00:00:00]+ ),+ build(:pageview,+ user_id: @user_id,+ timestamp: ~N[2021-01-01 00:25:00]+ ),+ build(:pageview, timestamp: ~N[2021-01-01 00:00:00])+ ])++ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "date_range" => "all",+ "metrics" => ["pageviews", "visitors", "bounce_rate", "visit_duration"],+ "filters" => [["is", "visit:utm_campaign", ["profile"]]]+ })++ assert json_response(conn, 200)["results"] == [+ %{"metrics" => [2, 1, 0, 1500], "dimensions" => []}+ ]+ end++ test "can filter by device type", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:pageview,+ screen_size: "Desktop",+ user_id: @user_id,+ timestamp: ~N[2021-01-01 00:00:00]+ ),+ build(:pageview,+ user_id: @user_id,+ timestamp: ~N[2021-01-01 00:25:00]+ ),+ build(:pageview, timestamp: ~N[2021-01-01 00:00:00])+ ])++ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "date_range" => "all",+ "metrics" => ["pageviews", "visitors", "bounce_rate", "visit_duration"],+ "filters" => [["is", "visit:device", ["Desktop"]]]+ })++ assert json_response(conn, 200)["results"] == [+ %{"metrics" => [2, 1, 0, 1500], "dimensions" => []}+ ]+ end++ test "can filter by browser", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:pageview,+ browser: "Chrome",+ user_id: @user_id,+ timestamp: ~N[2021-01-01 00:00:00]+ ),+ build(:pageview,+ user_id: @user_id,+ timestamp: ~N[2021-01-01 00:25:00]+ ),+ build(:pageview, timestamp: ~N[2021-01-01 00:00:00])+ ])++ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "date_range" => "all",+ "metrics" => ["pageviews", "visitors", "bounce_rate", "visit_duration"],+ "filters" => [["is", "visit:browser", ["Chrome"]]]+ })++ assert json_response(conn, 200)["results"] == [+ %{"metrics" => [2, 1, 0, 1500], "dimensions" => []}+ ]+ end++ test "can filter by browser version", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:pageview,+ browser: "Chrome",+ browser_version: "56",+ user_id: @user_id,+ timestamp: ~N[2021-01-01 00:00:00]+ ),+ build(:pageview,+ user_id: @user_id,+ timestamp: ~N[2021-01-01 00:25:00]+ ),+ build(:pageview, timestamp: ~N[2021-01-01 00:00:00])+ ])++ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "date_range" => "all",+ "metrics" => ["pageviews", "visitors", "bounce_rate", "visit_duration"],+ "filters" => [["is", "visit:browser_version", ["56"]]]+ })++ assert json_response(conn, 200)["results"] == [+ %{"metrics" => [2, 1, 0, 1500], "dimensions" => []}+ ]+ end++ test "can filter by operating system", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:pageview,+ operating_system: "Mac",+ user_id: @user_id,+ timestamp: ~N[2021-01-01 00:00:00]+ ),+ build(:pageview,+ user_id: @user_id,+ timestamp: ~N[2021-01-01 00:25:00]+ ),+ build(:pageview, timestamp: ~N[2021-01-01 00:00:00])+ ])++ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "date_range" => "all",+ "metrics" => ["pageviews", "visitors", "bounce_rate", "visit_duration"],+ "filters" => [["is", "visit:os", ["Mac"]]]+ })++ assert json_response(conn, 200)["results"] == [+ %{"metrics" => [2, 1, 0, 1500], "dimensions" => []}+ ]+ end++ test "can filter by operating system version", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:pageview,+ operating_system_version: "10.5",+ user_id: @user_id,+ timestamp: ~N[2021-01-01 00:00:00]+ ),+ build(:pageview,+ user_id: @user_id,+ timestamp: ~N[2021-01-01 00:25:00]+ ),+ build(:pageview, timestamp: ~N[2021-01-01 00:00:00])+ ])++ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "date_range" => "all",+ "metrics" => ["pageviews", "visitors", "bounce_rate", "visit_duration"],+ "filters" => [["is", "visit:os_version", ["10.5"]]]+ })++ assert json_response(conn, 200)["results"] == [+ %{"metrics" => [2, 1, 0, 1500], "dimensions" => []}+ ]+ end++ test "can filter by country", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:pageview,+ country_code: "EE",+ user_id: @user_id,+ timestamp: ~N[2021-01-01 00:00:00]+ ),+ build(:pageview,+ user_id: @user_id,+ timestamp: ~N[2021-01-01 00:25:00]+ ),+ build(:pageview, timestamp: ~N[2021-01-01 00:00:00])+ ])++ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "date_range" => "all",+ "metrics" => ["pageviews", "visitors", "bounce_rate", "visit_duration"],+ "filters" => [["is", "visit:country", ["EE"]]]+ })++ assert json_response(conn, 200)["results"] == [+ %{"metrics" => [2, 1, 0, 1500], "dimensions" => []}+ ]+ end++ test "can filter by page", %{+ conn: conn,+ site: site+ } do+ populate_stats(site, [+ build(:pageview,+ user_id: @user_id,+ timestamp: ~N[2021-01-01 00:00:00]+ ),+ build(:pageview,+ user_id: @user_id,+ pathname: "/blogpost",+ timestamp: ~N[2021-01-01 00:25:00]+ ),+ build(:pageview, timestamp: ~N[2021-01-01 00:00:00]),+ build(:pageview,+ pathname: "/blogpost",+ timestamp: ~N[2021-01-01 00:00:00]+ )+ ])++ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "date_range" => "all",+ "metrics" => ["pageviews", "visitors", "bounce_rate", "visit_duration"],+ "filters" => [+ ["is", "event:page", ["/blogpost"]]+ ]+ })++ assert json_response(conn, 200)["results"] == [+ %{"metrics" => [2, 2, 100, 750], "dimensions" => []}+ ]+ end++ test "can filter by hostname", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:pageview,+ hostname: "one.example.com",+ timestamp: ~N[2021-01-01 00:00:00]+ ),+ build(:pageview,+ hostname: "example.com",+ timestamp: ~N[2021-01-01 00:25:00]+ ),+ build(:pageview, timestamp: ~N[2021-01-01 00:00:00])+ ])++ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "date_range" => "all",+ "metrics" => ["pageviews", "visitors", "bounce_rate", "visit_duration"],+ "filters" => [+ ["matches", "event:hostname", ["*.example.com", "example.com"]]+ ]+ })++ assert json_response(conn, 200)["results"] == [+ %{"metrics" => [2, 2, 100, 0], "dimensions" => []}+ ]+ end++ test "filtering by event:name", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:event,+ name: "Signup",+ timestamp: ~N[2021-01-01 00:00:00]+ ),+ build(:event,+ name: "Signup",+ user_id: @user_id,+ timestamp: ~N[2021-01-01 00:25:00]+ ),+ build(:event,+ name: "Signup",+ user_id: @user_id,+ timestamp: ~N[2021-01-01 00:25:00]+ ),+ build(:pageview, timestamp: ~N[2021-01-01 00:00:00])+ ])++ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "date_range" => "all",+ "metrics" => ["visitors", "pageviews"],+ "filters" => [+ ["is", "event:name", ["Signup"]]+ ]+ })++ assert json_response(conn, 200)["results"] == [%{"metrics" => [2, 0], "dimensions" => []}]+ end++ test "filtering by a custom event goal", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:event,+ name: "Signup",+ timestamp: ~N[2021-01-01 00:00:00]+ ),+ build(:event,+ name: "Signup",+ user_id: @user_id,+ timestamp: ~N[2021-01-01 00:25:00]+ ),+ build(:event,+ name: "Signup",+ user_id: @user_id,+ timestamp: ~N[2021-01-01 00:25:00]+ ),+ build(:event,+ name: "NotConfigured",+ timestamp: ~N[2021-01-01 00:25:00]+ )+ ])++ insert(:goal, %{site: site, event_name: "Signup"})++ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "date_range" => "all",+ "metrics" => ["visitors", "events"],+ "filters" => [+ ["is", "event:goal", ["Signup"]]+ ]+ })++ assert json_response(conn, 200)["results"] == [%{"metrics" => [2, 3], "dimensions" => []}]+ end++ test "filtering by a revenue goal", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:event,+ name: "Purchase",+ timestamp: ~N[2021-01-01 00:00:00]+ ),+ build(:event,+ name: "Purchase",+ user_id: @user_id,+ timestamp: ~N[2021-01-01 00:25:00]+ ),+ build(:event,+ name: "Purchase",+ user_id: @user_id,+ timestamp: ~N[2021-01-01 00:25:00]+ )+ ])++ insert(:goal, site: site, currency: :USD, event_name: "Purchase")++ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "date_range" => "all",+ "metrics" => ["visitors", "events"],+ "filters" => [+ ["is", "event:goal", ["Purchase"]]+ ]+ })++ assert json_response(conn, 200)["results"] == [%{"metrics" => [2, 3], "dimensions" => []}]+ end++ test "filtering by a simple pageview goal", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:pageview,+ pathname: "/register",+ timestamp: ~N[2021-01-01 00:00:00]+ ),+ build(:pageview,+ pathname: "/register",+ user_id: @user_id,+ timestamp: ~N[2021-01-01 00:25:00]+ ),+ build(:pageview,+ pathname: "/register",+ user_id: @user_id,+ timestamp: ~N[2021-01-01 00:25:00]+ ),+ build(:pageview,+ pathname: "/",+ timestamp: ~N[2021-01-01 00:25:00]+ )+ ])++ insert(:goal, %{site: site, page_path: "/register"})++ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "date_range" => "all",+ "metrics" => ["visitors", "pageviews"],+ "filters" => [+ ["is", "event:goal", ["Visit /register"]]+ ]+ })++ assert json_response(conn, 200)["results"] == [%{"metrics" => [2, 3], "dimensions" => []}]+ end++ test "filtering by a wildcard pageview goal", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:pageview, pathname: "/blog/post-1"),+ build(:pageview, pathname: "/blog/post-2", user_id: @user_id),+ build(:pageview, pathname: "/blog", user_id: @user_id),+ build(:pageview, pathname: "/")+ ])++ insert(:goal, %{site: site, page_path: "/blog**"})++ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "date_range" => "all",+ "metrics" => ["visitors", "pageviews"],+ "filters" => [+ ["matches", "event:goal", ["Visit /blog**"]]+ ]+ })++ assert json_response(conn, 200)["results"] == [%{"metrics" => [2, 3], "dimensions" => []}]+ end++ test "filtering by multiple custom event goals", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:event, name: "Signup"),+ build(:event, name: "Purchase", user_id: @user_id),+ build(:event, name: "Purchase", user_id: @user_id),+ build(:pageview)+ ])++ insert(:goal, %{site: site, event_name: "Signup"})+ insert(:goal, %{site: site, event_name: "Purchase"})++ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "date_range" => "all",+ "metrics" => ["visitors", "events"],+ "filters" => [+ ["is", "event:goal", ["Signup", "Purchase"]]+ ]+ })++ assert json_response(conn, 200)["results"] == [%{"metrics" => [2, 3], "dimensions" => []}]+ end++ test "filtering by multiple mixed goals", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:pageview, pathname: "/account/register"),+ build(:pageview, pathname: "/register", user_id: @user_id),+ build(:event, name: "Signup", user_id: @user_id),+ build(:pageview)+ ])++ insert(:goal, %{site: site, event_name: "Signup"})+ insert(:goal, %{site: site, page_path: "/**register"})++ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "date_range" => "all",+ "metrics" => ["visitors", "events", "pageviews"],+ "filters" => [+ ["matches", "event:goal", ["Signup", "Visit /**register"]]+ ]+ })++ assert json_response(conn, 200)["results"] == [+ %{"metrics" => [2, 3, 2], "dimensions" => []}+ ]+ end++ test "combining filters", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:pageview,+ pathname: "/blogpost",+ country_code: "EE",+ user_id: @user_id,+ timestamp: ~N[2021-01-01 00:00:00]+ ),+ build(:pageview,+ user_id: @user_id,+ timestamp: ~N[2021-01-01 00:25:00]+ ),+ build(:pageview, timestamp: ~N[2021-01-01 00:00:00]),+ build(:pageview,+ pathname: "/blogpost",+ timestamp: ~N[2021-01-01 00:00:00]+ )+ ])+conn =post(conn, "/api/v2/query", %{"site_id" => site.domain,- "metrics" => ["visitors", "bounce_rate"],"date_range" => "all",- "dimensions" => ["visit:device"],+ "metrics" => ["pageviews", "visitors", "bounce_rate", "visit_duration"],"filters" => [- ["is", "event:name", ["pageview"]]+ ["is", "event:page", ["/blogpost"]],+ ["is", "visit:country", ["EE"]]]})- assert json_response(conn, 400)["error"] =~- "Session metric(s) `bounce_rate` cannot be queried along with event filters or dimensions"+ assert json_response(conn, 200)["results"] == [+ %{"metrics" => [1, 1, 0, 1500], "dimensions" => []}+ ]+ end++ test "wildcard page filter", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:pageview, pathname: "/en/page1"),+ build(:pageview, pathname: "/en/page2"),+ build(:pageview, pathname: "/pl/page1")+ ])++ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "date_range" => "all",+ "metrics" => ["visitors"],+ "filters" => [+ ["matches", "event:page", ["/en/**"]]+ ]+ })++ assert json_response(conn, 200)["results"] == [%{"metrics" => [2], "dimensions" => []}]+ end++ test "negated wildcard page filter", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:pageview, pathname: "/en/page1"),+ build(:pageview, pathname: "/en/page2"),+ build(:pageview, pathname: "/pl/page1")+ ])++ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "date_range" => "all",+ "metrics" => ["visitors"],+ "filters" => [+ ["does_not_match", "event:page", ["/en/**"]]+ ]+ })++ assert json_response(conn, 200)["results"] == [%{"metrics" => [1], "dimensions" => []}]+ end++ test "wildcard and member filter combined", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:pageview, pathname: "/en/page1"),+ build(:pageview, pathname: "/en/page2"),+ build(:pageview, pathname: "/pl/page1"),+ build(:pageview, pathname: "/ee/page1")+ ])++ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "date_range" => "all",+ "metrics" => ["visitors"],+ "filters" => [+ ["matches", "event:page", ["/en/**", "/pl/**"]]+ ]+ })++ assert json_response(conn, 200)["results"] == [%{"metrics" => [3], "dimensions" => []}]+ end++ test "can escape pipe character in member + wildcard filter", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:pageview, pathname: "/blog/post|1"),+ build(:pageview, pathname: "/otherpost|1"),+ build(:pageview, pathname: "/blog/post|2"),+ build(:pageview, pathname: "/something-else")+ ])++ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "date_range" => "all",+ "metrics" => ["visitors"],+ "filters" => [+ ["matches", "event:page", ["**post\\|1", "/something-else"]]+ ]+ })++ assert json_response(conn, 200)["results"] == [%{"metrics" => [3], "dimensions" => []}]+ end++ test "handles filtering by visit country", %{+ conn: conn,+ site: site+ } do+ populate_stats(site, [+ build(:pageview, country_code: "EE"),+ build(:pageview, country_code: "EE"),+ build(:pageview, country_code: "EE")+ ])++ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "date_range" => "all",+ "metrics" => ["pageviews"],+ "filters" => [["is", "visit:country", ["EE"]]]+ })++ assert json_response(conn, 200)["results"] == [%{"metrics" => [3], "dimensions" => []}]+ end+ end++ describe "aggregation with imported data" do+ setup :create_site_import++ test "does not count imported stats unless specified", %{+ conn: conn,+ site: site,+ site_import: site_import+ } do+ populate_stats(site, site_import.id, [+ build(:imported_visitors, date: ~D[2023-01-01]),+ build(:pageview, timestamp: ~N[2023-01-01 00:00:00])+ ])++ query_params = %{+ "site_id" => site.domain,+ "date_range" => "all",+ "metrics" => ["pageviews"]+ }++ conn1 = post(conn, "/api/v2/query", query_params)++ assert json_response(conn1, 200)["results"] == [%{"metrics" => [1], "dimensions" => []}]++ conn2 = post(conn, "/api/v2/query", Map.put(query_params, "include", %{"imports" => true}))++ assert json_response(conn2, 200)["results"] == [%{"metrics" => [2], "dimensions" => []}]+ refute json_response(conn2, 200)["meta"]["warning"]+ end+ end++ describe "timeseries" do+ test "shows hourly data for a certain date", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:pageview, user_id: @user_id, timestamp: ~N[2021-01-01 00:00:00]),+ build(:pageview, user_id: @user_id, timestamp: ~N[2021-01-01 00:10:00]),+ build(:pageview, timestamp: ~N[2021-01-01 23:59:00])+ ])++ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "metrics" => ["visitors", "pageviews", "visits", "visit_duration", "bounce_rate"],+ "date_range" => ["2021-01-01", "2021-01-01"],+ "dimensions" => ["time:hour"]+ })++ assert json_response(conn, 200)["results"] == [+ %{"dimensions" => ["2021-01-01T00:00:00Z"], "metrics" => [1, 2, 1, 600, 0]},+ %{"dimensions" => ["2021-01-01T23:00:00Z"], "metrics" => [1, 1, 1, 0, 100]}+ ]+ end++ test "shows last 7 days of visitors", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:pageview, timestamp: ~N[2021-01-01 00:00:00]),+ build(:pageview, timestamp: ~N[2021-01-07 23:59:00])+ ])++ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "metrics" => ["visitors"],+ "date_range" => ["2021-01-01", "2021-01-07"],+ "dimensions" => ["time"]+ })++ assert json_response(conn, 200)["results"] == [+ %{"dimensions" => ["2021-01-01"], "metrics" => [1]},+ %{"dimensions" => ["2021-01-07"], "metrics" => [1]}+ ]+ end++ test "shows last 6 months of visitors", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:pageview, timestamp: ~N[2020-08-13 00:00:00]),+ build(:pageview, timestamp: ~N[2020-12-31 00:00:00]),+ build(:pageview, timestamp: ~N[2021-01-01 00:00:00]),+ build(:pageview, timestamp: ~N[2021-01-01 00:00:00])+ ])++ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "metrics" => ["visitors"],+ "date_range" => ["2020-07-01", "2021-01-31"],+ "dimensions" => ["time"]+ })++ assert json_response(conn, 200)["results"] == [+ %{"dimensions" => ["2020-08-01"], "metrics" => [1]},+ %{"dimensions" => ["2020-12-01"], "metrics" => [1]},+ %{"dimensions" => ["2021-01-01"], "metrics" => [2]}+ ]+ end++ test "shows last 12 months of visitors", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:pageview, timestamp: ~N[2020-02-01 00:00:00]),+ build(:pageview, timestamp: ~N[2020-12-31 00:00:00]),+ build(:pageview, timestamp: ~N[2021-01-01 00:00:00]),+ build(:pageview, timestamp: ~N[2021-01-01 00:00:00])+ ])++ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "metrics" => ["visitors"],+ "date_range" => ["2020-01-01", "2021-01-01"],+ "dimensions" => ["time"]+ })++ assert json_response(conn, 200)["results"] == [+ %{"dimensions" => ["2020-02-01"], "metrics" => [1]},+ %{"dimensions" => ["2020-12-01"], "metrics" => [1]},+ %{"dimensions" => ["2021-01-01"], "metrics" => [2]}+ ]+ end++ test "shows last 12 months of visitors with interval daily", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:pageview, timestamp: ~N[2020-02-01 00:00:00]),+ build(:pageview, timestamp: ~N[2020-12-31 00:00:00]),+ build(:pageview, timestamp: ~N[2021-01-01 00:00:00]),+ build(:pageview, timestamp: ~N[2021-01-01 00:00:00])+ ])++ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "metrics" => ["visitors"],+ "date_range" => ["2020-01-01", "2021-01-07"],+ "dimensions" => ["time:day"]+ })++ assert json_response(conn, 200)["results"] == [+ %{"dimensions" => ["2020-02-01"], "metrics" => [1]},+ %{"dimensions" => ["2020-12-31"], "metrics" => [1]},+ %{"dimensions" => ["2021-01-01"], "metrics" => [2]}+ ]+ end++ test "shows a custom range with daily interval", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:pageview, timestamp: ~N[2021-01-01 00:00:00]),+ build(:pageview, timestamp: ~N[2021-01-01 00:00:00]),+ build(:pageview, timestamp: ~N[2021-01-02 00:00:00])+ ])++ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "metrics" => ["visitors"],+ "date_range" => ["2021-01-01", "2021-01-02"],+ "dimensions" => ["time:day"]+ })++ assert json_response(conn, 200)["results"] == [+ %{"dimensions" => ["2021-01-01"], "metrics" => [2]},+ %{"dimensions" => ["2021-01-02"], "metrics" => [1]}+ ]+ end++ test "shows a custom range with monthly interval", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:pageview, user_id: @user_id, timestamp: ~N[2020-12-01 00:00:00]),+ build(:pageview, user_id: @user_id, timestamp: ~N[2020-12-01 00:05:00]),+ build(:pageview, timestamp: ~N[2021-01-01 00:00:00]),+ build(:pageview, timestamp: ~N[2021-01-02 00:00:00])+ ])++ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "metrics" => ["pageviews", "visitors", "bounce_rate", "visit_duration"],+ "date_range" => ["2020-12-01", "2021-01-02"],+ "dimensions" => ["time:month"]+ })++ assert json_response(conn, 200)["results"] == [+ %{"dimensions" => ["2020-12-01"], "metrics" => [2, 1, 0, 300]},+ %{"dimensions" => ["2021-01-01"], "metrics" => [2, 2, 100, 0]}+ ]+ end++ test "timeseries with explicit order_by", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:pageview, timestamp: ~N[2021-01-01 00:00:00]),+ build(:pageview, timestamp: ~N[2021-01-02 00:00:00]),+ build(:pageview, timestamp: ~N[2021-01-02 00:00:00]),+ build(:pageview, timestamp: ~N[2021-01-02 00:00:00]),+ build(:pageview, timestamp: ~N[2021-01-03 00:00:00]),+ build(:pageview, timestamp: ~N[2021-01-03 00:00:00]),+ build(:pageview, timestamp: ~N[2021-01-04 00:00:00]),+ build(:pageview, timestamp: ~N[2021-01-04 00:00:00])+ ])++ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "metrics" => ["pageviews"],+ "date_range" => ["2020-12-01", "2021-01-04"],+ "dimensions" => ["time"],+ "order_by" => [["pageviews", "desc"], ["time", "asc"]]+ })++ assert json_response(conn, 200)["results"] == [+ %{"dimensions" => ["2021-01-02"], "metrics" => [3]},+ %{"dimensions" => ["2021-01-03"], "metrics" => [2]},+ %{"dimensions" => ["2021-01-04"], "metrics" => [2]},+ %{"dimensions" => ["2021-01-01"], "metrics" => [1]}+ ]endend@@ -246,159 +1326,113 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest do"dimensions" => ["visit:country"]})- %{"results" => results} = json_response(conn, 200)-- assert results == [- %{"dimensions" => ["EE"], "metrics" => [2]},- %{"dimensions" => ["US"], "metrics" => [1]}+ %{"results" => results} = json_response(conn, 200)++ assert results == [+ %{"dimensions" => ["EE"], "metrics" => [2]},+ %{"dimensions" => ["US"], "metrics" => [1]}+ ]+ end++ test "breaks down all metrics by visit:referrer with imported data", %{conn: conn, site: site} do+ site_import =+ insert(:site_import,+ site: site,+ start_date: ~D[2005-01-01],+ end_date: Timex.today(),+ source: :universal_analytics+ )++ populate_stats(site, site_import.id, [+ build(:pageview, referrer: "site.com", timestamp: ~N[2021-01-01 00:00:00]),+ build(:pageview, referrer: "site.com/1", timestamp: ~N[2021-01-01 00:00:00]),+ build(:imported_sources,+ referrer: "site.com",+ date: ~D[2021-01-01],+ visitors: 2,+ visits: 2,+ pageviews: 2,+ bounces: 1,+ visit_duration: 120+ ),+ build(:imported_sources,+ referrer: "site.com/2",+ date: ~D[2021-01-01],+ visitors: 2,+ visits: 2,+ pageviews: 2,+ bounces: 2,+ visit_duration: 0+ ),+ build(:imported_sources,+ date: ~D[2021-01-01],+ visitors: 10,+ visits: 11,+ pageviews: 50,+ bounces: 0,+ visit_duration: 1100+ )+ ])++ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "metrics" => ["visitors", "visits", "pageviews", "bounce_rate", "visit_duration"],+ "date_range" => "all",+ "dimensions" => ["visit:referrer"],+ "include" => %{"imports" => true}+ })++ assert json_response(conn, 200)["results"] == [+ %{"dimensions" => ["Direct / None"], "metrics" => [10, 11, 50, 0.0, 100.0]},+ %{"dimensions" => ["site.com"], "metrics" => [3, 3, 3, 67.0, 40.0]},+ %{"dimensions" => ["site.com/2"], "metrics" => [2, 2, 2, 100.0, 0.0]},+ %{"dimensions" => ["site.com/1"], "metrics" => [1, 1, 1, 100.0, 0.0]}]end- # test "breaks down all metrics by visit:referrer with imported data", %{conn: conn, site: site} do- # site_import =- # insert(:site_import,- # site: site,- # start_date: ~D[2005-01-01],- # end_date: Timex.today(),- # source: :universal_analytics- # )-- # populate_stats(site, site_import.id, [- # build(:pageview, referrer: "site.com", timestamp: ~N[2021-01-01 00:00:00]),- # build(:pageview, referrer: "site.com/1", timestamp: ~N[2021-01-01 00:00:00]),- # build(:imported_sources,- # referrer: "site.com",- # date: ~D[2021-01-01],- # visitors: 2,- # visits: 2,- # pageviews: 2,- # bounces: 1,- # visit_duration: 120- # ),- # build(:imported_sources,- # referrer: "site.com/2",- # date: ~D[2021-01-01],- # visitors: 2,- # visits: 2,- # pageviews: 2,- # bounces: 2,- # visit_duration: 0- # ),- # build(:imported_sources,- # date: ~D[2021-01-01],- # visitors: 10,- # visits: 11,- # pageviews: 50,- # bounces: 0,- # visit_duration: 1100- # )- # ])-- # conn =- # get(conn, "/api/v1/stats/breakdown", %{- # "site_id" => site.domain,- # "period" => "day",- # "metrics" => "visitors,visits,pageviews,bounce_rate,visit_duration",- # "date" => "2021-01-01",- # "property" => "visit:referrer",- # "with_imported" => "true"- # })-- # assert json_response(conn, 200) == %{- # "results" => [- # %{- # "referrer" => "Direct / None",- # "visitors" => 10,- # "visits" => 11,- # "pageviews" => 50,- # "bounce_rate" => 0,- # "visit_duration" => 100- # },- # %{- # "referrer" => "site.com",- # "visitors" => 3,- # "visits" => 3,- # "pageviews" => 3,- # "bounce_rate" => 67.0,- # "visit_duration" => 40- # },- # %{- # "referrer" => "site.com/2",- # "visitors" => 2,- # "visits" => 2,- # "pageviews" => 2,- # "bounce_rate" => 100.0,- # "visit_duration" => 0- # },- # %{- # "referrer" => "site.com/1",- # "visitors" => 1,- # "visits" => 1,- # "pageviews" => 1,- # "bounce_rate" => 100.0,- # "visit_duration" => 0- # }- # ]- # }-- # conn =- # post(conn, "/api/v2/query", %{- # "site_id" => site.domain,- # "metrics" => ["visitors", "visits", "pageviews", "bounce_rate", "visit_duration"],- # "date_range" => "all",- # "dimensions" => ["visit:referrer"],- # })-- # %{"results" => results} = json_response(conn, 200)-- # assert results == [- # %{"dimensions" => ["EE"], "metrics" => [2]},- # %{"dimensions" => ["US"], "metrics" => [1]},- # ]- # end-- # for {property, attr} <- [- # {"visit:utm_campaign", :utm_campaign},- # {"visit:utm_source", :utm_source},- # {"visit:utm_term", :utm_term},- # {"visit:utm_content", :utm_content}- # ] do- # test "breakdown by #{property} when filtered by hostname", %{conn: conn, site: site} do- # populate_stats(site, [- # # session starts at two.example.com with utm_param=ad- # build(- # :pageview,- # [- # {unquote(attr), "ad"},- # {:user_id, @user_id},- # {:hostname, "two.example.com"},- # {:timestamp, ~N[2021-01-01 00:00:00]}- # ]- # ),- # # session continues on one.example.com without any utm_params- # build(- # :pageview,- # [- # {:user_id, @user_id},- # {:hostname, "one.example.com"},- # {:timestamp, ~N[2021-01-01 00:15:00]}- # ]- # )- # ])-- # conn =- # get(conn, "/api/v1/stats/breakdown", %{- # "site_id" => site.domain,- # "period" => "day",- # "date" => "2021-01-01",- # "filters" => "event:hostname==one.example.com",- # "property" => unquote(property)- # })-- # # nobody landed on one.example.com from utm_param=ad- # assert json_response(conn, 200) == %{"results" => []}- # end- # end+ for {dimension, attr} <- [+ {"visit:utm_campaign", :utm_campaign},+ {"visit:utm_source", :utm_source},+ {"visit:utm_term", :utm_term},+ {"visit:utm_content", :utm_content}+ ] do+ test "breakdown by #{dimension} when filtered by hostname", %{conn: conn, site: site} do+ populate_stats(site, [+ # session starts at two.example.com with utm_param=ad+ build(+ :pageview,+ [+ {unquote(attr), "ad"},+ {:user_id, @user_id},+ {:hostname, "two.example.com"},+ {:timestamp, ~N[2021-01-01 00:00:00]}+ ]+ ),+ # session continues on one.example.com without any utm_params+ build(+ :pageview,+ [+ {:user_id, @user_id},+ {:hostname, "one.example.com"},+ {:timestamp, ~N[2021-01-01 00:15:00]}+ ]+ )+ ])++ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "metrics" => ["visitors"],+ "date_range" => "all",+ "filters" => [["is", "event:hostname", ["one.example.com"]]],+ "dimensions" => [unquote(dimension)]+ })++ # nobody landed on one.example.com from utm_param=ad+ assert json_response(conn, 200)["results"] == []+ end+ endfor {dimension, column, value1, value2, blank_value} <- [{"visit:source", :referrer_source, "Google", "Twitter", "Direct / None"},@@ -610,125 +1644,101 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest do]end- # test "pageviews breakdown by event:page - imported data having pageviews=0 and visitors=n should be bypassed",- # %{conn: conn, site: site} do- # site_import =- # insert(:site_import,- # site: site,- # start_date: ~D[2005-01-01],- # end_date: Timex.today(),- # source: :universal_analytics- # )-- # populate_stats(site, site_import.id, [- # build(:pageview, pathname: "/", timestamp: ~N[2021-01-01 00:00:00]),- # build(:pageview, pathname: "/", timestamp: ~N[2021-01-01 00:25:00]),- # build(:pageview,- # pathname: "/plausible.io",- # timestamp: ~N[2021-01-01 00:00:00]- # ),- # build(:imported_pages,- # page: "/skip-me",- # date: ~D[2021-01-01],- # visitors: 1,- # pageviews: 0- # ),- # build(:imported_pages,- # page: "/include-me",- # date: ~D[2021-01-01],- # visitors: 1,- # pageviews: 1- # )- # ])-- # conn =- # get(conn, "/api/v1/stats/breakdown", %{- # "site_id" => site.domain,- # "period" => "day",- # "date" => "2021-01-01",- # "property" => "event:page",- # "with_imported" => "true",- # "metrics" => "pageviews"- # })-- # assert json_response(conn, 200) == %{- # "results" => [- # %{"page" => "/", "pageviews" => 2},- # %{"page" => "/plausible.io", "pageviews" => 1},- # %{"page" => "/include-me", "pageviews" => 1}- # ]- # }- # end-- # test "breakdown by event:page", %{conn: conn, site: site} do- # populate_stats(site, [- # build(:pageview, pathname: "/", timestamp: ~N[2021-01-01 00:00:00]),- # build(:pageview, pathname: "/", timestamp: ~N[2021-01-01 00:25:00]),- # build(:pageview,- # pathname: "/plausible.io",- # timestamp: ~N[2021-01-01 00:00:00]- # )- # ])-- # conn =- # get(conn, "/api/v1/stats/breakdown", %{- # "site_id" => site.domain,- # "period" => "day",- # "date" => "2021-01-01",- # "property" => "event:page"- # })-- # assert json_response(conn, 200) == %{- # "results" => [- # %{"page" => "/", "visitors" => 2},- # %{"page" => "/plausible.io", "visitors" => 1}- # ]- # }- # end-- # test "breakdown by event:page when there are no events in the second page", %{- # conn: conn,- # site: site- # } do- # populate_stats(site, [- # build(:pageview, pathname: "/", timestamp: ~N[2021-01-01 00:00:00]),- # build(:pageview, pathname: "/", timestamp: ~N[2021-01-01 00:25:00]),- # build(:pageview,- # pathname: "/plausible.io",- # timestamp: ~N[2021-01-01 00:00:00]- # )- # ])-- # conn =- # get(conn, "/api/v1/stats/breakdown", %{- # "site_id" => site.domain,- # "period" => "day",- # "date" => "2021-01-01",- # "property" => "event:page",- # "metrics" => "visitors,bounce_rate",- # "page" => 2,- # "limit" => 2- # })-- # assert json_response(conn, 200) == %{"results" => []}- # end-- # test "attempting to breakdown by event:hostname returns an error", %{conn: conn, site: site} do- # conn =- # get(conn, "/api/v1/stats/breakdown", %{- # "site_id" => site.domain,- # "period" => "day",- # "date" => "2021-01-01",- # "property" => "event:hostname",- # "with_imported" => "true"- # })-- # assert %{- # "error" => error- # } = json_response(conn, 400)-- # assert error =~ "Property 'event:hostname' is currently not supported for breakdowns."- # end+ test "pageviews breakdown by event:page - imported data having pageviews=0 and visitors=n should be bypassed",+ %{conn: conn, site: site} do+ site_import =+ insert(:site_import,+ site: site,+ start_date: ~D[2005-01-01],+ end_date: Timex.today(),+ source: :universal_analytics+ )++ populate_stats(site, site_import.id, [+ build(:pageview, pathname: "/", timestamp: ~N[2021-01-01 00:00:00]),+ build(:pageview, pathname: "/", timestamp: ~N[2021-01-01 00:25:00]),+ build(:pageview,+ pathname: "/plausible.io",+ timestamp: ~N[2021-01-01 00:00:00]+ ),+ build(:imported_pages,+ page: "/skip-me",+ date: ~D[2021-01-01],+ visitors: 1,+ pageviews: 0+ ),+ build(:imported_pages,+ page: "/include-me",+ date: ~D[2021-01-01],+ visitors: 1,+ pageviews: 1+ )+ ])++ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "metrics" => ["pageviews"],+ "date_range" => "all",+ "dimensions" => ["event:page"],+ "include" => %{"imports" => true}+ })++ assert json_response(conn, 200)["results"] == [+ %{"dimensions" => ["/"], "metrics" => [2]},+ %{"dimensions" => ["/plausible.io"], "metrics" => [1]},+ %{"dimensions" => ["/include-me"], "metrics" => [1]}+ ]+ end++ test "breakdown by event:page", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:pageview, pathname: "/", timestamp: ~N[2021-01-01 00:00:00]),+ build(:pageview, pathname: "/", timestamp: ~N[2021-01-01 00:25:00]),+ build(:pageview,+ pathname: "/plausible.io",+ timestamp: ~N[2021-01-01 00:00:00]+ )+ ])++ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "metrics" => ["visitors"],+ "date_range" => "all",+ "dimensions" => ["event:page"]+ })++ assert json_response(conn, 200)["results"] == [+ %{"dimensions" => ["/"], "metrics" => [2]},+ %{"dimensions" => ["/plausible.io"], "metrics" => [1]}+ ]+ end++ test "attempting to breakdown by event:hostname returns an error", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:pageview, hostname: "a.example.com"),+ build(:pageview, hostname: "a.example.com"),+ build(:pageview, hostname: "a.example.com"),+ build(:pageview, hostname: "b.example.com")+ ])++ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "date_range" => "all",+ "metrics" => ["pageviews"],+ "dimensions" => ["event:hostname"],+ "with_imported" => "true"+ })++ %{"results" => results} = json_response(conn, 200)++ assert results == [+ %{"dimensions" => ["a.example.com"], "metrics" => [3]},+ %{"dimensions" => ["b.example.com"], "metrics" => [1]}+ ]+ enddescribe "breakdown by visit:exit_page" dosetup %{site: site} do@@ -1234,183 +2244,87 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest do%{"dimensions" => ["(none)"], "metrics" => [1]}]end+ end++ describe "breakdown by event:goal" do+ test "returns custom event goals and pageview goals", %{conn: conn, site: site} do+ insert(:goal, %{site: site, event_name: "Purchase"})+ insert(:goal, %{site: site, page_path: "/test"})++ populate_stats(site, [+ build(:pageview,+ timestamp: ~N[2021-01-01 00:00:01],+ pathname: "/test"+ ),+ build(:event,+ name: "Purchase",+ timestamp: ~N[2021-01-01 00:00:03]+ ),+ build(:event,+ name: "Purchase",+ timestamp: ~N[2021-01-01 00:00:03]+ )+ ])++ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "date_range" => "all",+ "metrics" => ["visitors"],+ "dimensions" => ["event:goal"]+ })++ assert json_response(conn, 200)["results"] == [+ %{"dimensions" => ["Purchase"], "metrics" => [2]},+ %{"dimensions" => ["Visit /test"], "metrics" => [1]}+ ]+ end++ test "returns pageview goals containing wildcards", %{conn: conn, site: site} do+ insert(:goal, %{site: site, page_path: "/**/post"})+ insert(:goal, %{site: site, page_path: "/blog**"})++ populate_stats(site, [+ build(:pageview, pathname: "/blog", user_id: @user_id),+ build(:pageview, pathname: "/blog/post-1", user_id: @user_id),+ build(:pageview, pathname: "/blog/post-2", user_id: @user_id),+ build(:pageview, pathname: "/blog/something/post"),+ build(:pageview, pathname: "/different/page/post")+ ])++ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "date_range" => "all",+ "metrics" => ["visitors", "pageviews"],+ "dimensions" => ["event:goal"],+ "order_by" => [["pageviews", "desc"]]+ })++ assert json_response(conn, 200)["results"] == [+ %{"dimensions" => ["Visit /blog**"], "metrics" => [2, 4]},+ %{"dimensions" => ["Visit /**/post"], "metrics" => [2, 2]}+ ]+ end++ test "does not return goals that are not configured for the site", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:pageview, pathname: "/register"),+ build(:event, name: "Signup")+ ])++ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "date_range" => "all",+ "metrics" => ["visitors", "pageviews"],+ "dimensions" => ["event:goal"]+ })- # test "breakdown by custom event property, limited", %{conn: conn, site: site} do- # populate_stats(site, [- # build(:event,- # name: "Purchase",- # "meta.key": ["cost"],- # "meta.value": ["16"],- # timestamp: ~N[2021-01-01 00:00:00]- # ),- # build(:event,- # name: "Purchase",- # "meta.key": ["cost"],- # "meta.value": ["18"],- # timestamp: ~N[2021-01-01 00:25:00]- # ),- # build(:event,- # name: "Purchase",- # "meta.key": ["cost"],- # "meta.value": ["14"],- # timestamp: ~N[2021-01-01 00:25:00]- # ),- # build(:event,- # name: "Purchase",- # "meta.key": ["cost"],- # "meta.value": ["14"],- # timestamp: ~N[2021-01-01 00:26:00]- # )- # ])-- # conn =- # get(conn, "/api/v1/stats/breakdown", %{- # "site_id" => site.domain,- # "period" => "day",- # "date" => "2021-01-01",- # "property" => "event:props:cost",- # "filters" => "event:name==Purchase",- # "limit" => 2- # })-- # assert json_response(conn, 200) == %{- # "results" => [- # %{"cost" => "14", "visitors" => 2},- # %{"cost" => "16", "visitors" => 1}- # ]- # }- # end-- # test "breakdown by custom event property, paginated", %{conn: conn, site: site} do- # populate_stats(site, [- # build(:event,- # name: "Purchase",- # "meta.key": ["cost"],- # "meta.value": ["16"],- # timestamp: ~N[2021-01-01 00:00:00]- # ),- # build(:event,- # name: "Purchase",- # "meta.key": ["cost"],- # "meta.value": ["16"],- # timestamp: ~N[2021-01-01 00:00:00]- # ),- # build(:event,- # name: "Purchase",- # "meta.key": ["cost"],- # "meta.value": ["18"],- # timestamp: ~N[2021-01-01 00:25:00]- # ),- # build(:event,- # name: "Purchase",- # "meta.key": ["cost"],- # "meta.value": ["14"],- # timestamp: ~N[2021-01-01 00:25:00]- # ),- # build(:event,- # name: "Purchase",- # "meta.key": ["cost"],- # "meta.value": ["14"],- # timestamp: ~N[2021-01-01 00:26:00]- # )- # ])-- # conn =- # get(conn, "/api/v1/stats/breakdown", %{- # "site_id" => site.domain,- # "period" => "day",- # "date" => "2021-01-01",- # "property" => "event:props:cost",- # "filters" => "event:name==Purchase",- # "limit" => 2,- # "page" => 2- # })-- # assert json_response(conn, 200) == %{- # "results" => [- # %{"cost" => "18", "visitors" => 1}- # ]- # }- # end+ assert json_response(conn, 200)["results"] == []+ endend- # describe "breakdown by event:goal" do- # test "returns custom event goals and pageview goals", %{conn: conn, site: site} do- # insert(:goal, %{site: site, event_name: "Purchase"})- # insert(:goal, %{site: site, page_path: "/test"})-- # populate_stats(site, [- # build(:pageview,- # timestamp: ~N[2021-01-01 00:00:01],- # pathname: "/test"- # ),- # build(:event,- # name: "Purchase",- # timestamp: ~N[2021-01-01 00:00:03]- # ),- # build(:event,- # name: "Purchase",- # timestamp: ~N[2021-01-01 00:00:03]- # )- # ])-- # conn =- # get(conn, "/api/v1/stats/breakdown", %{- # "site_id" => site.domain,- # "period" => "day",- # "date" => "2021-01-01",- # "property" => "event:goal"- # })-- # assert [- # %{"goal" => "Purchase", "visitors" => 2},- # %{"goal" => "Visit /test", "visitors" => 1}- # ] = json_response(conn, 200)["results"]- # end-- # test "returns pageview goals containing wildcards", %{conn: conn, site: site} do- # insert(:goal, %{site: site, page_path: "/**/post"})- # insert(:goal, %{site: site, page_path: "/blog**"})-- # populate_stats(site, [- # build(:pageview, pathname: "/blog", user_id: @user_id),- # build(:pageview, pathname: "/blog/post-1", user_id: @user_id),- # build(:pageview, pathname: "/blog/post-2", user_id: @user_id),- # build(:pageview, pathname: "/blog/something/post"),- # build(:pageview, pathname: "/different/page/post")- # ])-- # conn =- # get(conn, "/api/v1/stats/breakdown", %{- # "site_id" => site.domain,- # "metrics" => "visitors,pageviews",- # "property" => "event:goal"- # })-- # assert [- # %{"goal" => "Visit /**/post", "visitors" => 2, "pageviews" => 2},- # %{"goal" => "Visit /blog**", "visitors" => 2, "pageviews" => 4}- # ] = json_response(conn, 200)["results"]- # end-- # test "does not return goals that are not configured for the site", %{conn: conn, site: site} do- # populate_stats(site, [- # build(:pageview, pathname: "/register"),- # build(:event, name: "Signup")- # ])-- # conn =- # get(conn, "/api/v1/stats/breakdown", %{- # "site_id" => site.domain,- # "metrics" => "visitors,pageviews",- # "property" => "event:goal"- # })-- # assert [] = json_response(conn, 200)["results"]- # end- # end-- # describe "filtering" dotest "event:goal filter returns 400 when goal not configured", %{conn: conn, site: site} doconn =post(conn, "/api/v2/query", %{@@ -1456,14 +2370,144 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest dotimestamp: ~N[2021-01-01 00:00:00]),build(:pageview,- pathname: "/plausible.io",- browser: "Chrome",- timestamp: ~N[2021-01-01 00:25:00]+ pathname: "/plausible.io",+ browser: "Chrome",+ timestamp: ~N[2021-01-01 00:25:00]+ ),+ build(:pageview,+ browser: "Safari",+ pathname: "/plausible.io",+ timestamp: ~N[2021-01-01 00:00:00]+ )+ ])++ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "metrics" => ["visitors"],+ "date_range" => "all",+ "dimensions" => ["visit:browser"],+ "filters" => [+ ["is", "event:page", ["/plausible.io"]]+ ]+ })++ %{"results" => results} = json_response(conn, 200)++ assert results == [+ %{"dimensions" => ["Chrome"], "metrics" => [2]},+ %{"dimensions" => ["Safari"], "metrics" => [1]}+ ]+ end++ test "event:page filter shows sources of sessions that have visited that page", %{+ conn: conn,+ site: site+ } do+ populate_stats(site, [+ build(:pageview,+ pathname: "/",+ referrer_source: "Twitter",+ utm_medium: "Twitter",+ utm_source: "Twitter",+ utm_campaign: "Twitter",+ user_id: @user_id+ ),+ build(:pageview,+ pathname: "/plausible.io",+ user_id: @user_id+ ),+ build(:pageview,+ pathname: "/plausible.io",+ referrer_source: "Google",+ utm_medium: "Google",+ utm_source: "Google",+ utm_campaign: "Google"+ ),+ build(:pageview,+ pathname: "/plausible.io",+ referrer_source: "Google",+ utm_medium: "Google",+ utm_source: "Google",+ utm_campaign: "Google"+ )+ ])++ for dimension <- [+ "visit:source",+ "visit:utm_medium",+ "visit:utm_source",+ "visit:utm_campaign"+ ] do+ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "metrics" => ["visitors"],+ "date_range" => "all",+ "dimensions" => [dimension],+ "filters" => [+ ["is", "event:page", ["/plausible.io"]]+ ]+ })++ %{"results" => results} = json_response(conn, 200)++ assert results == [+ %{"dimensions" => ["Google"], "metrics" => [2]},+ %{"dimensions" => ["Twitter"], "metrics" => [1]}+ ]+ end+ end++ test "top sources for a custom goal and filtered by hostname", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:pageview,+ hostname: "blog.example.com",+ referrer_source: "Facebook",+ user_id: @user_id+ ),+ build(:pageview,+ hostname: "app.example.com",+ pathname: "/register",+ user_id: @user_id+ ),+ build(:event,+ name: "Signup",+ hostname: "app.example.com",+ pathname: "/register",+ user_id: @user_id+ )+ ])++ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "metrics" => ["visitors"],+ "date_range" => "all",+ "dimensions" => ["visit:source"],+ "filters" => [+ ["is", "event:hostname", ["app.example.com"]]+ ]+ })++ %{"results" => results} = json_response(conn, 200)++ assert results == []+ end++ test "top sources for a custom goal and filtered by hostname (2)", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:pageview,+ hostname: "app.example.com",+ referrer_source: "Facebook",+ pathname: "/register",+ user_id: @user_id),- build(:pageview,- browser: "Safari",- pathname: "/plausible.io",- timestamp: ~N[2021-01-01 00:00:00]+ build(:event,+ name: "Signup",+ hostname: "app.example.com",+ pathname: "/register",+ user_id: @user_id)])@@ -1472,201 +2516,67 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest do"site_id" => site.domain,"metrics" => ["visitors"],"date_range" => "all",- "dimensions" => ["visit:browser"],+ "dimensions" => ["visit:source"],"filters" => [- ["is", "event:page", ["/plausible.io"]]+ ["is", "event:hostname", ["app.example.com"]]]})- %{"results" => results} = json_response(conn, 200)-- assert results == [- %{"dimensions" => ["Chrome"], "metrics" => [2]},- %{"dimensions" => ["Safari"], "metrics" => [1]}+ assert json_response(conn, 200)["results"] == [+ %{"dimensions" => ["Facebook"], "metrics" => [1]}]end- test "event:page filter shows sources of sessions that have visited that page", %{+ test "event:page filter is interpreted as entry_page filter only for bounce_rate", %{conn: conn,site: site} dopopulate_stats(site, [build(:pageview,- pathname: "/",- referrer_source: "Twitter",- utm_medium: "Twitter",- utm_source: "Twitter",- utm_campaign: "Twitter",- user_id: @user_id+ browser: "Chrome",+ user_id: @user_id,+ pathname: "/ignore",+ timestamp: ~N[2021-01-01 00:00:00]),build(:pageview,+ browser: "Chrome",+ user_id: @user_id,pathname: "/plausible.io",- user_id: @user_id+ timestamp: ~N[2021-01-01 00:01:00]),build(:pageview,- pathname: "/plausible.io",- referrer_source: "Google",- utm_medium: "Google",- utm_source: "Google",- utm_campaign: "Google"+ browser: "Chrome",+ user_id: 456,+ pathname: "/important-page",+ timestamp: ~N[2021-01-01 00:00:00]+ ),+ build(:pageview,+ browser: "Chrome",+ user_id: 456,+ pathname: "/",+ timestamp: ~N[2021-01-01 00:01:00]),build(:pageview,+ browser: "Chrome",pathname: "/plausible.io",- referrer_source: "Google",- utm_medium: "Google",- utm_source: "Google",- utm_campaign: "Google"+ timestamp: ~N[2021-01-01 00:01:00])])- for dimension <- [- "visit:source",- "visit:utm_medium",- "visit:utm_source",- "visit:utm_campaign"- ] do- conn =- post(conn, "/api/v2/query", %{- "site_id" => site.domain,- "metrics" => ["visitors"],- "date_range" => "all",- "dimensions" => [dimension],- "filters" => [- ["is", "event:page", ["/plausible.io"]]- ]- })-- %{"results" => results} = json_response(conn, 200)+ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "date_range" => "all",+ "metrics" => ["visitors", "bounce_rate"],+ "filters" => [["is", "event:page", ["/plausible.io", "/important-page"]]],+ "dimensions" => ["visit:browser"]+ })- assert results == [- %{"dimensions" => ["Google"], "metrics" => [2]},- %{"dimensions" => ["Twitter"], "metrics" => [1]}- ]- end+ assert json_response(conn, 200)["results"] == [+ %{"dimensions" => ["Chrome"], "metrics" => [3, 50]}+ ]end- # test "top sources for a custom goal and filtered by hostname", %{conn: conn, site: site} do- # populate_stats(site, [- # build(:pageview,- # hostname: "blog.example.com",- # referrer_source: "Facebook",- # user_id: @user_id- # ),- # build(:pageview,- # hostname: "app.example.com",- # pathname: "/register",- # user_id: @user_id- # ),- # build(:event,- # name: "Signup",- # hostname: "app.example.com",- # pathname: "/register",- # user_id: @user_id- # )- # ])-- # conn =- # post(conn, "/api/v2/query", %{- # "site_id" => site.domain,- # "metrics" => ["visitors"],- # "date_range" => "all",- # "dimensions" => ["visit:source"],- # "filters" => [- # ["is", "event:hostname", ["app.example.com"]]- # ]- # })-- # %{"results" => results} = json_response(conn, 200)-- # assert results == []- # end-- # test "top sources for a custom goal and filtered by hostname (2)", %{conn: conn, site: site} do- # populate_stats(site, [- # build(:pageview,- # hostname: "app.example.com",- # referrer_source: "Facebook",- # pathname: "/register",- # user_id: @user_id- # ),- # build(:event,- # name: "Signup",- # hostname: "app.example.com",- # pathname: "/register",- # user_id: @user_id- # )- # ])-- # conn =- # get(conn, "/api/v1/stats/breakdown", %{- # "site_id" => site.domain,- # "period" => "day",- # "property" => "visit:source",- # "filters" => "event:hostname==app.example.com"- # })-- # assert json_response(conn, 200) == %{- # "results" => [%{"source" => "Facebook", "visitors" => 1}]- # }- # end-- # test "event:page filter is interpreted as entry_page filter only for bounce_rate", %{- # conn: conn,- # site: site- # } do- # populate_stats(site, [- # build(:pageview,- # browser: "Chrome",- # user_id: @user_id,- # pathname: "/ignore",- # timestamp: ~N[2021-01-01 00:00:00]- # ),- # build(:pageview,- # browser: "Chrome",- # user_id: @user_id,- # pathname: "/plausible.io",- # timestamp: ~N[2021-01-01 00:01:00]- # ),- # build(:pageview,- # browser: "Chrome",- # user_id: 456,- # pathname: "/important-page",- # timestamp: ~N[2021-01-01 00:00:00]- # ),- # build(:pageview,- # browser: "Chrome",- # user_id: 456,- # pathname: "/",- # timestamp: ~N[2021-01-01 00:01:00]- # ),- # build(:pageview,- # browser: "Chrome",- # pathname: "/plausible.io",- # timestamp: ~N[2021-01-01 00:01:00]- # )- # ])-- # conn =- # get(conn, "/api/v1/stats/breakdown", %{- # "site_id" => site.domain,- # "period" => "day",- # "date" => "2021-01-01",- # "metrics" => "visitors,bounce_rate",- # "property" => "visit:browser",- # "filters" => "event:page == /plausible.io|/important-page"- # })-- # assert json_response(conn, 200) == %{- # "results" => [- # %{- # "browser" => "Chrome",- # "bounce_rate" => 50,- # "visitors" => 3- # }- # ]- # }- # end-test "event:goal pageview filter for breakdown by visit source", %{conn: conn, site: site} doinsert(:goal, %{site: site, page_path: "/plausible.io"})@@ -1937,44 +2847,42 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest do]end- # test "IN filter for visit:entry_page", %{conn: conn, site: site} do- # populate_stats(site, [- # build(:pageview,- # pathname: "/ignore",- # timestamp: ~N[2021-01-01 00:00:00]- # ),- # build(:pageview,- # pathname: "/plausible.io",- # timestamp: ~N[2021-01-01 00:00:00]- # ),- # build(:pageview,- # pathname: "/plausible.io",- # timestamp: ~N[2021-01-01 00:00:00]- # ),- # build(:pageview,- # pathname: "/important-page",- # timestamp: ~N[2021-01-01 00:00:00]- # )- # ])-- # conn =- # post(conn, "/api/v2/query", %{- # "site_id" => site.domain,- # "metrics" => ["bounce_rate"],- # "date_range" => "all",- # "dimensions" => ["event:page"],- # "filters" => [- # ["is", "event:page", ["/plausible.io", "/important-page"]]- # ]- # })-- # %{"results" => results} = json_response(conn, 200)-- # assert results == [- # %{"dimensions" => ["plausible.io"], "metrics" => [100]},- # %{"dimensions" => ["/important-page"], "metrics" => [100]},- # ]- # end+ test "IN filter for visit:entry_page", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:pageview,+ pathname: "/ignore",+ timestamp: ~N[2021-01-01 00:00:00]+ ),+ build(:pageview,+ pathname: "/plausible.io",+ timestamp: ~N[2021-01-01 00:00:00]+ ),+ build(:pageview,+ pathname: "/plausible.io",+ timestamp: ~N[2021-01-01 00:00:00]+ ),+ build(:pageview,+ pathname: "/important-page",+ timestamp: ~N[2021-01-01 00:00:00]+ )+ ])++ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "metrics" => ["bounce_rate"],+ "date_range" => "all",+ "dimensions" => ["event:page"],+ "filters" => [+ ["is", "event:page", ["/plausible.io", "/important-page"]]+ ]+ })++ assert json_response(conn, 200)["results"] == [+ %{"dimensions" => ["/plausible.io"], "metrics" => [100]},+ %{"dimensions" => ["/important-page"], "metrics" => [100]}+ ]+ endtest "IN filter for event:name", %{conn: conn, site: site} dopopulate_stats(site, [@@ -2179,516 +3087,384 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest do]end- # describe "metrics" do- # test "returns conversion_rate in an event:goal breakdown", %{conn: conn, site: site} do- # populate_stats(site, [- # build(:event, name: "Signup", user_id: 1),- # build(:event, name: "Signup", user_id: 1),- # build(:pageview, pathname: "/blog"),- # build(:pageview, pathname: "/blog/post"),- # build(:pageview)- # ])-- # insert(:goal, %{site: site, event_name: "Signup"})- # insert(:goal, %{site: site, page_path: "/blog**"})-- # conn =- # get(conn, "/api/v1/stats/breakdown", %{- # "site_id" => site.domain,- # "period" => "day",- # "property" => "event:goal",- # "metrics" => "visitors,events,conversion_rate"- # })-- # assert json_response(conn, 200) == %{- # "results" => [- # %{- # "goal" => "Visit /blog**",- # "visitors" => 2,- # "events" => 2,- # "conversion_rate" => 50- # },- # %{- # "goal" => "Signup",- # "visitors" => 1,- # "events" => 2,- # "conversion_rate" => 25- # }- # ]- # }- # end-- # test "returns conversion_rate alone in an event:goal breakdown", %{conn: conn, site: site} do- # populate_stats(site, [- # build(:event, name: "Signup", user_id: 1),- # build(:pageview)- # ])-- # insert(:goal, %{site: site, event_name: "Signup"})-- # conn =- # get(conn, "/api/v1/stats/breakdown", %{- # "site_id" => site.domain,- # "period" => "day",- # "property" => "event:goal",- # "metrics" => "conversion_rate"- # })-- # assert json_response(conn, 200) == %{- # "results" => [- # %{- # "goal" => "Signup",- # "conversion_rate" => 50- # }- # ]- # }- # end-- # test "returns conversion_rate in a goal filtered custom prop breakdown", %{- # conn: conn,- # site: site- # } do- # populate_stats(site, [- # build(:pageview, pathname: "/blog/1", "meta.key": ["author"], "meta.value": ["Uku"]),- # build(:pageview, pathname: "/blog/2", "meta.key": ["author"], "meta.value": ["Uku"]),- # build(:pageview, pathname: "/blog/3", "meta.key": ["author"], "meta.value": ["Uku"]),- # build(:pageview, pathname: "/blog/1", "meta.key": ["author"], "meta.value": ["Marko"]),- # build(:pageview,- # pathname: "/blog/2",- # "meta.key": ["author"],- # "meta.value": ["Marko"],- # user_id: 1- # ),- # build(:pageview,- # pathname: "/blog/3",- # "meta.key": ["author"],- # "meta.value": ["Marko"],- # user_id: 1- # ),- # build(:pageview, pathname: "/blog"),- # build(:pageview, "meta.key": ["author"], "meta.value": ["Marko"]),- # build(:pageview)- # ])-- # insert(:goal, %{site: site, page_path: "/blog**"})-- # conn =- # get(conn, "/api/v1/stats/breakdown", %{- # "site_id" => site.domain,- # "period" => "day",- # "property" => "event:props:author",- # "filters" => "event:goal==Visit /blog**",- # "metrics" => "visitors,events,conversion_rate"- # })-- # assert json_response(conn, 200) == %{- # "results" => [- # %{- # "author" => "Uku",- # "visitors" => 3,- # "events" => 3,- # "conversion_rate" => 37.5- # },- # %{- # "author" => "Marko",- # "visitors" => 2,- # "events" => 3,- # "conversion_rate" => 25- # },- # %{- # "author" => "(none)",- # "visitors" => 1,- # "events" => 1,- # "conversion_rate" => 12.5- # }- # ]- # }- # end-- test "returns conversion_rate alone in a goal filtered custom prop breakdown", %{- conn: conn,- site: site- } do- populate_stats(site, [- build(:pageview, pathname: "/blog/1", "meta.key": ["author"], "meta.value": ["Uku"]),- build(:pageview)- ])+ describe "metrics" do+ test "returns conversion_rate in an event:goal breakdown", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:event, name: "Signup", user_id: 1),+ build(:event, name: "Signup", user_id: 1),+ build(:pageview, pathname: "/blog"),+ build(:pageview, pathname: "/blog/post"),+ build(:pageview)+ ])- insert(:goal, %{site: site, page_path: "/blog**"})+ insert(:goal, %{site: site, event_name: "Signup"})+ insert(:goal, %{site: site, page_path: "/blog**"})- conn =- post(conn, "/api/v2/query", %{- "site_id" => site.domain,- "metrics" => ["conversion_rate"],- "date_range" => "all",- "dimensions" => ["event:props:author"],- "filters" => [["matches", "event:goal", ["Visit /blog**"]]]- })+ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "date_range" => "all",+ "metrics" => ["visitors", "events", "conversion_rate"],+ "dimensions" => ["event:goal"]+ })- %{"results" => results} = json_response(conn, 200)+ assert json_response(conn, 200)["results"] == [+ %{"dimensions" => ["Visit /blog**"], "metrics" => [2, 2, 50.0]},+ %{"dimensions" => ["Signup"], "metrics" => [1, 2, 25.0]}+ ]+ end- assert results == [- %{"dimensions" => ["Uku"], "metrics" => [50]}- ]- end+ test "returns conversion_rate alone in an event:goal breakdown", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:event, name: "Signup", user_id: 1),+ build(:pageview)+ ])- # test "returns conversion_rate in a goal filtered event:page breakdown", %{- # conn: conn,- # site: site- # } do- # populate_stats(site, [- # build(:event, pathname: "/en/register", name: "pageview"),- # build(:event, pathname: "/en/register", name: "Signup"),- # build(:event, pathname: "/en/register", name: "Signup"),- # build(:event, pathname: "/it/register", name: "Signup", user_id: 1),- # build(:event, pathname: "/it/register", name: "Signup", user_id: 1),- # build(:event, pathname: "/it/register", name: "pageview")- # ])-- # insert(:goal, %{site: site, event_name: "Signup"})-- # conn =- # get(conn, "/api/v1/stats/breakdown", %{- # "site_id" => site.domain,- # "period" => "day",- # "property" => "event:page",- # "filters" => "event:goal==Signup",- # "metrics" => "visitors,events,conversion_rate"- # })-- # assert json_response(conn, 200) == %{- # "results" => [- # %{- # "page" => "/en/register",- # "visitors" => 2,- # "events" => 2,- # "conversion_rate" => 66.7- # },- # %{- # "page" => "/it/register",- # "visitors" => 1,- # "events" => 2,- # "conversion_rate" => 50- # }- # ]- # }- # end-- # test "returns conversion_rate alone in a goal filtered event:page breakdown", %{- # conn: conn,- # site: site- # } do- # populate_stats(site, [- # build(:event, pathname: "/en/register", name: "pageview"),- # build(:event, pathname: "/en/register", name: "Signup")- # ])-- # insert(:goal, %{site: site, event_name: "Signup"})-- # conn =- # get(conn, "/api/v1/stats/breakdown", %{- # "site_id" => site.domain,- # "period" => "day",- # "property" => "event:page",- # "filters" => "event:goal==Signup",- # "metrics" => "conversion_rate"- # })-- # assert json_response(conn, 200) == %{- # "results" => [- # %{- # "page" => "/en/register",- # "conversion_rate" => 50- # }- # ]- # }- # end-- # test "returns conversion_rate in a multi-goal filtered visit:screen_size breakdown", %{- # conn: conn,- # site: site- # } do- # populate_stats(site, [- # build(:event, screen_size: "Mobile", name: "pageview"),- # build(:event, screen_size: "Mobile", name: "AddToCart"),- # build(:event, screen_size: "Mobile", name: "AddToCart"),- # build(:event, screen_size: "Desktop", name: "AddToCart", user_id: 1),- # build(:event, screen_size: "Desktop", name: "Purchase", user_id: 1),- # build(:event, screen_size: "Desktop", name: "pageview")- # ])-- # # Make sure that revenue goals are treated the same- # # way as regular custom event goals- # insert(:goal, %{site: site, event_name: "Purchase", currency: :EUR})- # insert(:goal, %{site: site, event_name: "AddToCart"})-- # conn =- # post(conn, "/api/v2/query", %{- # "site_id" => site.domain,- # "metrics" => ["visitors", "events", "conversion_rate"],- # "date_range" => "all",- # "dimensions" => ["visit:device"],- # "filters" => [["is", "event:goal", ["AddToCart", "Purchase"]]]- # })-- # %{"results" => results} = json_response(conn, 200)-- # assert results == [- # %{"dimensions" => ["Mobile"], "metrics" => [2, 2, 66.7]},- # %{"dimensions" => ["Desktop"], "metrics" => [1, 2, 50]},- # ]- # end-- test "returns conversion_rate alone in a goal filtered visit:screen_size breakdown", %{- conn: conn,- site: site- } do- populate_stats(site, [- build(:event, screen_size: "Mobile", name: "pageview"),- build(:event, screen_size: "Mobile", name: "AddToCart")- ])+ insert(:goal, %{site: site, event_name: "Signup"})++ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "date_range" => "all",+ "metrics" => ["conversion_rate"],+ "dimensions" => ["event:goal"]+ })++ assert json_response(conn, 200)["results"] == [+ %{"dimensions" => ["Signup"], "metrics" => [50.0]}+ ]+ end++ test "returns conversion_rate in a goal filtered custom prop breakdown", %{+ conn: conn,+ site: site+ } do+ populate_stats(site, [+ build(:pageview, pathname: "/blog/1", "meta.key": ["author"], "meta.value": ["Uku"]),+ build(:pageview, pathname: "/blog/2", "meta.key": ["author"], "meta.value": ["Uku"]),+ build(:pageview, pathname: "/blog/3", "meta.key": ["author"], "meta.value": ["Uku"]),+ build(:pageview, pathname: "/blog/1", "meta.key": ["author"], "meta.value": ["Marko"]),+ build(:pageview,+ pathname: "/blog/2",+ "meta.key": ["author"],+ "meta.value": ["Marko"],+ user_id: 1+ ),+ build(:pageview,+ pathname: "/blog/3",+ "meta.key": ["author"],+ "meta.value": ["Marko"],+ user_id: 1+ ),+ build(:pageview, pathname: "/blog"),+ build(:pageview, "meta.key": ["author"], "meta.value": ["Marko"]),+ build(:pageview)+ ])++ insert(:goal, %{site: site, page_path: "/blog**"})++ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "date_range" => "all",+ "filters" => [["matches", "event:goal", ["Visit /blog**"]]],+ "metrics" => ["visitors", "events", "conversion_rate"],+ "dimensions" => ["event:props:author"]+ })++ assert json_response(conn, 200)["results"] == [+ %{"dimensions" => ["Uku"], "metrics" => [3, 3, 37.5]},+ %{"dimensions" => ["Marko"], "metrics" => [2, 3, 25.0]},+ %{"dimensions" => ["(none)"], "metrics" => [1, 1, 12.5]}+ ]+ end++ test "returns conversion_rate alone in a goal filtered custom prop breakdown", %{+ conn: conn,+ site: site+ } do+ populate_stats(site, [+ build(:pageview, pathname: "/blog/1", "meta.key": ["author"], "meta.value": ["Uku"]),+ build(:pageview)+ ])++ insert(:goal, %{site: site, page_path: "/blog**"})++ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "metrics" => ["conversion_rate"],+ "date_range" => "all",+ "dimensions" => ["event:props:author"],+ "filters" => [["matches", "event:goal", ["Visit /blog**"]]]+ })++ %{"results" => results} = json_response(conn, 200)++ assert results == [+ %{"dimensions" => ["Uku"], "metrics" => [50]}+ ]+ end++ test "returns conversion_rate in a goal filtered event:page breakdown", %{+ conn: conn,+ site: site+ } do+ populate_stats(site, [+ build(:event, pathname: "/en/register", name: "pageview"),+ build(:event, pathname: "/en/register", name: "Signup"),+ build(:event, pathname: "/en/register", name: "Signup"),+ build(:event, pathname: "/it/register", name: "Signup", user_id: 1),+ build(:event, pathname: "/it/register", name: "Signup", user_id: 1),+ build(:event, pathname: "/it/register", name: "pageview")+ ])++ insert(:goal, %{site: site, event_name: "Signup"})++ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "date_range" => "all",+ "dimensions" => ["event:page"],+ "filters" => [["is", "event:goal", ["Signup"]]],+ "metrics" => ["visitors", "events", "group_conversion_rate"]+ })++ assert json_response(conn, 200)["results"] == [+ %{"dimensions" => ["/en/register"], "metrics" => [2, 2, 66.7]},+ %{"dimensions" => ["/it/register"], "metrics" => [1, 2, 50.0]}+ ]+ end++ test "returns conversion_rate alone in a goal filtered event:page breakdown", %{+ conn: conn,+ site: site+ } do+ populate_stats(site, [+ build(:event, pathname: "/en/register", name: "pageview"),+ build(:event, pathname: "/en/register", name: "Signup")+ ])++ insert(:goal, %{site: site, event_name: "Signup"})++ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "date_range" => "all",+ "filters" => [["is", "event:goal", ["Signup"]]],+ "metrics" => ["group_conversion_rate"],+ "dimensions" => ["event:page"]+ })++ assert json_response(conn, 200)["results"] == [+ %{"dimensions" => ["/en/register"], "metrics" => [50.0]}+ ]+ end++ test "returns conversion_rate in a multi-goal filtered visit:screen_size breakdown", %{+ conn: conn,+ site: site+ } do+ populate_stats(site, [+ build(:event, screen_size: "Mobile", name: "pageview"),+ build(:event, screen_size: "Mobile", name: "AddToCart"),+ build(:event, screen_size: "Mobile", name: "AddToCart"),+ build(:event, screen_size: "Desktop", name: "AddToCart", user_id: 1),+ build(:event, screen_size: "Desktop", name: "Purchase", user_id: 1),+ build(:event, screen_size: "Desktop", name: "pageview")+ ])++ # Make sure that revenue goals are treated the same+ # way as regular custom event goals+ insert(:goal, %{site: site, event_name: "Purchase", currency: :EUR})+ insert(:goal, %{site: site, event_name: "AddToCart"})++ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "metrics" => ["visitors", "events", "group_conversion_rate"],+ "date_range" => "all",+ "dimensions" => ["visit:device"],+ "filters" => [["is", "event:goal", ["AddToCart", "Purchase"]]]+ })++ %{"results" => results} = json_response(conn, 200)++ assert results == [+ %{"dimensions" => ["Mobile"], "metrics" => [2, 2, 66.7]},+ %{"dimensions" => ["Desktop"], "metrics" => [1, 2, 50]}+ ]+ end++ test "returns conversion_rate alone in a goal filtered visit:screen_size breakdown", %{+ conn: conn,+ site: site+ } do+ populate_stats(site, [+ build(:event, screen_size: "Mobile", name: "pageview"),+ build(:event, screen_size: "Mobile", name: "AddToCart")+ ])- insert(:goal, %{site: site, event_name: "AddToCart"})+ insert(:goal, %{site: site, event_name: "AddToCart"})- conn =- post(conn, "/api/v2/query", %{- "site_id" => site.domain,- "metrics" => ["conversion_rate"],- "date_range" => "all",- "dimensions" => ["visit:device"],- "filters" => [["is", "event:goal", ["AddToCart"]]]- })+ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "metrics" => ["conversion_rate"],+ "date_range" => "all",+ "dimensions" => ["visit:device"],+ "filters" => [["is", "event:goal", ["AddToCart"]]]+ })- %{"results" => results} = json_response(conn, 200)+ %{"results" => results} = json_response(conn, 200)- assert results == [- %{"dimensions" => ["Mobile"], "metrics" => [50]}- ]- end+ assert results == [+ %{"dimensions" => ["Mobile"], "metrics" => [50]}+ ]+ end- # test "returns conversion_rate for a browser_version breakdown with pagination limit", %{- # site: site,- # conn: conn- # } do- # populate_stats(site, [- # build(:pageview, browser: "Firefox", browser_version: "110"),- # build(:pageview, browser: "Firefox", browser_version: "110"),- # build(:pageview, browser: "Chrome", browser_version: "110"),- # build(:pageview, browser: "Chrome", browser_version: "110"),- # build(:pageview, browser: "Avast Secure Browser", browser_version: "110"),- # build(:pageview, browser: "Avast Secure Browser", browser_version: "110"),- # build(:event, name: "Signup", browser: "Edge", browser_version: "110")- # ])-- # insert(:goal, site: site, event_name: "Signup")-- # conn =- # get(conn, "/api/v1/stats/breakdown", %{- # "site_id" => site.domain,- # "period" => "day",- # "property" => "visit:browser_version",- # "filters" => "event:goal==Signup",- # "metrics" => "visitors,conversion_rate",- # "page" => 1,- # "limit" => 1- # })-- # assert json_response(conn, 200) == %{- # "results" => [- # %{- # "browser" => "Edge",- # "browser_version" => "110",- # "visitors" => 1,- # "conversion_rate" => 100.0- # }- # ]- # }- # end-- test "all metrics for breakdown by visit prop", %{conn: conn, site: site} do- populate_stats(site, [- build(:pageview,- user_id: 1,- referrer_source: "Google",- timestamp: ~N[2021-01-01 00:00:00]- ),- build(:event,- name: "signup",- user_id: 1,- referrer_source: "Google",- timestamp: ~N[2021-01-01 00:05:00]- ),- build(:pageview,- user_id: 1,- referrer_source: "Google",- timestamp: ~N[2021-01-01 00:10:00]- ),- build(:pageview,- referrer_source: "Google",- timestamp: ~N[2021-01-01 00:25:00]- ),- build(:pageview,- referrer_source: "Twitter",- timestamp: ~N[2021-01-01 00:00:00]- )- ])+ test "all metrics for breakdown by visit prop", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:pageview,+ user_id: 1,+ referrer_source: "Google",+ timestamp: ~N[2021-01-01 00:00:00]+ ),+ build(:event,+ name: "signup",+ user_id: 1,+ referrer_source: "Google",+ timestamp: ~N[2021-01-01 00:05:00]+ ),+ build(:pageview,+ user_id: 1,+ referrer_source: "Google",+ timestamp: ~N[2021-01-01 00:10:00]+ ),+ build(:pageview,+ referrer_source: "Google",+ timestamp: ~N[2021-01-01 00:25:00]+ ),+ build(:pageview,+ referrer_source: "Twitter",+ timestamp: ~N[2021-01-01 00:00:00]+ )+ ])- conn =- post(conn, "/api/v2/query", %{- "site_id" => site.domain,- "metrics" => [- "visitors",- "visits",- "pageviews",- "events",- "bounce_rate",- "visit_duration"- ],- "date_range" => "all",- "dimensions" => ["visit:source"],- "filters" => []- })+ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "metrics" => [+ "visitors",+ "visits",+ "pageviews",+ "events",+ "bounce_rate",+ "visit_duration"+ ],+ "date_range" => "all",+ "dimensions" => ["visit:source"],+ "filters" => []+ })- %{"results" => results} = json_response(conn, 200)+ %{"results" => results} = json_response(conn, 200)- assert results == [- %{"dimensions" => ["Google"], "metrics" => [2, 2, 3, 4, 50, 300]},- %{"dimensions" => ["Twitter"], "metrics" => [1, 1, 1, 1, 100, 0]}- ]- end+ assert results == [+ %{"dimensions" => ["Google"], "metrics" => [2, 2, 3, 4, 50, 300]},+ %{"dimensions" => ["Twitter"], "metrics" => [1, 1, 1, 1, 100, 0]}+ ]+ end- test "metrics=bounce_rate does not add visits to the response", %{conn: conn, site: site} do- populate_stats(site, [- build(:pageview,- user_id: 1,- pathname: "/entry-page-1",- timestamp: ~N[2021-01-01 00:00:00]- ),- build(:pageview,- user_id: 1,- pathname: "/some-page",- timestamp: ~N[2021-01-01 00:10:00]- ),- build(:pageview,- user_id: 2,- pathname: "/entry-page-2",- referrer_source: "Google",- timestamp: ~N[2021-01-01 00:05:00]- )- ])+ test "metrics=bounce_rate does not add visits to the response", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:pageview,+ user_id: 1,+ pathname: "/entry-page-1",+ timestamp: ~N[2021-01-01 00:00:00]+ ),+ build(:pageview,+ user_id: 1,+ pathname: "/some-page",+ timestamp: ~N[2021-01-01 00:10:00]+ ),+ build(:pageview,+ user_id: 2,+ pathname: "/entry-page-2",+ referrer_source: "Google",+ timestamp: ~N[2021-01-01 00:05:00]+ )+ ])- conn =- post(conn, "/api/v2/query", %{- "site_id" => site.domain,- "metrics" => ["bounce_rate"],- "date_range" => "all",- "dimensions" => ["visit:entry_page"],- "order_by" => [["visit:entry_page", "asc"]]- })+ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "metrics" => ["bounce_rate"],+ "date_range" => "all",+ "dimensions" => ["visit:entry_page"],+ "order_by" => [["visit:entry_page", "asc"]]+ })- %{"results" => results} = json_response(conn, 200)+ %{"results" => results} = json_response(conn, 200)- assert results == [- %{"dimensions" => ["/entry-page-1"], "metrics" => [0]},- %{"dimensions" => ["/entry-page-2"], "metrics" => [100]}- ]- end+ assert results == [+ %{"dimensions" => ["/entry-page-1"], "metrics" => [0]},+ %{"dimensions" => ["/entry-page-2"], "metrics" => [100]}+ ]+ end- test "filter by custom event property", %{conn: conn, site: site} do- populate_stats(site, [- build(:event,- name: "Purchase",- "meta.key": ["package"],- "meta.value": ["business"],- browser: "Chrome",- timestamp: ~N[2021-01-01 00:00:00]- ),- build(:event,- name: "Purchase",- "meta.key": ["package"],- "meta.value": ["business"],- browser: "Safari",- timestamp: ~N[2021-01-01 00:00:00]- ),- build(:event,- name: "Purchase",- "meta.key": ["package"],- "meta.value": ["business"],- browser: "Safari",- timestamp: ~N[2021-01-01 00:25:00]- ),- build(:event,- name: "Purchase",- "meta.key": ["package"],- "meta.value": ["personal"],- browser: "IE",- timestamp: ~N[2021-01-01 00:25:00]- )- ])+ test "filter by custom event property", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:event,+ name: "Purchase",+ "meta.key": ["package"],+ "meta.value": ["business"],+ browser: "Chrome",+ timestamp: ~N[2021-01-01 00:00:00]+ ),+ build(:event,+ name: "Purchase",+ "meta.key": ["package"],+ "meta.value": ["business"],+ browser: "Safari",+ timestamp: ~N[2021-01-01 00:00:00]+ ),+ build(:event,+ name: "Purchase",+ "meta.key": ["package"],+ "meta.value": ["business"],+ browser: "Safari",+ timestamp: ~N[2021-01-01 00:25:00]+ ),+ build(:event,+ name: "Purchase",+ "meta.key": ["package"],+ "meta.value": ["personal"],+ browser: "IE",+ timestamp: ~N[2021-01-01 00:25:00]+ )+ ])- conn =- post(conn, "/api/v2/query", %{- "site_id" => site.domain,- "metrics" => ["visitors"],- "date_range" => "all",- "dimensions" => ["visit:browser"],- "filters" => [- ["is", "event:name", ["Purchase"]],- ["is", "event:props:package", ["business"]]- ]- })+ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "metrics" => ["visitors"],+ "date_range" => "all",+ "dimensions" => ["visit:browser"],+ "filters" => [+ ["is", "event:name", ["Purchase"]],+ ["is", "event:props:package", ["business"]]+ ]+ })- %{"results" => results} = json_response(conn, 200)+ %{"results" => results} = json_response(conn, 200)- assert results == [- %{"dimensions" => ["Safari"], "metrics" => [2]},- %{"dimensions" => ["Chrome"], "metrics" => [1]}- ]+ assert results == [+ %{"dimensions" => ["Safari"], "metrics" => [2]},+ %{"dimensions" => ["Chrome"], "metrics" => [1]}+ ]+ endend- # test "all metrics for breakdown by event prop", %{conn: conn, site: site} do- # populate_stats(site, [- # build(:pageview,- # user_id: 1,- # pathname: "/",- # timestamp: ~N[2021-01-01 00:00:00]- # ),- # build(:pageview,- # user_id: 1,- # pathname: "/plausible.io",- # timestamp: ~N[2021-01-01 00:10:00]- # ),- # build(:pageview, pathname: "/", timestamp: ~N[2021-01-01 00:25:00]),- # build(:pageview,- # pathname: "/plausible.io",- # timestamp: ~N[2021-01-01 00:00:00]- # )- # ])-- # conn =- # post(conn, "/api/v2/query", %{- # "site_id" => site.domain,- # "metrics" => [- # "visitors",- # "visits",- # "pageviews",- # "events",- # "bounce_rate",- # "visit_duration"- # ],- # "date_range" => "all",- # "dimensions" => ["event:page"],- # })-- # %{"results" => results} = json_response(conn, 200)-- # assert results == [- # %{"dimensions" => ["/"], "metrics" => [2, 2, 3, 4, 50, 300]},- # %{"dimensions" => ["/plausible.io"], "metrics" => [1, 1, 1, 1, 100, 0]}- # ]- # end- # end-describe "imported data" dotest "returns screen sizes breakdown when filtering by screen size", %{conn: conn, site: site} dosite_import = insert(:site_import, site: site)@@ -2723,67 +3499,54 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest doassert results == [%{"dimensions" => ["Mobile"], "metrics" => [4, 6]}]end- # test "returns custom event goals and pageview goals", %{conn: conn, site: site} do- # insert(:goal, site: site, event_name: "Purchase")- # insert(:goal, site: site, page_path: "/test")-- # site_import = insert(:site_import, site: site)-- # populate_stats(site, site_import.id, [- # build(:pageview,- # timestamp: ~N[2021-01-01 00:00:01],- # pathname: "/test"- # ),- # build(:event,- # name: "Purchase",- # timestamp: ~N[2021-01-01 00:00:03]- # ),- # build(:event,- # name: "Purchase",- # timestamp: ~N[2021-01-01 00:00:03]- # ),- # build(:imported_custom_events,- # name: "Purchase",- # visitors: 3,- # events: 5,- # date: ~D[2021-01-01]- # ),- # build(:imported_pages,- # page: "/test",- # visitors: 2,- # pageviews: 2,- # date: ~D[2021-01-01]- # ),- # build(:imported_visitors, visitors: 5, date: ~D[2021-01-01])- # ])-- # conn =- # get(conn, "/api/v1/stats/breakdown", %{- # "site_id" => site.domain,- # "period" => "day",- # "date" => "2021-01-01",- # "property" => "event:goal",- # "metrics" => "visitors,events,pageviews,conversion_rate",- # "with_imported" => "true"- # })-- # assert [- # %{- # "goal" => "Purchase",- # "visitors" => 5,- # "events" => 7,- # "pageviews" => 0,- # "conversion_rate" => 62.5- # },- # %{- # "goal" => "Visit /test",- # "visitors" => 3,- # "events" => 3,- # "pageviews" => 3,- # "conversion_rate" => 37.5- # }- # ] = json_response(conn, 200)["results"]- # end+ test "returns custom event goals and pageview goals", %{conn: conn, site: site} do+ insert(:goal, site: site, event_name: "Purchase")+ insert(:goal, site: site, page_path: "/test")++ site_import = insert(:site_import, site: site)++ populate_stats(site, site_import.id, [+ build(:pageview,+ timestamp: ~N[2021-01-01 00:00:01],+ pathname: "/test"+ ),+ build(:event,+ name: "Purchase",+ timestamp: ~N[2021-01-01 00:00:03]+ ),+ build(:event,+ name: "Purchase",+ timestamp: ~N[2021-01-01 00:00:03]+ ),+ build(:imported_custom_events,+ name: "Purchase",+ visitors: 3,+ events: 5,+ date: ~D[2021-01-01]+ ),+ build(:imported_pages,+ page: "/test",+ visitors: 2,+ pageviews: 2,+ date: ~D[2021-01-01]+ ),+ build(:imported_visitors, visitors: 5, date: ~D[2021-01-01])+ ])++ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "date_range" => "all",+ "dimensions" => ["event:goal"],+ "metrics" => ["visitors", "events", "pageviews", "conversion_rate"],+ "include" => %{"imports" => true}+ })++ assert json_response(conn, 200)["results"] == [+ %{"dimensions" => ["Purchase"], "metrics" => [5, 7, 0, 62.5]},+ %{"dimensions" => ["Visit /test"], "metrics" => [3, 3, 3, 37.5]}+ ]+ endtest "pageviews are returned as events for breakdown reports other than custom events", %{conn: conn,@@ -2826,6 +3589,8 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest doassert %{"dimensions" => ["Desktop"], "metrics" => [1]} =breakdown_and_first.("visit:device")+ # :TODO: These should not pass validation - not available on events.+ # visit dimension and event-only metric# assert %{"dimensions" => ["/test"], "metrics" => [1]} = breakdown_and_first.("visit:entry_page")# assert %{"dimensions" => ["/test"], "metrics" => [1]} = breakdown_and_first.("visit:exit_page")assert %{"dimensions" => ["EE"], "metrics" => [1]} = breakdown_and_first.("visit:country")@@ -2870,8 +3635,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest doconn =post(conn, "/api/v2/query", %{"site_id" => site.domain,- "metrics" => ["visitors", "events"],- # "metrics" => ["visitors", "events", "conversion_rate"],+ "metrics" => ["visitors", "events", "conversion_rate"],"date_range" => "all","dimensions" => ["event:props:url"],"filters" => [@@ -2881,8 +3645,8 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest do})assert json_response(conn, 200)["results"] == [- %{"dimensions" => ["https://two.com"], "metrics" => [5, 10]},- %{"dimensions" => ["https://one.com"], "metrics" => [3, 6]}+ %{"dimensions" => ["https://two.com"], "metrics" => [5, 10, 50]},+ %{"dimensions" => ["https://one.com"], "metrics" => [3, 6, 30]}]refute json_response(conn, 200)["meta"]["warning"]@@ -2923,8 +3687,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest doconn =post(conn, "/api/v2/query", %{"site_id" => site.domain,- "metrics" => ["visitors", "events"],- # "metrics" => ["visitors", "events", "conversion_rate"],+ "metrics" => ["visitors", "events", "conversion_rate"],"date_range" => "all","dimensions" => ["event:props:path"],"filters" => [@@ -2934,8 +3697,8 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest do})assert json_response(conn, 200)["results"] == [- %{"dimensions" => ["/two"], "metrics" => [5, 10]},- %{"dimensions" => ["/one"], "metrics" => [3, 6]}+ %{"dimensions" => ["/two"], "metrics" => [5, 10, 50]},+ %{"dimensions" => ["/one"], "metrics" => [3, 6, 30]}]refute json_response(conn, 200)["meta"]["warning"]@@ -3103,4 +3866,39 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest doassert meta["warning"] =~ "Imported stats are not included in the results"endend++ test "multiple breakdown timeseries with sources", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:pageview,+ referrer_source: "Google",+ timestamp: ~N[2021-01-01 00:00:00]+ ),+ build(:pageview, timestamp: ~N[2021-01-02 00:00:00]),+ build(:pageview,+ referrer_source: "Google",+ timestamp: ~N[2021-01-02 00:00:00]+ ),+ build(:pageview, timestamp: ~N[2021-01-03 00:00:00]),+ build(:pageview,+ referrer_source: "Twitter",+ timestamp: ~N[2021-01-03 00:00:00]+ )+ ])++ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "metrics" => ["visitors"],+ "date_range" => ["2021-01-01", "2021-01-03"],+ "dimensions" => ["time", "visit:source"]+ })++ assert json_response(conn, 200)["results"] == [+ %{"dimensions" => ["2021-01-01T00:00:00Z", "Google"], "metrics" => [1]},+ %{"dimensions" => ["2021-01-02T00:00:00Z", "Google"], "metrics" => [1]},+ %{"dimensions" => ["2021-01-02T00:00:00Z", "Direct / None"], "metrics" => [1]},+ %{"dimensions" => ["2021-01-03T00:00:00Z", "Direct / None"], "metrics" => [1]},+ %{"dimensions" => ["2021-01-03T00:00:00Z", "Twitter"], "metrics" => [1]}+ ]+ endend