Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.
TypeScript analyticsanalytics-dashboardclickhouseelixirgoogle-analyticsmarketingopen-source-analyticsphoenixplausible-analyticspostgresqlprivacyprivacy-friendlyself-hostedsimple-analyticsstatisticstailwindcssweb-analyticswebsite-analyticswebsite-statswebsite-tracking
APIv2 - initial PR (#4216)
Karl-Aksel Puulmann58a66a952c1c
diff --git a/extra/lib/plausible/stats/goal/revenue.ex b/extra/lib/plausible/stats/goal/revenue.exindex d531b5d638..7443a1758d 100644--- a/extra/lib/plausible/stats/goal/revenue.ex+++ b/extra/lib/plausible/stats/goal/revenue.ex@@ -15,14 +15,20 @@ defmodule Plausible.Stats.Goal.Revenue dodef total_revenue_query() dodynamic([e],- fragment("toDecimal64(sum(?) * any(_sample_factor), 3)", e.revenue_reporting_amount)+ selected_as(+ fragment("toDecimal64(sum(?) * any(_sample_factor), 3)", e.revenue_reporting_amount),+ :total_revenue+ ))enddef average_revenue_query() dodynamic([e],- fragment("toDecimal64(avg(?) * any(_sample_factor), 3)", e.revenue_reporting_amount)+ selected_as(+ fragment("toDecimal64(avg(?) * any(_sample_factor), 3)", e.revenue_reporting_amount),+ :average_revenue+ ))enddiff --git a/lib/plausible/stats.ex b/lib/plausible/stats.exindex 42ee7b47c2..ffa59389ea 100644--- a/lib/plausible/stats.ex+++ b/lib/plausible/stats.ex@@ -1,12 +1,16 @@defmodule Plausible.Stats douse Plausible+ alias Plausible.Stats.QueryResult+ use Plausible.ClickhouseRepoalias Plausible.Stats.{Breakdown,Aggregate,Timeseries,CurrentVisitors,- FilterSuggestions+ FilterSuggestions,+ QueryOptimizer,+ SQL}use Plausible.DebugReplayInfo@@ -31,6 +35,15 @@ defmodule Plausible.Stats doCurrentVisitors.current_visitors(site)end+ def query(site, query) do+ optimized_query = QueryOptimizer.optimize(query)++ optimized_query+ |> SQL.QueryBuilder.build(site)+ |> ClickhouseRepo.all()+ |> QueryResult.from(optimized_query)+ end+on_ee dodef funnel(site, query, funnel) doinclude_sentry_replay_info()diff --git a/lib/plausible/stats/aggregate.ex b/lib/plausible/stats/aggregate.exindex 01eef54e63..82d1c8a180 100644--- a/lib/plausible/stats/aggregate.ex+++ b/lib/plausible/stats/aggregate.ex@@ -1,7 +1,7 @@defmodule Plausible.Stats.Aggregate douse Plausible.ClickhouseRepouse Plausible- import Plausible.Stats.{Base, Imported}+ import Plausible.Stats.Baseimport Ecto.Queryalias Plausible.Stats.{Query, Util}@@ -15,23 +15,24 @@ defmodule Plausible.Stats.Aggregate doQuery.trace(query, metrics)- {event_metrics, session_metrics, other_metrics} =- metrics- |> Util.maybe_add_visitors_metric()- |> Plausible.Stats.TableDecider.partition_metrics(query)+ query_with_metrics = %Plausible.Stats.Query{+ query+ | metrics: Util.maybe_add_visitors_metric(metrics)+ }- event_task = fn -> aggregate_events(site, query, event_metrics) end-- session_task = fn -> aggregate_sessions(site, query, session_metrics) end+ q = Plausible.Stats.SQL.QueryBuilder.build(query_with_metrics, site)time_on_page_task =- if :time_on_page in other_metrics do+ if :time_on_page in query_with_metrics.metrics dofn -> aggregate_time_on_page(site, query) endelsefn -> %{} endend- Plausible.ClickhouseRepo.parallel_tasks([session_task, event_task, time_on_page_task])+ Plausible.ClickhouseRepo.parallel_tasks([+ run_query_task(q),+ time_on_page_task+ ])|> Enum.reduce(%{}, fn aggregate, task_result -> Map.merge(aggregate, task_result) end)|> Util.keep_requested_metrics(metrics)|> cast_revenue_metrics_to_money(currency)@@ -40,24 +41,8 @@ defmodule Plausible.Stats.Aggregate do|> Enum.into(%{})end- defp aggregate_events(_, _, []), do: %{}-- defp aggregate_events(site, query, metrics) do- from(e in base_event_query(site, query), select: ^select_event_metrics(metrics))- |> merge_imported(site, query, metrics)- |> maybe_add_conversion_rate(site, query, metrics)- |> ClickhouseRepo.one()- end-- defp aggregate_sessions(_, _, []), do: %{}-- defp aggregate_sessions(site, query, metrics) do- from(e in query_sessions(site, query), select: ^select_session_metrics(metrics, query))- |> filter_converted_sessions(site, query)- |> merge_imported(site, query, metrics)- |> ClickhouseRepo.one()- |> Util.keep_requested_metrics(metrics)- end+ defp run_query_task(nil), do: fn -> %{} end+ defp run_query_task(q), do: fn -> ClickhouseRepo.one(q) enddefp aggregate_time_on_page(site, query) dowindowed_pages_q =diff --git a/lib/plausible/stats/base.ex b/lib/plausible/stats/base.exindex a449d0fd30..bd01257fb7 100644--- a/lib/plausible/stats/base.ex+++ b/lib/plausible/stats/base.ex@@ -31,7 +31,7 @@ defmodule Plausible.Stats.Base doendend- defp query_events(site, query) do+ def query_events(site, query) doq = from(e in "events_v2", where: ^Filters.WhereBuilder.build(:events, site, query))on_ee do@@ -62,14 +62,21 @@ defmodule Plausible.Stats.Base dopageviews:dynamic([e],- fragment("toUInt64(round(countIf(? = 'pageview') * any(_sample_factor)))", e.name)+ selected_as(+ fragment("toUInt64(round(countIf(? = 'pageview') * any(_sample_factor)))", e.name),+ :pageviews+ ))}enddefp select_event_metric(:events) do%{- events: dynamic([], fragment("toUInt64(round(count(*) * any(_sample_factor)))"))+ events:+ dynamic(+ [],+ selected_as(fragment("toUInt64(round(count(*) * any(_sample_factor)))"), :events)+ )}end@@ -82,7 +89,13 @@ defmodule Plausible.Stats.Base dodefp select_event_metric(:visits) do%{visits:- dynamic([e], fragment("toUInt64(round(uniq(?) * any(_sample_factor)))", e.session_id))+ dynamic(+ [e],+ selected_as(+ fragment("toUInt64(round(uniq(?) * any(_sample_factor)))", e.session_id),+ :visits+ )+ )}end@@ -127,10 +140,13 @@ defmodule Plausible.Stats.Base dobounce_rate:dynamic([],- fragment(- "toUInt32(ifNotFinite(round(sumIf(is_bounce * sign, ?) / sumIf(sign, ?) * 100), 0))",- ^condition,- ^condition+ selected_as(+ fragment(+ "toUInt32(ifNotFinite(round(sumIf(is_bounce * sign, ?) / sumIf(sign, ?) * 100), 0))",+ ^condition,+ ^condition+ ),+ :bounce_rate)),__internal_visits: dynamic([], fragment("toUInt32(sum(sign))"))@@ -139,7 +155,14 @@ defmodule Plausible.Stats.Base dodefp select_session_metric(:visits, _query) do%{- visits: dynamic([s], fragment("toUInt64(round(sum(?) * any(_sample_factor)))", s.sign))+ visits:+ dynamic(+ [s],+ selected_as(+ fragment("toUInt64(round(sum(?) * any(_sample_factor)))", s.sign),+ :visits+ )+ )}end@@ -148,7 +171,10 @@ defmodule Plausible.Stats.Base dopageviews:dynamic([s],- fragment("toUInt64(round(sum(? * ?) * any(_sample_factor)))", s.sign, s.pageviews)+ selected_as(+ fragment("toUInt64(round(sum(? * ?) * any(_sample_factor)))", s.sign, s.pageviews),+ :pageviews+ ))}end@@ -158,7 +184,10 @@ defmodule Plausible.Stats.Base doevents:dynamic([s],- fragment("toUInt64(round(sum(? * ?) * any(_sample_factor)))", s.sign, s.events)+ selected_as(+ fragment("toUInt64(round(sum(? * ?) * any(_sample_factor)))", s.sign, s.events),+ :events+ ))}end@@ -179,7 +208,13 @@ defmodule Plausible.Stats.Base dodefp select_session_metric(:visit_duration, _query) do%{visit_duration:- dynamic([], fragment("toUInt32(ifNotFinite(round(sum(duration * sign) / sum(sign)), 0))")),+ dynamic(+ [],+ selected_as(+ fragment("toUInt32(ifNotFinite(round(sum(duration * sign) / sum(sign)), 0))"),+ :visit_duration+ )+ ),__internal_visits: dynamic([], fragment("toUInt32(sum(sign))"))}end@@ -189,7 +224,15 @@ defmodule Plausible.Stats.Base doviews_per_visit:dynamic([s],- fragment("ifNotFinite(round(sum(? * ?) / sum(?), 2), 0)", s.sign, s.pageviews, s.sign)+ selected_as(+ fragment(+ "ifNotFinite(round(sum(? * ?) / sum(?), 2), 0)",+ s.sign,+ s.pageviews,+ s.sign+ ),+ :views_per_visit+ )),__internal_visits: dynamic([], fragment("toUInt32(sum(sign))"))}@@ -328,11 +371,14 @@ defmodule Plausible.Stats.Base do)|> select_merge(%{percentage:- fragment(- "if(? > 0, round(? / ? * 100, 1), null)",- selected_as(:__total_visitors),- selected_as(:visitors),- selected_as(:__total_visitors)+ selected_as(+ fragment(+ "if(? > 0, round(? / ? * 100, 1), null)",+ selected_as(:__total_visitors),+ selected_as(:visitors),+ selected_as(:__total_visitors)+ ),+ :percentage)})else@@ -357,11 +403,14 @@ defmodule Plausible.Stats.Base do)|> select_merge([e], %{conversion_rate:- fragment(- "if(? > 0, round(? / ? * 100, 1), 0)",- selected_as(:__total_visitors),- e.visitors,- selected_as(:__total_visitors)+ selected_as(+ fragment(+ "if(? > 0, round(? / ? * 100, 1), 0)",+ selected_as(:__total_visitors),+ e.visitors,+ selected_as(:__total_visitors)+ ),+ :conversion_rate)})elsediff --git a/lib/plausible/stats/breakdown.ex b/lib/plausible/stats/breakdown.exindex 6f3799e831..d68e9fcad0 100644--- a/lib/plausible/stats/breakdown.ex+++ b/lib/plausible/stats/breakdown.ex@@ -276,8 +276,8 @@ defmodule Plausible.Stats.Breakdown dodefp breakdown_table(%Query{dimensions: ["visit:exit_page"]}, _metrics), do: :sessiondefp breakdown_table(%Query{dimensions: ["visit:exit_page_hostname"]}, _metrics), do: :session- defp breakdown_table(%Query{dimensions: [dimension]} = query, metrics) do- {_, session_metrics, _} = TableDecider.partition_metrics(metrics, query, dimension)+ defp breakdown_table(%Query{dimensions: [_dimension]} = query, metrics) do+ {_, session_metrics, _} = TableDecider.partition_metrics(metrics, query)if not Enum.empty?(session_metrics) do:sessiondiff --git a/lib/plausible/stats/filters/filters.ex b/lib/plausible/stats/filters/filters.exindex b6760965ed..dc13c7dcd9 100644--- a/lib/plausible/stats/filters/filters.ex+++ b/lib/plausible/stats/filters/filters.ex@@ -87,6 +87,6 @@ defmodule Plausible.Stats.Filters doproperty|> String.split(":")|> List.last()- |> String.to_existing_atom()+ |> String.to_atom()endenddiff --git a/lib/plausible/stats/filters/query_parser.ex b/lib/plausible/stats/filters/query_parser.exindex b2c7690e57..5023261899 100644--- a/lib/plausible/stats/filters/query_parser.ex+++ b/lib/plausible/stats/filters/query_parser.ex@@ -1,22 +1,30 @@defmodule Plausible.Stats.Filters.QueryParser do@moduledoc false+ alias Plausible.Stats.TableDecideralias Plausible.Stats.Filters+ alias Plausible.Stats.Query- def parse(params) when is_map(params) do+ def parse(site, params) when is_map(params) dowith {:ok, metrics} <- parse_metrics(Map.get(params, "metrics", [])),{:ok, filters} <- parse_filters(Map.get(params, "filters", [])),- {:ok, date_range} <- parse_date_range(Map.get(params, "date_range")),+ {:ok, date_range} <- parse_date_range(site, Map.get(params, "date_range")),{:ok, dimensions} <- parse_dimensions(Map.get(params, "dimensions", [])),{:ok, order_by} <- parse_order_by(Map.get(params, "order_by")),+ {:ok, include} <- parse_include(Map.get(params, "include", %{})),query = %{metrics: metrics,filters: filters,date_range: date_range,dimensions: dimensions,- order_by: order_by+ order_by: order_by,+ timezone: site.timezone,+ imported_data_requested: Map.get(include, :imports, false)},- :ok <- validate_order_by(query) do+ :ok <- validate_order_by(query),+ :ok <- validate_goal_filters(site, query),+ :ok <- validate_custom_props_access(site, query),+ :ok <- validate_metrics(query) do{:ok, query}endend@@ -87,16 +95,26 @@ defmodule Plausible.Stats.Filters.QueryParser dodefp parse_clauses_list(filter), do: {:error, "Invalid filter '#{inspect(filter)}'"}- defp parse_date_range("day"), do: {:ok, "day"}- defp parse_date_range("7d"), do: {:ok, "7d"}- defp parse_date_range("30d"), do: {:ok, "30d"}- defp parse_date_range("month"), do: {:ok, "month"}- defp parse_date_range("6mo"), do: {:ok, "6mo"}- defp parse_date_range("12mo"), do: {:ok, "6mo"}- defp parse_date_range("year"), do: {:ok, "year"}- defp parse_date_range("all"), do: {:ok, "all"}+ defp parse_date_range(site, "day") do+ today = DateTime.now!(site.timezone) |> DateTime.to_date()+ {:ok, Date.range(today, today)}+ end++ defp parse_date_range(_site, "7d"), do: {:ok, "7d"}+ defp parse_date_range(_site, "30d"), do: {:ok, "30d"}+ defp parse_date_range(_site, "month"), do: {:ok, "month"}+ defp parse_date_range(_site, "6mo"), do: {:ok, "6mo"}+ defp parse_date_range(_site, "12mo"), do: {:ok, "12mo"}+ defp parse_date_range(_site, "year"), do: {:ok, "year"}++ defp parse_date_range(site, "all") do+ today = DateTime.now!(site.timezone) |> DateTime.to_date()+ start_date = Plausible.Sites.stats_start_date(site) || today++ {:ok, Date.range(start_date, today)}+ end- defp parse_date_range([from_date_string, to_date_string])+ defp parse_date_range(_site, [from_date_string, to_date_string])when is_bitstring(from_date_string) and is_bitstring(to_date_string) dowith {:ok, from_date} <- Date.from_iso8601(from_date_string),{:ok, to_date} <- Date.from_iso8601(to_date_string) do@@ -106,13 +124,13 @@ defmodule Plausible.Stats.Filters.QueryParser doendend- defp parse_date_range(unknown), do: {:error, "Invalid date range '#{inspect(unknown)}'"}+ defp parse_date_range(_site, unknown), do: {:error, "Invalid date range '#{inspect(unknown)}'"}defp parse_dimensions(dimensions) when is_list(dimensions) doif length(dimensions) == length(Enum.uniq(dimensions)) doparse_list(dimensions,- &parse_filter_key_string(&1, "Invalid dimensions '#{inspect(dimensions)}'")+ &parse_dimension_entry(&1, "Invalid dimensions '#{inspect(dimensions)}'"))else{:error, "Some dimensions are listed multiple times"}@@ -121,36 +139,67 @@ defmodule Plausible.Stats.Filters.QueryParser dodefp parse_dimensions(dimensions), do: {:error, "Invalid dimensions '#{inspect(dimensions)}'"}- def parse_order_by(order_by) when is_list(order_by) do+ defp parse_order_by(order_by) when is_list(order_by) doparse_list(order_by, &parse_order_by_entry/1)end- def parse_order_by(nil), do: {:ok, nil}- def parse_order_by(order_by), do: {:error, "Invalid order_by '#{inspect(order_by)}'"}+ defp parse_order_by(nil), do: {:ok, nil}+ defp parse_order_by(order_by), do: {:error, "Invalid order_by '#{inspect(order_by)}'"}- def parse_order_by_entry(entry) do- with {:ok, metric_or_dimension} <- parse_metric_or_dimension(entry),+ defp parse_order_by_entry(entry) do+ with {:ok, value} <- parse_metric_or_dimension(entry),{:ok, order_direction} <- parse_order_direction(entry) do- {:ok, {metric_or_dimension, order_direction}}+ {:ok, {value, order_direction}}+ end+ end++ defp parse_dimension_entry(key, error_message) do+ case {+ parse_time(key),+ parse_filter_key_string(key)+ } do+ {{:ok, time}, _} -> {:ok, time}+ {_, {:ok, filter_key}} -> {:ok, filter_key}+ _ -> {:error, error_message}endend- def parse_metric_or_dimension([metric_or_dimension, _] = entry) do- case {parse_metric(metric_or_dimension), parse_filter_key_string(metric_or_dimension)} do- {{:ok, metric}, _} -> {:ok, metric}- {_, {:ok, dimension}} -> {:ok, dimension}+ defp parse_metric_or_dimension([value, _] = entry) do+ case {+ parse_time(value),+ parse_metric(value),+ parse_filter_key_string(value)+ } do+ {{:ok, time}, _, _} -> {:ok, time}+ {_, {:ok, metric}, _} -> {:ok, metric}+ {_, _, {:ok, dimension}} -> {:ok, dimension}_ -> {:error, "Invalid order_by entry '#{inspect(entry)}'"}endend- def parse_order_direction([_, "asc"]), do: {:ok, :asc}- def parse_order_direction([_, "desc"]), do: {:ok, :desc}- def parse_order_direction(entry), do: {:error, "Invalid order_by entry '#{inspect(entry)}'"}+ defp parse_time("time"), do: {:ok, "time"}+ defp parse_time("time:day"), do: {:ok, "time:day"}+ defp parse_time("time:week"), do: {:ok, "time:week"}+ defp parse_time("time:month"), do: {:ok, "time:month"}+ defp parse_time("time:year"), do: {:ok, "time:year"}+ defp parse_time(_), do: :error++ defp parse_order_direction([_, "asc"]), do: {:ok, :asc}+ defp parse_order_direction([_, "desc"]), do: {:ok, :desc}+ defp parse_order_direction(entry), do: {:error, "Invalid order_by entry '#{inspect(entry)}'"}++ defp parse_include(%{"imports" => value}) when is_boolean(value), do: {:ok, %{imports: value}}+ defp parse_include(%{}), do: {:ok, %{}}+ defp parse_include(include), do: {:error, "Invalid include passed '#{inspect(include)}'"}defp parse_filter_key_string(filter_key, error_message \\ "") docase filter_key do- "event:props:" <> _property_name ->- {:ok, filter_key}+ "event:props:" <> property_name ->+ if String.length(property_name) > 0 do+ {:ok, filter_key}+ else+ {:error, error_message}+ end"event:" <> key ->if key in Filters.event_props() do@@ -171,7 +220,7 @@ defmodule Plausible.Stats.Filters.QueryParser doendend- def validate_order_by(query) do+ defp validate_order_by(query) doif query.order_by dovalid_values = query.metrics ++ query.dimensions@@ -193,6 +242,92 @@ defmodule Plausible.Stats.Filters.QueryParser doendend+ defp validate_goal_filters(site, query) do+ goal_filter_clauses =+ Enum.flat_map(query.filters, fn+ [_operation, "event:goal", clauses] -> clauses+ _ -> []+ end)++ if length(goal_filter_clauses) > 0 do+ configured_goals =+ Plausible.Goals.for_site(site)+ |> Enum.map(fn+ %{page_path: path} when is_binary(path) -> {:page, path}+ %{event_name: event_name} -> {:event, event_name}+ end)++ validate_list(goal_filter_clauses, &validate_goal_filter(&1, configured_goals))+ else+ :ok+ end+ end++ defp validate_goal_filter(clause, configured_goals) do+ if Enum.member?(configured_goals, clause) do+ :ok+ else+ {:error,+ "The goal `#{Filters.Utils.unwrap_goal_value(clause)}` is not configured for this site. Find out how to configure goals here: https://plausible.io/docs/stats-api#filtering-by-goals"}+ end+ end++ defp validate_custom_props_access(site, query) do+ allowed_props = Plausible.Props.allowed_for(site, bypass_setup?: true)++ validate_custom_props_access(site, query, allowed_props)+ end++ defp validate_custom_props_access(_site, _query, :all), do: :ok++ defp validate_custom_props_access(_site, query, allowed_props) do+ valid? =+ query.filters+ |> Enum.map(fn [_operation, filter_key | _rest] -> filter_key end)+ |> Enum.concat(query.dimensions)+ |> Enum.all?(fn+ "event:props:" <> prop -> prop in allowed_props+ _ -> true+ end)++ if valid? do+ :ok+ else+ {:error, "The owner of this site does not have access to the custom properties feature"}+ end+ end++ defp validate_metrics(query) do+ validate_list(query.metrics, &validate_metric(&1, query))++ with :ok <- validate_list(query.metrics, &validate_metric(&1, query)) do+ validate_no_metrics_filters_conflict(query)+ end+ end++ defp validate_metric(:conversion_rate = metric, query) do+ if Enum.member?(query.dimensions, "event:goal") or+ not is_nil(Query.get_filter(query, "event:goal")) do+ :ok+ else+ {:error, "Metric `#{metric}` can only be queried with event:goal filters or dimensions"}+ end+ end++ defp validate_metric(_, _), do: :ok++ defp validate_no_metrics_filters_conflict(query) do+ {_event_metrics, sessions_metrics, _other_metrics} =+ TableDecider.partition_metrics(query.metrics, query)++ if Enum.empty?(sessions_metrics) or not TableDecider.event_filters?(query) do+ :ok+ else+ {:error,+ "Session metric(s) `#{sessions_metrics |> Enum.join(", ")}` cannot be queried along with event filters or dimensions"}+ end+ end+defp parse_list(list, parser_function) doEnum.reduce_while(list, {:ok, []}, fn value, {:ok, results} ->case parser_function.(value) do@@ -201,4 +336,13 @@ defmodule Plausible.Stats.Filters.QueryParser doendend)end++ defp validate_list(list, parser_function) do+ Enum.reduce_while(list, :ok, fn value, :ok ->+ case parser_function.(value) do+ :ok -> {:cont, :ok}+ {:error, _} = error -> {:halt, error}+ end+ end)+ endenddiff --git a/lib/plausible/stats/imported/imported.ex b/lib/plausible/stats/imported/imported.exindex 2e680f824f..5c1d77cc34 100644--- a/lib/plausible/stats/imported/imported.ex+++ b/lib/plausible/stats/imported/imported.ex@@ -274,18 +274,18 @@ defmodule Plausible.Stats.Imported dosite|> Imported.Base.query_imported(query)|> where([i], i.visitors > 0)- |> group_imported_by(dim)+ |> group_imported_by(dim, query)|> select_imported_metrics(metrics)join_on =case dim do- _ when dim in [:url, :path] ->+ _ when dim in [:url, :path] and not query.v2 ->dynamic([s, i], s.breakdown_prop_value == i.breakdown_prop_value)- :os_version ->+ :os_version when not query.v2 ->dynamic([s, i], s.os == i.os and s.os_version == i.os_version)- :browser_version ->+ :browser_version when not query.v2 ->dynamic([s, i], s.browser == i.browser and s.browser_version == i.browser_version)dim ->@@ -297,7 +297,7 @@ defmodule Plausible.Stats.Imported doon: ^join_on,select: %{})- |> select_joined_dimension(dim)+ |> select_joined_dimension(dim, query)|> select_joined_metrics(metrics)|> apply_order_by(metrics)end@@ -344,7 +344,7 @@ defmodule Plausible.Stats.Imported doon: s.name == i.name,select: %{})- |> select_joined_dimension(:name)+ |> select_joined_dimension(:name, query)|> select_joined_metrics(metrics)elseq@@ -551,7 +551,7 @@ defmodule Plausible.Stats.Imported do|> select_imported_metrics(rest)end- defp group_imported_by(q, dim) when dim in [:source, :referrer] do+ defp group_imported_by(q, dim, _query) when dim in [:source, :referrer] doq|> group_by([i], field(i, ^dim))|> select_merge([i], %{@@ -559,7 +559,7 @@ defmodule Plausible.Stats.Imported do})end- defp group_imported_by(q, dim)+ defp group_imported_by(q, dim, _query)when dim in [:utm_source, :utm_medium, :utm_campaign, :utm_term, :utm_content] doq|> group_by([i], field(i, ^dim))@@ -567,34 +567,34 @@ defmodule Plausible.Stats.Imported do|> select_merge([i], %{^dim => field(i, ^dim)})end- defp group_imported_by(q, :page) do+ defp group_imported_by(q, :page, _query) doq|> group_by([i], i.page)|> select_merge([i], %{page: i.page, time_on_page: sum(i.time_on_page)})end- defp group_imported_by(q, :country) do+ defp group_imported_by(q, :country, _query) doq|> group_by([i], i.country)|> where([i], i.country != "ZZ")|> select_merge([i], %{country: i.country})end- defp group_imported_by(q, :region) do+ defp group_imported_by(q, :region, _query) doq|> group_by([i], i.region)|> where([i], i.region != "")|> select_merge([i], %{region: i.region})end- defp group_imported_by(q, :city) do+ defp group_imported_by(q, :city, _query) doq|> group_by([i], i.city)|> where([i], i.city != 0 and not is_nil(i.city))|> select_merge([i], %{city: i.city})end- defp group_imported_by(q, dim) when dim in [:device, :browser] do+ defp group_imported_by(q, dim, _query) when dim in [:device, :browser] doq|> group_by([i], field(i, ^dim))|> select_merge([i], %{@@ -602,7 +602,7 @@ defmodule Plausible.Stats.Imported do})end- defp group_imported_by(q, :browser_version) do+ defp group_imported_by(q, :browser_version, _query) doq|> group_by([i], [i.browser, i.browser_version])|> select_merge([i], %{@@ -617,7 +617,7 @@ defmodule Plausible.Stats.Imported do})end- defp group_imported_by(q, :os) do+ defp group_imported_by(q, :os, _query) doq|> group_by([i], i.operating_system)|> select_merge([i], %{@@ -625,7 +625,7 @@ defmodule Plausible.Stats.Imported do})end- defp group_imported_by(q, :os_version) do+ defp group_imported_by(q, :os_version, _query) doq|> group_by([i], [i.operating_system, i.operating_system_version])|> select_merge([i], %{@@ -640,19 +640,27 @@ defmodule Plausible.Stats.Imported do})end- defp group_imported_by(q, dim) when dim in [:entry_page, :exit_page] do+ defp group_imported_by(q, dim, _query) when dim in [:entry_page, :exit_page] doq|> group_by([i], field(i, ^dim))|> select_merge([i], %{^dim => field(i, ^dim)})end- defp group_imported_by(q, :name) do+ defp group_imported_by(q, :name, _query) doq|> group_by([i], i.name)|> select_merge([i], %{name: i.name})end- defp group_imported_by(q, :url) do+ defp group_imported_by(q, :url, query) when query.v2 do+ q+ |> group_by([i], i.link_url)+ |> select_merge([i], %{+ url: fragment("if(not empty(?), ?, ?)", i.link_url, i.link_url, @none)+ })+ end++ defp group_imported_by(q, :url, _query) doq|> group_by([i], i.link_url)|> select_merge([i], %{@@ -660,7 +668,15 @@ defmodule Plausible.Stats.Imported do})end- defp group_imported_by(q, :path) do+ defp group_imported_by(q, :path, query) when query.v2 do+ q+ |> group_by([i], i.path)+ |> select_merge([i], %{+ path: fragment("if(not empty(?), ?, ?)", i.path, i.path, @none)+ })+ end++ defp group_imported_by(q, :path, _query) doq|> group_by([i], i.path)|> select_merge([i], %{@@ -668,20 +684,20 @@ defmodule Plausible.Stats.Imported do})end- defp select_joined_dimension(q, :city) do+ defp select_joined_dimension(q, :city, _query) doselect_merge(q, [s, i], %{city: fragment("greatest(?,?)", i.city, s.city)})end- defp select_joined_dimension(q, :os_version) do+ defp select_joined_dimension(q, :os_version, query) when not query.v2 doselect_merge(q, [s, i], %{os: fragment("if(empty(?), ?, ?)", s.os, i.os, s.os),os_version: fragment("if(empty(?), ?, ?)", s.os_version, i.os_version, s.os_version)})end- defp select_joined_dimension(q, :browser_version) do+ defp select_joined_dimension(q, :browser_version, query) when not query.v2 doselect_merge(q, [s, i], %{browser: fragment("if(empty(?), ?, ?)", s.browser, i.browser, s.browser),browser_version:@@ -689,7 +705,7 @@ defmodule Plausible.Stats.Imported do})end- defp select_joined_dimension(q, dim) when dim in [:url, :path] do+ defp select_joined_dimension(q, dim, query) when dim in [:url, :path] and not query.v2 doselect_merge(q, [s, i], %{breakdown_prop_value:fragment(@@ -701,7 +717,7 @@ defmodule Plausible.Stats.Imported do})end- defp select_joined_dimension(q, dim) do+ defp select_joined_dimension(q, dim, _query) doselect_merge(q, [s, i], %{^dim => fragment("if(empty(?), ?, ?)", field(s, ^dim), field(i, ^dim), field(s, ^dim))})@@ -716,7 +732,7 @@ defmodule Plausible.Stats.Imported dodefp select_joined_metrics(q, [:visits | rest]) doq- |> select_merge([s, i], %{visits: s.visits + i.visits})+ |> select_merge([s, i], %{visits: selected_as(s.visits + i.visits, :visits)})|> select_joined_metrics(rest)end@@ -728,13 +744,13 @@ defmodule Plausible.Stats.Imported dodefp select_joined_metrics(q, [:events | rest]) doq- |> select_merge([s, i], %{events: s.events + i.events})+ |> select_merge([s, i], %{events: selected_as(s.events + i.events, :events)})|> select_joined_metrics(rest)enddefp select_joined_metrics(q, [:pageviews | rest]) doq- |> select_merge([s, i], %{pageviews: s.pageviews + i.pageviews})+ |> select_merge([s, i], %{pageviews: selected_as(s.pageviews + i.pageviews, :pageviews)})|> select_joined_metrics(rest)enddiff --git a/lib/plausible/stats/metrics.ex b/lib/plausible/stats/metrics.exindex 36a1c3743e..733c7d6e1c 100644--- a/lib/plausible/stats/metrics.ex+++ b/lib/plausible/stats/metrics.ex@@ -21,6 +21,8 @@ defmodule Plausible.Stats.Metrics do@metric_mappings Enum.into(@all_metrics, %{}, fn metric -> {to_string(metric), metric} end)+ def metric?(value), do: Enum.member?(@all_metrics, value)+def from_string!(str) doMap.fetch!(@metric_mappings, str)enddiff --git a/lib/plausible/stats/query.ex b/lib/plausible/stats/query.exindex 737b3ba0bf..ae25c6536b 100644--- a/lib/plausible/stats/query.ex+++ b/lib/plausible/stats/query.ex@@ -13,7 +13,11 @@ defmodule Plausible.Stats.Query donow: nil,experimental_session_count?: false,experimental_reduced_joins?: false,- latest_import_end_date: nil+ latest_import_end_date: nil,+ metrics: [],+ order_by: [],+ timezone: nil,+ v2: falserequire OpenTelemetry.Tracer, as: Traceralias Plausible.Stats.{Filters, Interval, Imported}@@ -42,6 +46,19 @@ defmodule Plausible.Stats.Query doqueryend+ def build(site, params) do+ with {:ok, query_data} <- Filters.QueryParser.parse(site, params) do+ query =+ struct!(__MODULE__, Map.to_list(query_data))+ |> put_imported_opts(site, %{})+ |> put_experimental_session_count(site, params)+ |> put_experimental_reduced_joins(site, params)+ |> struct!(v2: true)++ {:ok, query}+ end+ end+defp put_experimental_session_count(query, site, params) doif Map.has_key?(params, "experimental_session_count") dostruct!(query,diff --git a/lib/plausible/stats/query_optimizer.ex b/lib/plausible/stats/query_optimizer.exnew file mode 100644index 0000000000..936fa2758f--- /dev/null+++ b/lib/plausible/stats/query_optimizer.ex@@ -0,0 +1,28 @@+defmodule Plausible.Stats.QueryOptimizer do+ @moduledoc false++ alias Plausible.Stats.Query++ def optimize(query) do+ Enum.reduce(pipeline(), query, fn step, acc -> step.(acc) end)+ end++ defp pipeline() do+ [+ &add_missing_order_by/1,+ &update_group_by_time/1+ ]+ end++ defp add_missing_order_by(%Query{order_by: nil} = query) do+ %Query{query | order_by: [{hd(query.metrics), :desc}]}+ end++ defp add_missing_order_by(query), do: query++ defp update_group_by_time(%Query{dimensions: ["time" | rest]} = query) do+ %Query{query | dimensions: ["time:month" | rest]}+ end++ defp update_group_by_time(query), do: query+enddiff --git a/lib/plausible/stats/query_result.ex b/lib/plausible/stats/query_result.exnew file mode 100644index 0000000000..d7b72bf80d--- /dev/null+++ b/lib/plausible/stats/query_result.ex@@ -0,0 +1,52 @@+defmodule Plausible.Stats.QueryResult do+ @moduledoc false++ alias Plausible.Stats.SQL.QueryBuilder+ alias Plausible.Stats.Filters+ alias Plausible.Stats.Query++ @derive Jason.Encoder+ defstruct results: [],+ query: nil,+ meta: %{}++ def from(results, query) do+ results_list =+ results+ |> Enum.map(fn entry ->+ %{+ dimensions: Enum.map(query.dimensions, &Map.get(entry, QueryBuilder.shortname(&1))),+ metrics: Enum.map(query.metrics, &Map.get(entry, &1))+ }+ end)++ struct!(+ __MODULE__,+ results: results_list,+ query: %{+ metrics: query.metrics,+ date_range: [query.date_range.first, query.date_range.last],+ filters: query.filters |> Enum.map(&serializable_filter/1),+ dimensions: query.dimensions,+ order_by: query.order_by |> Enum.map(&Tuple.to_list/1)+ },+ meta: meta(query)+ )+ end++ defp meta(%Query{skip_imported_reason: :unsupported_query}) do+ %{+ warning:+ "Imported stats are not included in the results because query parameters are not supported. " <>+ "For more information, see: https://plausible.io/docs/stats-api#filtering-imported-stats"+ }+ end++ defp meta(_), do: %{}++ defp serializable_filter([operation, "event:goal", clauses]) do+ [operation, "event:goal", Enum.map(clauses, &Filters.Utils.unwrap_goal_value/1)]+ end++ defp serializable_filter(filter), do: filter+enddiff --git a/lib/plausible/stats/sql/expression.ex b/lib/plausible/stats/sql/expression.exnew file mode 100644index 0000000000..cdd1cffe44--- /dev/null+++ b/lib/plausible/stats/sql/expression.ex@@ -0,0 +1,89 @@+defmodule Plausible.Stats.SQL.Expression do+ @moduledoc false++ import Ecto.Query++ use Plausible.Stats.Fragments++ @no_ref "Direct / None"+ @not_set "(not set)"++ defmacrop field_or_blank_value(expr, empty_value) do+ quote do+ dynamic(+ [t],+ fragment("if(empty(?), ?, ?)", unquote(expr), unquote(empty_value), unquote(expr))+ )+ end+ end++ def dimension("time:hour", query) do+ dynamic([t], fragment("toStartOfHour(toTimeZone(?, ?))", t.timestamp, ^query.timezone))+ end++ def dimension("time:day", query) do+ dynamic([t], fragment("toDate(toTimeZone(?, ?))", t.timestamp, ^query.timezone))+ end++ def dimension("time:month", query) do+ dynamic([t], fragment("toStartOfMonth(toTimeZone(?, ?))", t.timestamp, ^query.timezone))+ end++ def dimension("event:name", _query), do: dynamic([t], t.name)+ def dimension("event:page", _query), do: dynamic([t], t.pathname)+ def dimension("event:hostname", _query), do: dynamic([t], t.hostname)++ def dimension("event:props:" <> property_name, _query) do+ dynamic(+ [t],+ fragment(+ "if(not empty(?), ?, '(none)')",+ get_by_key(t, :meta, ^property_name),+ get_by_key(t, :meta, ^property_name)+ )+ )+ end++ def dimension("visit:entry_page", _query), do: dynamic([t], t.entry_page)+ def dimension("visit:exit_page", _query), do: dynamic([t], t.exit_page)++ def dimension("visit:utm_medium", _query),+ do: field_or_blank_value(t.utm_medium, @not_set)++ def dimension("visit:utm_source", _query),+ do: field_or_blank_value(t.utm_source, @not_set)++ def dimension("visit:utm_campaign", _query),+ do: field_or_blank_value(t.utm_campaign, @not_set)++ def dimension("visit:utm_content", _query),+ do: field_or_blank_value(t.utm_content, @not_set)++ def dimension("visit:utm_term", _query),+ do: field_or_blank_value(t.utm_term, @not_set)++ def dimension("visit:source", _query),+ do: field_or_blank_value(t.source, @no_ref)++ def dimension("visit:referrer", _query),+ do: field_or_blank_value(t.referrer, @no_ref)++ def dimension("visit:device", _query),+ do: field_or_blank_value(t.device, @not_set)++ def dimension("visit:os", _query), do: field_or_blank_value(t.os, @not_set)++ def dimension("visit:os_version", _query),+ do: field_or_blank_value(t.os_version, @not_set)++ def dimension("visit:browser", _query),+ do: field_or_blank_value(t.browser, @not_set)++ def dimension("visit:browser_version", _query),+ do: field_or_blank_value(t.browser_version, @not_set)++ # :TODO: Locations also set extra filters+ def dimension("visit:country", _query), do: dynamic([t], t.country)+ def dimension("visit:region", _query), do: dynamic([t], t.region)+ def dimension("visit:city", _query), do: dynamic([t], t.city)+enddiff --git a/lib/plausible/stats/sql/query_builder.ex b/lib/plausible/stats/sql/query_builder.exnew file mode 100644index 0000000000..a966be4b12--- /dev/null+++ b/lib/plausible/stats/sql/query_builder.ex@@ -0,0 +1,200 @@+defmodule Plausible.Stats.SQL.QueryBuilder do+ @moduledoc false++ use Plausible++ import Ecto.Query+ import Plausible.Stats.Imported++ alias Plausible.Stats.{Base, Query, TableDecider, Util, Filters, Metrics}+ alias Plausible.Stats.SQL.Expression++ def build(query, site) do+ {event_metrics, sessions_metrics, _other_metrics} =+ query.metrics+ |> Util.maybe_add_visitors_metric()+ |> TableDecider.partition_metrics(query)++ join_query_results(+ build_events_query(site, query, event_metrics),+ event_metrics,+ build_sessions_query(site, query, sessions_metrics),+ sessions_metrics,+ query+ )+ end++ def shortname(metric) when is_atom(metric), do: metric+ def shortname(dimension), do: Plausible.Stats.Filters.without_prefix(dimension)++ defp build_events_query(_, _, []), do: nil++ defp build_events_query(site, query, event_metrics) do+ q =+ from(+ e in "events_v2",+ where: ^Filters.WhereBuilder.build(:events, site, query),+ select: ^Base.select_event_metrics(event_metrics)+ )++ on_ee do+ q = Plausible.Stats.Sampling.add_query_hint(q, query)+ end++ q+ |> join_sessions_if_needed(site, query)+ |> build_group_by(query)+ |> merge_imported(site, query, event_metrics)+ |> Base.maybe_add_conversion_rate(site, query, event_metrics)+ end++ defp join_sessions_if_needed(q, site, query) do+ if TableDecider.events_join_sessions?(query) do+ sessions_q =+ from(+ s in Base.query_sessions(site, query),+ select: %{session_id: s.session_id},+ where: s.sign == 1,+ group_by: s.session_id+ )++ from(+ e in q,+ join: sq in subquery(sessions_q),+ on: e.session_id == sq.session_id+ )+ else+ q+ end+ end++ def build_sessions_query(_, _, []), do: nil++ def build_sessions_query(site, query, session_metrics) do+ q =+ from(+ e in "sessions_v2",+ where: ^Filters.WhereBuilder.build(:sessions, site, query),+ select: ^Base.select_session_metrics(session_metrics, query)+ )++ on_ee do+ q = Plausible.Stats.Sampling.add_query_hint(q, query)+ end++ q+ |> join_events_if_needed(site, query)+ |> build_group_by(query)+ |> merge_imported(site, query, session_metrics)+ end++ def join_events_if_needed(q, site, query) do+ if Query.has_event_filters?(query) do+ events_q =+ from(e in "events_v2",+ where: ^Filters.WhereBuilder.build(:events, site, query),+ select: %{+ session_id: fragment("DISTINCT ?", e.session_id),+ _sample_factor: fragment("_sample_factor")+ }+ )++ on_ee do+ events_q = Plausible.Stats.Sampling.add_query_hint(events_q, query)+ end++ from(s in q,+ join: e in subquery(events_q),+ on: s.session_id == e.session_id+ )+ else+ q+ end+ end++ defp build_group_by(q, query) do+ Enum.reduce(query.dimensions, q, fn dimension, q ->+ q+ |> select_merge(^%{shortname(dimension) => Expression.dimension(dimension, query)})+ |> group_by(^Expression.dimension(dimension, query))+ end)+ end++ defp build_order_by(q, query, mode) do+ Enum.reduce(query.order_by, q, &build_order_by(&2, query, &1, mode))+ end++ def build_order_by(q, query, {metric_or_dimension, order_direction}, :inner) do+ order_by(+ q,+ [t],+ ^{+ order_direction,+ if(+ Metrics.metric?(metric_or_dimension),+ do: dynamic([], selected_as(^shortname(metric_or_dimension))),+ else: Expression.dimension(metric_or_dimension, query)+ )+ }+ )+ end++ def build_order_by(q, _query, {metric_or_dimension, order_direction}, :outer) do+ order_by(+ q,+ [t],+ ^{+ order_direction,+ dynamic([], selected_as(^shortname(metric_or_dimension)))+ }+ )+ end++ defmacrop select_join_fields(q, list, table_name) do+ quote do+ Enum.reduce(unquote(list), unquote(q), fn metric_or_dimension, q ->+ select_merge(+ q,+ ^%{+ shortname(metric_or_dimension) =>+ dynamic(+ [e, s],+ selected_as(+ field(unquote(table_name), ^shortname(metric_or_dimension)),+ ^shortname(metric_or_dimension)+ )+ )+ }+ )+ end)+ end+ end++ defp join_query_results(nil, _, nil, _, _query), do: nil++ defp join_query_results(events_q, _, nil, _, query),+ do: events_q |> build_order_by(query, :inner)++ defp join_query_results(nil, _, sessions_q, _, query),+ do: sessions_q |> build_order_by(query, :inner)++ defp join_query_results(events_q, event_metrics, sessions_q, sessions_metrics, query) do+ join(subquery(events_q), :left, [e], s in subquery(sessions_q),+ on: ^build_group_by_join(query)+ )+ |> select_join_fields(query.dimensions, e)+ |> select_join_fields(event_metrics, e)+ |> select_join_fields(List.delete(sessions_metrics, :sample_percent), s)+ |> build_order_by(query, :outer)+ end++ defp build_group_by_join(%Query{dimensions: []}), do: true++ defp build_group_by_join(query) do+ query.dimensions+ |> Enum.map(fn dim ->+ dynamic([e, s], field(e, ^shortname(dim)) == field(s, ^shortname(dim)))+ end)+ |> Enum.reduce(fn condition, acc -> dynamic([], ^acc and ^condition) end)+ end+enddiff --git a/lib/plausible/stats/table_decider.ex b/lib/plausible/stats/table_decider.exindex 8b77ca5928..629744228b 100644--- a/lib/plausible/stats/table_decider.ex+++ b/lib/plausible/stats/table_decider.ex@@ -9,10 +9,18 @@ defmodule Plausible.Stats.TableDecider doalias Plausible.Stats.Querydef events_join_sessions?(query) do- Enum.any?(query.filters, &(filters_partitioner(query, &1) == :session))+ query+ |> filter_keys()+ |> Enum.any?(&(filters_partitioner(query, &1) == :session))end- def partition_metrics(metrics, query, breakdown_property \\ nil) do+ def event_filters?(query) do+ query+ |> filter_keys()+ |> Enum.any?(&(filters_partitioner(query, &1) == :event))+ end++ def partition_metrics(metrics, query) do%{event: event_only_metrics,session: session_only_metrics,@@ -22,16 +30,10 @@ defmodule Plausible.Stats.TableDecider do} =partition(metrics, query, &metric_partitioner/2)- # Treat breakdown property as yet another filter- query =- if breakdown_property do- Query.put_filter(query, [:is, breakdown_property, []])- else- query- end-%{event: event_only_filters, session: session_only_filters} =- partition(query.filters, query, &filters_partitioner/2)+ query+ |> filter_keys()+ |> partition(query, &filters_partitioner/2)cond do# Only one table needs to be queried@@ -55,6 +57,12 @@ defmodule Plausible.Stats.TableDecider doendend+ defp filter_keys(query) do+ query.filters+ |> Enum.map(fn [_, filter_key | _rest] -> filter_key end)+ |> Enum.concat(query.dimensions)+ end+defp metric_partitioner(_, :conversion_rate), do: :eventdefp metric_partitioner(_, :average_revenue), do: :eventdefp metric_partitioner(_, :total_revenue), do: :event@@ -83,16 +91,16 @@ defmodule Plausible.Stats.TableDecider dodefp metric_partitioner(_, _), do: :either- defp filters_partitioner(_, [_, "event:" <> _ | _rest]), do: :event- defp filters_partitioner(_, [_, "visit:entry_page" | _rest]), do: :session- defp filters_partitioner(_, [_, "visit:entry_page_hostname" | _rest]), do: :session- defp filters_partitioner(_, [_, "visit:exit_page" | _rest]), do: :session- defp filters_partitioner(_, [_, "visit:exit_page_hostname" | _rest]), do: :session+ defp filters_partitioner(_, "event:" <> _), do: :event+ defp filters_partitioner(_, "visit:entry_page"), do: :session+ defp filters_partitioner(_, "visit:entry_page_hostname"), do: :session+ defp filters_partitioner(_, "visit:exit_page"), do: :session+ defp filters_partitioner(_, "visit:exit_page_hostname"), do: :session- defp filters_partitioner(%Query{experimental_reduced_joins?: true}, [_, "visit:" <> _ | _rest]),+ defp filters_partitioner(%Query{experimental_reduced_joins?: true}, "visit:" <> _),do: :either- defp filters_partitioner(_, [_, "visit:" <> _ | _rest]),+ defp filters_partitioner(_, "visit:" <> _),do: :sessiondefp filters_partitioner(%Query{experimental_reduced_joins?: false}, {unknown, _}) dodiff --git a/lib/plausible_web/controllers/api/external_query_api_controller.ex b/lib/plausible_web/controllers/api/external_query_api_controller.exnew file mode 100644index 0000000000..d2bf01401f--- /dev/null+++ b/lib/plausible_web/controllers/api/external_query_api_controller.ex@@ -0,0 +1,23 @@+defmodule PlausibleWeb.Api.ExternalQueryApiController do+ @moduledoc false++ use PlausibleWeb, :controller+ use Plausible.Repo+ use PlausibleWeb.Plugs.ErrorHandler+ alias Plausible.Stats.Query++ def query(conn, params) do+ site = Repo.preload(conn.assigns.site, :owner)++ case Query.build(site, params) do+ {:ok, query} ->+ results = Plausible.Stats.query(site, query)+ json(conn, results)++ {:error, message} ->+ conn+ |> put_status(400)+ |> json(%{error: message})+ end+ end+enddiff --git a/lib/plausible_web/router.ex b/lib/plausible_web/router.exindex ab83484c15..bd7179c13f 100644--- a/lib/plausible_web/router.ex+++ b/lib/plausible_web/router.ex@@ -175,6 +175,12 @@ defmodule PlausibleWeb.Router doget "/timeseries", ExternalStatsController, :timeseriesend+ scope "/api/v2", PlausibleWeb.Api do+ pipe_through [:public_api, PlausibleWeb.AuthorizeStatsApiPlug]++ post "/query", ExternalQueryApiController, :query+ end+on_ee doscope "/api/v1/sites", PlausibleWeb.Api dopipe_through [:public_api, PlausibleWeb.AuthorizeSitesApiPlug]diff --git a/test/plausible/stats/query_parser_test.exs b/test/plausible/stats/query_parser_test.exsindex 39baead3f3..481f0362dc 100644--- a/test/plausible/stats/query_parser_test.exs+++ b/test/plausible/stats/query_parser_test.exs@@ -1,44 +1,50 @@defmodule Plausible.Stats.Filters.QueryParserTest do- use ExUnit.Case, async: true+ use Plausible.DataCase+alias Plausible.Stats.Filtersimport Plausible.Stats.Filters.QueryParser- def check_success(params, expected_result) do- assert parse(params) == {:ok, expected_result}+ setup [:create_user, :create_new_site]++ @date_range Date.range(Timex.today(), Timex.today())++ def check_success(params, site, expected_result) do+ assert parse(site, params) == {:ok, expected_result}end- def check_error(params, expected_error_message) do- {:error, message} = parse(params)+ def check_error(params, site, expected_error_message) do+ {:error, message} = parse(site, params)assert message =~ expected_error_messageend- test "parsing empty map fails" do+ test "parsing empty map fails", %{site: site} do%{}- |> check_error("No valid metrics passed")+ |> check_error(site, "No valid metrics passed")enddescribe "metrics validation" do- test "valid metrics passed" do+ test "valid metrics passed", %{site: site} do%{"metrics" => ["visitors", "events"], "date_range" => "all"}- |> check_success(%{+ |> check_success(site, %{metrics: [:visitors, :events],- date_range: "all",+ date_range: @date_range,filters: [],dimensions: [],- order_by: nil+ order_by: nil,+ timezone: site.timezone,+ imported_data_requested: false})end- test "invalid metric passed" do+ test "invalid metric passed", %{site: site} do%{"metrics" => ["visitors", "event:name"], "date_range" => "all"}- |> check_error("Unknown metric '\"event:name\"'")+ |> check_error(site, "Unknown metric '\"event:name\"'")end- test "fuller list of metrics" do+ test "fuller list of metrics", %{site: site} do%{"metrics" => ["time_on_page",- "conversion_rate","visitors","pageviews","visits",@@ -48,10 +54,9 @@ defmodule Plausible.Stats.Filters.QueryParserTest do],"date_range" => "all"}- |> check_success(%{+ |> check_success(site, %{metrics: [:time_on_page,- :conversion_rate,:visitors,:pageviews,:visits,@@ -59,22 +64,24 @@ defmodule Plausible.Stats.Filters.QueryParserTest do:bounce_rate,:visit_duration],- date_range: "all",+ date_range: @date_range,filters: [],dimensions: [],- order_by: nil+ order_by: nil,+ timezone: site.timezone,+ imported_data_requested: false})end- test "same metric queried multiple times" do+ test "same metric queried multiple times", %{site: site} do%{"metrics" => ["events", "visitors", "visitors"], "date_range" => "all"}- |> check_error(~r/Metrics cannot be queried multiple times/)+ |> check_error(site, ~r/Metrics cannot be queried multiple times/)endenddescribe "filters validation" dofor operation <- [:is, :is_not, :matches, :does_not_match] do- test "#{operation} filter" do+ test "#{operation} filter", %{site: site} do%{"metrics" => ["visitors"],"date_range" => "all",@@ -82,18 +89,20 @@ defmodule Plausible.Stats.Filters.QueryParserTest do[Atom.to_string(unquote(operation)), "event:name", ["foo"]]]}- |> check_success(%{+ |> check_success(site, %{metrics: [:visitors],- date_range: "all",+ date_range: @date_range,filters: [[unquote(operation), "event:name", ["foo"]]],dimensions: [],- order_by: nil+ order_by: nil,+ timezone: site.timezone,+ imported_data_requested: false})end- test "#{operation} filter with invalid clause" do+ test "#{operation} filter with invalid clause", %{site: site} do%{"metrics" => ["visitors"],"date_range" => "all",@@ -101,11 +110,11 @@ defmodule Plausible.Stats.Filters.QueryParserTest do[Atom.to_string(unquote(operation)), "event:name", "foo"]]}- |> check_error(~r/Invalid filter/)+ |> check_error(site, ~r/Invalid filter/)endend- test "filtering by invalid operation" do+ test "filtering by invalid operation", %{site: site} do%{"metrics" => ["visitors"],"date_range" => "all",@@ -113,10 +122,10 @@ defmodule Plausible.Stats.Filters.QueryParserTest do["exists?", "event:name", ["foo"]]]}- |> check_error(~r/Unknown operator for filter/)+ |> check_error(site, ~r/Unknown operator for filter/)end- test "filtering by custom properties" do+ test "filtering by custom properties", %{site: site} do%{"metrics" => ["visitors"],"date_range" => "all",@@ -124,20 +133,22 @@ defmodule Plausible.Stats.Filters.QueryParserTest do["is", "event:props:foobar", ["value"]]]}- |> check_success(%{+ |> check_success(site, %{metrics: [:visitors],- date_range: "all",+ date_range: @date_range,filters: [[:is, "event:props:foobar", ["value"]]],dimensions: [],- order_by: nil+ order_by: nil,+ timezone: site.timezone,+ imported_data_requested: false})endfor dimension <- Filters.event_props() doif dimension != "goal" do- test "filtering by event:#{dimension} filter" do+ test "filtering by event:#{dimension} filter", %{site: site} do%{"metrics" => ["visitors"],"date_range" => "all",@@ -145,21 +156,23 @@ defmodule Plausible.Stats.Filters.QueryParserTest do["is", "event:#{unquote(dimension)}", ["foo"]]]}- |> check_success(%{+ |> check_success(site, %{metrics: [:visitors],- date_range: "all",+ date_range: @date_range,filters: [[:is, "event:#{unquote(dimension)}", ["foo"]]],dimensions: [],- order_by: nil+ order_by: nil,+ timezone: site.timezone,+ imported_data_requested: false})endendendfor dimension <- Filters.visit_props() do- test "filtering by visit:#{dimension} filter" do+ test "filtering by visit:#{dimension} filter", %{site: site} do%{"metrics" => ["visitors"],"date_range" => "all",@@ -167,66 +180,125 @@ defmodule Plausible.Stats.Filters.QueryParserTest do["is", "visit:#{unquote(dimension)}", ["foo"]]]}- |> check_success(%{+ |> check_success(site, %{metrics: [:visitors],- date_range: "all",+ date_range: @date_range,filters: [[:is, "visit:#{unquote(dimension)}", ["foo"]]],dimensions: [],- order_by: nil+ order_by: nil,+ timezone: site.timezone,+ imported_data_requested: false})endend- test "filtering by event:goal" do+ test "invalid event filter", %{site: site} do%{"metrics" => ["visitors"],"date_range" => "all","filters" => [- ["is", "event:goal", ["Signup", "Visit /thank-you"]]+ ["is", "event:device", ["foo"]]]}- |> check_success(%{+ |> check_error(site, ~r/Invalid filter /)+ end++ test "invalid visit filter", %{site: site} do+ %{+ "metrics" => ["visitors"],+ "date_range" => "all",+ "filters" => [+ ["is", "visit:name", ["foo"]]+ ]+ }+ |> check_error(site, ~r/Invalid filter /)+ end++ test "invalid filter", %{site: site} do+ %{+ "metrics" => ["visitors"],+ "date_range" => "all",+ "filters" => "foobar"+ }+ |> check_error(site, ~r/Invalid filters passed/)+ end+ end++ describe "include validation" do+ test "setting include.imports", %{site: site} do+ %{+ "metrics" => ["visitors"],+ "date_range" => "all",+ "include" => %{"imports" => true}+ }+ |> check_success(site, %{metrics: [:visitors],- date_range: "all",- filters: [- [:is, "event:goal", [{:event, "Signup"}, {:page, "/thank-you"}]]- ],+ date_range: @date_range,+ filters: [],dimensions: [],- order_by: nil+ order_by: nil,+ timezone: site.timezone,+ imported_data_requested: true})end- test "invalid event filter" do+ test "setting invalid imports value", %{site: site} do+ %{+ "metrics" => ["visitors"],+ "date_range" => "all",+ "include" => "foobar"+ }+ |> check_error(site, ~r/Invalid include passed/)+ end+ end++ describe "event:goal filter validation" do+ test "valid filters", %{site: site} do+ insert(:goal, %{site: site, event_name: "Signup"})+ insert(:goal, %{site: site, page_path: "/thank-you"})+%{"metrics" => ["visitors"],"date_range" => "all","filters" => [- ["is", "event:device", ["foo"]]+ ["is", "event:goal", ["Signup", "Visit /thank-you"]]]}- |> check_error(~r/Invalid filter /)+ |> check_success(site, %{+ metrics: [:visitors],+ date_range: @date_range,+ filters: [+ [:is, "event:goal", [{:event, "Signup"}, {:page, "/thank-you"}]]+ ],+ dimensions: [],+ order_by: nil,+ timezone: site.timezone,+ imported_data_requested: false+ })end- test "invalid visit filter" do+ test "invalid event filter", %{site: site} do%{"metrics" => ["visitors"],"date_range" => "all","filters" => [- ["is", "visit:name", ["foo"]]+ ["is", "event:goal", ["Signup"]]]}- |> check_error(~r/Invalid filter /)+ |> check_error(site, ~r/The goal `Signup` is not configured for this site/)end- test "invalid filter" do+ test "invalid page filter", %{site: site} do%{"metrics" => ["visitors"],"date_range" => "all",- "filters" => "foobar"+ "filters" => [+ ["is", "event:goal", ["Visit /thank-you"]]+ ]}- |> check_error(~r/Invalid filters passed/)+ |> check_error(site, ~r/The goal `Visit \/thank-you` is not configured for this site/)endend@@ -235,139 +307,288 @@ defmodule Plausible.Stats.Filters.QueryParserTest dodescribe "dimensions validation" dofor dimension <- Filters.event_props() do- test "event:#{dimension} dimension" do+ test "event:#{dimension} dimension", %{site: site} do%{"metrics" => ["visitors"],"date_range" => "all","dimensions" => ["event:#{unquote(dimension)}"]}- |> check_success(%{+ |> check_success(site, %{metrics: [:visitors],- date_range: "all",+ date_range: @date_range,filters: [],dimensions: ["event:#{unquote(dimension)}"],- order_by: nil+ order_by: nil,+ timezone: site.timezone,+ imported_data_requested: false})endendfor dimension <- Filters.visit_props() do- test "visit:#{dimension} dimension" do+ test "visit:#{dimension} dimension", %{site: site} do%{"metrics" => ["visitors"],"date_range" => "all","dimensions" => ["visit:#{unquote(dimension)}"]}- |> check_success(%{+ |> check_success(site, %{metrics: [:visitors],- date_range: "all",+ date_range: @date_range,filters: [],dimensions: ["visit:#{unquote(dimension)}"],- order_by: nil+ order_by: nil,+ timezone: site.timezone,+ imported_data_requested: false})endend- test "custom properties dimension" do+ test "custom properties dimension", %{site: site} do%{"metrics" => ["visitors"],"date_range" => "all","dimensions" => ["event:props:foobar"]}- |> check_success(%{+ |> check_success(site, %{metrics: [:visitors],- date_range: "all",+ date_range: @date_range,filters: [],dimensions: ["event:props:foobar"],- order_by: nil+ order_by: nil,+ timezone: site.timezone,+ imported_data_requested: false})end- test "invalid dimension name passed" do+ test "invalid custom property dimension", %{site: site} do+ %{+ "metrics" => ["visitors"],+ "date_range" => "all",+ "dimensions" => ["event:props:"]+ }+ |> check_error(site, ~r/Invalid dimensions/)+ end++ test "invalid dimension name passed", %{site: site} do%{"metrics" => ["visitors"],"date_range" => "all","dimensions" => ["visitors"]}- |> check_error(~r/Invalid dimensions/)+ |> check_error(site, ~r/Invalid dimensions/)end- test "invalid dimension" do+ test "invalid dimension", %{site: site} do%{"metrics" => ["visitors"],"date_range" => "all","dimensions" => "foobar"}- |> check_error(~r/Invalid dimensions/)+ |> check_error(site, ~r/Invalid dimensions/)end- test "dimensions are not unique" do+ test "dimensions are not unique", %{site: site} do%{"metrics" => ["visitors"],"date_range" => "all","dimensions" => ["event:name", "event:name"]}- |> check_error(~r/Some dimensions are listed multiple times/)+ |> check_error(site, ~r/Some dimensions are listed multiple times/)endenddescribe "order_by validation" do- test "ordering by metric" do+ test "ordering by metric", %{site: site} do%{"metrics" => ["visitors", "events"],"date_range" => "all","order_by" => [["events", "desc"], ["visitors", "asc"]]}- |> check_success(%{+ |> check_success(site, %{metrics: [:visitors, :events],- date_range: "all",+ date_range: @date_range,filters: [],dimensions: [],- order_by: [{:events, :desc}, {:visitors, :asc}]+ order_by: [{:events, :desc}, {:visitors, :asc}],+ timezone: site.timezone,+ imported_data_requested: false})end- test "ordering by dimension" do+ test "ordering by dimension", %{site: site} do%{"metrics" => ["visitors"],"date_range" => "all","dimensions" => ["event:name"],"order_by" => [["event:name", "desc"]]}- |> check_success(%{+ |> check_success(site, %{metrics: [:visitors],- date_range: "all",+ date_range: @date_range,filters: [],dimensions: ["event:name"],- order_by: [{"event:name", :desc}]+ order_by: [{"event:name", :desc}],+ timezone: site.timezone,+ imported_data_requested: false})end- test "ordering by invalid value" do+ test "ordering by invalid value", %{site: site} do%{"metrics" => ["visitors"],"date_range" => "all","order_by" => [["visssss", "desc"]]}- |> check_error(~r/Invalid order_by entry/)+ |> check_error(site, ~r/Invalid order_by entry/)end- test "ordering by not queried metric" do+ test "ordering by not queried metric", %{site: site} do%{"metrics" => ["visitors"],"date_range" => "all","order_by" => [["events", "desc"]]}- |> check_error(~r/Entry is not a queried metric or dimension/)+ |> check_error(site, ~r/Entry is not a queried metric or dimension/)end- test "ordering by not queried dimension" do+ test "ordering by not queried dimension", %{site: site} do%{"metrics" => ["visitors"],"date_range" => "all","order_by" => [["event:name", "desc"]]}- |> check_error(~r/Entry is not a queried metric or dimension/)+ |> check_error(site, ~r/Entry is not a queried metric or dimension/)+ end+ end++ describe "custom props access" do+ test "error if invalid filter", %{site: site, user: user} do+ ep =+ insert(:enterprise_plan, features: [Plausible.Billing.Feature.StatsAPI], user_id: user.id)++ insert(:subscription, user: user, paddle_plan_id: ep.paddle_plan_id)++ %{+ "metrics" => ["visitors"],+ "date_range" => "all",+ "filters" => [["is", "event:props:foobar", ["foo"]]]+ }+ |> check_error(+ site,+ ~r/The owner of this site does not have access to the custom properties feature/+ )+ end++ test "error if invalid dimension", %{site: site, user: user} do+ ep =+ insert(:enterprise_plan, features: [Plausible.Billing.Feature.StatsAPI], user_id: user.id)++ insert(:subscription, user: user, paddle_plan_id: ep.paddle_plan_id)++ %{+ "metrics" => ["visitors"],+ "date_range" => "all",+ "dimensions" => ["event:props:foobar"]+ }+ |> check_error(+ site,+ ~r/The owner of this site does not have access to the custom properties feature/+ )+ end+ end++ describe "conversion_rate metric" do+ test "fails validation on its own", %{site: site} do+ %{+ "metrics" => ["conversion_rate"],+ "date_range" => "all"+ }+ |> check_error(+ site,+ ~r/Metric `conversion_rate` can only be queried with event:goal filters or dimensions/+ )+ end++ test "succeeds with event:goal filter", %{site: site} do+ insert(:goal, %{site: site, event_name: "Signup"})++ %{+ "metrics" => ["conversion_rate"],+ "date_range" => "all",+ "filters" => [["is", "event:goal", ["Signup"]]]+ }+ |> check_success(site, %{+ metrics: [:conversion_rate],+ date_range: @date_range,+ filters: [[:is, "event:goal", [event: "Signup"]]],+ dimensions: [],+ order_by: nil,+ timezone: site.timezone,+ imported_data_requested: false+ })+ end++ test "succeeds with event:goal dimension", %{site: site} do+ insert(:goal, %{site: site, event_name: "Signup"})++ %{+ "metrics" => ["conversion_rate"],+ "date_range" => "all",+ "dimensions" => ["event:goal"]+ }+ |> check_success(site, %{+ metrics: [:conversion_rate],+ date_range: @date_range,+ filters: [],+ dimensions: ["event:goal"],+ order_by: nil,+ timezone: site.timezone,+ imported_data_requested: false+ })+ end+ end++ describe "session metrics" do+ test "single session metric succeeds", %{site: site} do+ %{+ "metrics" => ["bounce_rate"],+ "date_range" => "all",+ "dimensions" => ["visit:device"]+ }+ |> check_success(site, %{+ metrics: [:bounce_rate],+ date_range: @date_range,+ filters: [],+ dimensions: ["visit:device"],+ order_by: nil,+ timezone: site.timezone,+ imported_data_requested: false+ })+ end++ test "fails if using session metric with event dimension", %{site: site} do+ %{+ "metrics" => ["bounce_rate"],+ "date_range" => "all",+ "dimensions" => ["event:props:foo"]+ }+ |> check_error(+ site,+ "Session metric(s) `bounce_rate` cannot be queried along with event filters or dimensions"+ )+ end++ test "fails if using session metric with event filter", %{site: site} do+ %{+ "metrics" => ["bounce_rate"],+ "date_range" => "all",+ "filters" => [["is", "event:props:foo", ["(none)"]]]+ }+ |> check_error(+ site,+ "Session metric(s) `bounce_rate` cannot be queried along with event filters or dimensions"+ )endendenddiff --git a/test/plausible/stats/table_decider_test.exs b/test/plausible/stats/table_decider_test.exsindex c7d85bbe74..7226542c17 100644--- a/test/plausible/stats/table_decider_test.exs+++ b/test/plausible/stats/table_decider_test.exs@@ -5,62 +5,62 @@ defmodule Plausible.Stats.TableDeciderTest doimport Plausible.Stats.TableDecidertest "events_join_sessions? with experimental_reduced_joins disabled" do- assert not events_join_sessions?(make_query(false, %{}))- assert not events_join_sessions?(make_query(false, %{name: "pageview"}))- assert events_join_sessions?(make_query(false, %{source: "Google"}))- assert events_join_sessions?(make_query(false, %{entry_page: "/"}))- assert events_join_sessions?(make_query(false, %{exit_page: "/"}))+ assert not events_join_sessions?(make_query(false, []))+ assert not events_join_sessions?(make_query(false, ["event:name"]))+ assert events_join_sessions?(make_query(false, ["visit:source"]))+ assert events_join_sessions?(make_query(false, ["visit:entry_page"]))+ assert events_join_sessions?(make_query(false, ["visit:exit_page"]))endtest "events_join_sessions? with experimental_reduced_joins enabled" do- assert not events_join_sessions?(make_query(true, %{}))- assert not events_join_sessions?(make_query(true, %{name: "pageview"}))- assert not events_join_sessions?(make_query(true, %{source: "Google"}))- assert events_join_sessions?(make_query(true, %{entry_page: "/"}))- assert events_join_sessions?(make_query(true, %{exit_page: "/"}))+ assert not events_join_sessions?(make_query(true, []))+ assert not events_join_sessions?(make_query(true, ["event:name"]))+ assert not events_join_sessions?(make_query(true, ["visit:source"]))+ assert events_join_sessions?(make_query(true, ["visit:entry_page"]))+ assert events_join_sessions?(make_query(true, ["visit:exit_page"]))enddescribe "partition_metrics" dotest "with no metrics or filters" do- query = make_query(false, %{})+ query = make_query(false, [])assert partition_metrics([], query) == {[], [], []}endtest "session-only metrics accordingly" do- query = make_query(false, %{})+ query = make_query(false, [])assert partition_metrics([:bounce_rate, :views_per_visit], query) =={[], [:bounce_rate, :views_per_visit], []}endtest "event-only metrics accordingly" do- query = make_query(false, %{})+ query = make_query(false, [])assert partition_metrics([:total_revenue, :visitors], query) =={[:total_revenue, :visitors], [], []}endtest "filters from both, event-only metrics" do- query = make_query(false, %{name: "pageview", source: "Google"})+ query = make_query(false, ["event:name", "visit:source"])assert partition_metrics([:total_revenue], query) == {[:total_revenue], [], []}endtest "filters from both, session-only metrics" do- query = make_query(false, %{name: "pageview", source: "Google"})+ query = make_query(false, ["event:name", "visit:source"])assert partition_metrics([:bounce_rate], query) == {[], [:bounce_rate], []}endtest "session filters but no session metrics" do- query = make_query(false, %{source: "Google"})+ query = make_query(false, ["visit:source"])assert partition_metrics([:total_revenue], query) == {[:total_revenue], [], []}endtest "sample_percent is added to both types of metrics" do- query = make_query(false, %{})+ query = make_query(false, [])assert partition_metrics([:total_revenue, :sample_percent], query) =={[:total_revenue, :sample_percent], [], []}@@ -73,14 +73,14 @@ defmodule Plausible.Stats.TableDeciderTest doendtest "other metrics put in its own result" do- query = make_query(false, %{})+ query = make_query(false, [])assert partition_metrics([:time_on_page, :percentage, :total_visitors], query) =={[], [], [:time_on_page, :percentage, :total_visitors]}endtest "raises if unknown metric" do- query = make_query(false, %{})+ query = make_query(false, [])assert_raise ArgumentError, fn ->partition_metrics([:foobar], query)@@ -90,7 +90,7 @@ defmodule Plausible.Stats.TableDeciderTest dodescribe "partition_metrics with experimental_reduced_joins enabled" dotest "metrics that can be calculated on either when event-only metrics" do- query = make_query(true, %{})+ query = make_query(true, [])assert partition_metrics([:total_revenue, :visitors], query) =={[:total_revenue, :visitors], [], []}@@ -99,7 +99,7 @@ defmodule Plausible.Stats.TableDeciderTest doendtest "metrics that can be calculated on either when session-only metrics" do- query = make_query(true, %{})+ query = make_query(true, [])assert partition_metrics([:bounce_rate, :visitors], query) =={[], [:bounce_rate, :visitors], []}@@ -109,56 +109,60 @@ defmodule Plausible.Stats.TableDeciderTest doendtest "metrics that can be calculated on either are biased to sessions" do- query = make_query(true, %{})+ query = make_query(true, [])assert partition_metrics([:bounce_rate, :total_revenue, :visitors], query) =={[:total_revenue], [:bounce_rate, :visitors], []}endtest "sample_percent is handled with either metrics" do- query = make_query(true, %{})+ query = make_query(true, [])assert partition_metrics([:visitors, :sample_percent], query) =={[], [:visitors, :sample_percent], []}endtest "metric can be calculated on either, but filtering on events" do- query = make_query(true, %{name: "pageview"})+ query = make_query(true, ["event:name"])assert partition_metrics([:visitors], query) == {[:visitors], [], []}endtest "metric can be calculated on either, but filtering on events and sessions" do- query = make_query(true, %{name: "pageview", exit_page: "/"})+ query = make_query(true, ["event:name", "visit:exit_page"])assert partition_metrics([:visitors], query) == {[], [:visitors], []}endtest "metric can be calculated on either, filtering on either" do- query = make_query(true, %{source: "Google"})+ query = make_query(true, ["visit:source"])assert partition_metrics([:visitors], query) == {[], [:visitors], []}endtest "metric can be calculated on either, filtering on sessions" do- query = make_query(true, %{exit_page: "/"})+ query = make_query(true, ["visit:exit_page"])assert partition_metrics([:visitors], query) == {[], [:visitors], []}end- test "breakdown value leans metric" do- query = make_query(true, %{})+ test "query dimensions lean metric" do+ assert partition_metrics([:visitors], make_query(true, [], ["event:name"])) ==+ {[:visitors], [], []}- assert partition_metrics([:visitors], query, "event:name") == {[:visitors], [], []}- assert partition_metrics([:visitors], query, "visit:source") == {[], [:visitors], []}- assert partition_metrics([:visitors], query, "visit:exit_page") == {[], [:visitors], []}+ assert partition_metrics([:visitors], make_query(true, [], ["visit:source"])) ==+ {[], [:visitors], []}++ assert partition_metrics([:visitors], make_query(true, [], ["visit:exit_page"])) ==+ {[], [:visitors], []}endend- defp make_query(experimental_reduced_joins?, filters) do+ defp make_query(experimental_reduced_joins?, filter_keys, dimensions \\ []) doQuery.from(build(:site), %{"experimental_reduced_joins" => to_string(experimental_reduced_joins?),- "filters" => Jason.encode!(filters)+ "filters" => Enum.map(filter_keys, fn filter_key -> ["is", filter_key, []] end),+ "dimensions" => dimensions})endenddiff --git a/test/plausible_web/controllers/api/external_stats_controller/query_test.exs b/test/plausible_web/controllers/api/external_stats_controller/query_test.exsnew file mode 100644index 0000000000..fa6ab457f0--- /dev/null+++ b/test/plausible_web/controllers/api/external_stats_controller/query_test.exs@@ -0,0 +1,3106 @@+defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest do+ use PlausibleWeb.ConnCase+ alias Plausible.Billing.Feature++ @user_id 1231++ setup [:create_user, :create_new_site, :create_api_key, :use_api_key]++ describe "feature access" do+ test "cannot break down by a custom prop without access to the props feature", %{+ conn: conn,+ user: user,+ site: site+ } do+ ep = insert(:enterprise_plan, features: [Feature.StatsAPI], user_id: user.id)+ insert(:subscription, user: user, paddle_plan_id: ep.paddle_plan_id)++ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "metrics" => ["visitors"],+ "date_range" => "all",+ "dimensions" => ["event:props:author"]+ })++ assert json_response(conn, 400)["error"] ==+ "The owner of this site does not have access to the custom properties feature"+ end++ test "can break down by an internal prop key without access to the props feature", %{+ conn: conn,+ user: user,+ site: site+ } do+ ep = insert(:enterprise_plan, features: [Feature.StatsAPI], user_id: user.id)+ insert(:subscription, user: user, paddle_plan_id: ep.paddle_plan_id)++ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "metrics" => ["visitors"],+ "date_range" => "all",+ "dimensions" => ["event:props:path"]+ })++ assert json_response(conn, 200)["results"]+ end++ test "cannot filter by a custom prop without access to the props feature", %{+ conn: conn,+ user: user,+ site: site+ } do+ ep =+ insert(:enterprise_plan, features: [Feature.StatsAPI], user_id: user.id)++ insert(:subscription, user: user, paddle_plan_id: ep.paddle_plan_id)++ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "metrics" => ["visitors"],+ "date_range" => "all",+ "dimensions" => ["visit:source"],+ "filters" => [["is", "event:props:author", ["Uku"]]]+ })++ assert json_response(conn, 400)["error"] ==+ "The owner of this site does not have access to the custom properties feature"+ end++ test "can filter by an internal prop key without access to the props feature", %{+ conn: conn,+ user: user,+ site: site+ } do+ ep = insert(:enterprise_plan, features: [Feature.StatsAPI], user_id: user.id)+ insert(:subscription, user: user, paddle_plan_id: ep.paddle_plan_id)++ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "metrics" => ["visitors"],+ "date_range" => "all",+ "dimensions" => ["visit:source"],+ "filters" => [["is", "event:props:url", ["whatever"]]]+ })++ assert json_response(conn, 200)["results"]+ end+ end++ describe "param validation" do+ test "does not allow querying conversion_rate without a goal filter", %{+ conn: conn,+ site: site+ } do+ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "metrics" => ["conversion_rate"],+ "date_range" => "all",+ "dimensions" => ["event:page"],+ "filters" => [["is", "event:props:author", ["Uku"]]]+ })++ assert json_response(conn, 400)["error"] ==+ "Metric `conversion_rate` can only be queried with event:goal filters or dimensions"+ end++ test "validates that dimensions are valid", %{conn: conn, site: site} do+ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "metrics" => ["visitors"],+ "date_range" => "all",+ "dimensions" => ["badproperty"]+ })++ assert json_response(conn, 400)["error"] =~ "Invalid dimensions"+ end++ test "empty custom property is invalid", %{conn: conn, site: site} do+ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "metrics" => ["visitors"],+ "date_range" => "all",+ "dimensions" => ["event:props:"]+ })++ assert json_response(conn, 400)["error"] =~ "Invalid dimensions"+ end++ test "validates that correct date range is used", %{conn: conn, site: site} do+ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "metrics" => ["visitors"],+ "date_range" => "bad_period",+ "dimensions" => ["event:name"]+ })++ assert json_response(conn, 400)["error"] =~ "Invalid date range"+ end++ test "fails when an invalid metric is provided", %{conn: conn, site: site} do+ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "metrics" => ["visitors", "baa"],+ "date_range" => "all",+ "dimensions" => ["event:name"]+ })++ assert json_response(conn, 400)["error"] =~ "Unknown metric '\"baa\"'"+ end++ test "session metrics cannot be used with event:name property", %{conn: conn, site: site} do+ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "metrics" => ["visitors", "bounce_rate"],+ "date_range" => "all",+ "dimensions" => ["event:name"]+ })++ assert json_response(conn, 400)["error"] =~+ "Session metric(s) `bounce_rate` cannot be queried along with event filters or dimensions"+ end++ test "session metrics cannot be used with event:props:* property", %{conn: conn, site: site} do+ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "metrics" => ["visitors", "bounce_rate"],+ "date_range" => "all",+ "dimensions" => ["event:props:url"]+ })++ assert json_response(conn, 400)["error"] =~+ "Session metric(s) `bounce_rate` cannot be queried along with event filters or dimensions"+ end++ test "session metrics cannot be used with event:name filter", %{conn: conn, site: site} do+ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "metrics" => ["visitors", "bounce_rate"],+ "date_range" => "all",+ "dimensions" => ["visit:device"],+ "filters" => [+ ["is", "event:name", ["pageview"]]+ ]+ })++ assert json_response(conn, 400)["error"] =~+ "Session metric(s) `bounce_rate` cannot be queried along with event filters or dimensions"+ end+ end++ test "breakdown by visit:source", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:pageview,+ referrer_source: "Google",+ timestamp: ~N[2021-01-01 00:00:00]+ ),+ build(:pageview,+ referrer_source: "Google",+ timestamp: ~N[2021-01-01 00:25:00]+ ),+ build(:pageview,+ referrer_source: "",+ timestamp: ~N[2021-01-01 00:00:00]+ )+ ])++ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "metrics" => ["visitors"],+ "date_range" => "all",+ "dimensions" => ["visit:source"]+ })++ %{"results" => results} = json_response(conn, 200)++ assert results == [+ %{"dimensions" => ["Google"], "metrics" => [2]},+ %{"dimensions" => ["Direct / None"], "metrics" => [1]}+ ]+ end++ test "breakdown by visit:country", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:pageview, country_code: "EE", timestamp: ~N[2021-01-01 00:00:00]),+ build(:pageview, country_code: "EE", timestamp: ~N[2021-01-01 00:25:00]),+ build(:pageview, country_code: "US", timestamp: ~N[2021-01-01 00:00:00])+ ])++ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "metrics" => ["visitors"],+ "date_range" => "all",+ "dimensions" => ["visit:country"]+ })++ %{"results" => results} = json_response(conn, 200)++ assert results == [+ %{"dimensions" => ["EE"], "metrics" => [2]},+ %{"dimensions" => ["US"], "metrics" => [1]}+ ]+ end++ # test "breaks down all metrics by visit:referrer with imported data", %{conn: conn, site: site} do+ # site_import =+ # insert(:site_import,+ # site: site,+ # start_date: ~D[2005-01-01],+ # end_date: Timex.today(),+ # source: :universal_analytics+ # )++ # populate_stats(site, site_import.id, [+ # build(:pageview, referrer: "site.com", timestamp: ~N[2021-01-01 00:00:00]),+ # build(:pageview, referrer: "site.com/1", timestamp: ~N[2021-01-01 00:00:00]),+ # build(:imported_sources,+ # referrer: "site.com",+ # date: ~D[2021-01-01],+ # visitors: 2,+ # visits: 2,+ # pageviews: 2,+ # bounces: 1,+ # visit_duration: 120+ # ),+ # build(:imported_sources,+ # referrer: "site.com/2",+ # date: ~D[2021-01-01],+ # visitors: 2,+ # visits: 2,+ # pageviews: 2,+ # bounces: 2,+ # visit_duration: 0+ # ),+ # build(:imported_sources,+ # date: ~D[2021-01-01],+ # visitors: 10,+ # visits: 11,+ # pageviews: 50,+ # bounces: 0,+ # visit_duration: 1100+ # )+ # ])++ # conn =+ # get(conn, "/api/v1/stats/breakdown", %{+ # "site_id" => site.domain,+ # "period" => "day",+ # "metrics" => "visitors,visits,pageviews,bounce_rate,visit_duration",+ # "date" => "2021-01-01",+ # "property" => "visit:referrer",+ # "with_imported" => "true"+ # })++ # assert json_response(conn, 200) == %{+ # "results" => [+ # %{+ # "referrer" => "Direct / None",+ # "visitors" => 10,+ # "visits" => 11,+ # "pageviews" => 50,+ # "bounce_rate" => 0,+ # "visit_duration" => 100+ # },+ # %{+ # "referrer" => "site.com",+ # "visitors" => 3,+ # "visits" => 3,+ # "pageviews" => 3,+ # "bounce_rate" => 67.0,+ # "visit_duration" => 40+ # },+ # %{+ # "referrer" => "site.com/2",+ # "visitors" => 2,+ # "visits" => 2,+ # "pageviews" => 2,+ # "bounce_rate" => 100.0,+ # "visit_duration" => 0+ # },+ # %{+ # "referrer" => "site.com/1",+ # "visitors" => 1,+ # "visits" => 1,+ # "pageviews" => 1,+ # "bounce_rate" => 100.0,+ # "visit_duration" => 0+ # }+ # ]+ # }++ # conn =+ # post(conn, "/api/v2/query", %{+ # "site_id" => site.domain,+ # "metrics" => ["visitors", "visits", "pageviews", "bounce_rate", "visit_duration"],+ # "date_range" => "all",+ # "dimensions" => ["visit:referrer"],+ # })++ # %{"results" => results} = json_response(conn, 200)++ # assert results == [+ # %{"dimensions" => ["EE"], "metrics" => [2]},+ # %{"dimensions" => ["US"], "metrics" => [1]},+ # ]+ # end++ # for {property, attr} <- [+ # {"visit:utm_campaign", :utm_campaign},+ # {"visit:utm_source", :utm_source},+ # {"visit:utm_term", :utm_term},+ # {"visit:utm_content", :utm_content}+ # ] do+ # test "breakdown by #{property} when filtered by hostname", %{conn: conn, site: site} do+ # populate_stats(site, [+ # # session starts at two.example.com with utm_param=ad+ # build(+ # :pageview,+ # [+ # {unquote(attr), "ad"},+ # {:user_id, @user_id},+ # {:hostname, "two.example.com"},+ # {:timestamp, ~N[2021-01-01 00:00:00]}+ # ]+ # ),+ # # session continues on one.example.com without any utm_params+ # build(+ # :pageview,+ # [+ # {:user_id, @user_id},+ # {:hostname, "one.example.com"},+ # {:timestamp, ~N[2021-01-01 00:15:00]}+ # ]+ # )+ # ])++ # conn =+ # get(conn, "/api/v1/stats/breakdown", %{+ # "site_id" => site.domain,+ # "period" => "day",+ # "date" => "2021-01-01",+ # "filters" => "event:hostname==one.example.com",+ # "property" => unquote(property)+ # })++ # # nobody landed on one.example.com from utm_param=ad+ # assert json_response(conn, 200) == %{"results" => []}+ # end+ # end++ for {dimension, column, value1, value2, blank_value} <- [+ {"visit:source", :referrer_source, "Google", "Twitter", "Direct / None"},+ {"visit:referrer", :referrer, "example.com", "google.com", "Direct / None"},+ {"visit:utm_medium", :utm_medium, "Search", "social", "(not set)"},+ {"visit:utm_source", :utm_source, "Google", "Bing", "(not set)"},+ {"visit:utm_campaign", :utm_campaign, "ads", "profile", "(not set)"},+ {"visit:utm_content", :utm_content, "Content1", "blog2", "(not set)"},+ {"visit:utm_term", :utm_term, "Term1", "favicon", "(not set)"},+ {"visit:os", :operating_system, "Mac", "Windows", "(not set)"},+ {"visit:browser", :browser, "Chrome", "Safari", "(not set)"},+ {"visit:device", :screen_size, "Mobile", "Desktop", "(not set)"}+ ] do+ test "simple breakdown by #{dimension}", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:pageview, [+ {unquote(column), unquote(value1)},+ {:timestamp, ~N[2021-01-01 00:00:00]}+ ]),+ build(:pageview, [+ {unquote(column), unquote(value1)},+ {:timestamp, ~N[2021-01-01 00:25:00]}+ ]),+ build(:pageview, [+ {unquote(column), unquote(value1)},+ {:timestamp, ~N[2021-01-01 00:55:00]}+ ]),+ build(:pageview, [+ {unquote(column), unquote(value2)},+ {:timestamp, ~N[2021-01-01 01:00:00]}+ ]),+ build(:pageview, [+ {unquote(column), unquote(value2)},+ {:timestamp, ~N[2021-01-01 01:25:00]}+ ]),+ build(:pageview, [+ {unquote(column), ""},+ {:timestamp, ~N[2021-01-01 00:00:00]}+ ])+ ])++ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "metrics" => ["visitors"],+ "date_range" => ["2021-01-01", "2021-01-01"],+ "dimensions" => [unquote(dimension)]+ })++ %{"results" => results} = json_response(conn, 200)++ assert results == [+ %{"dimensions" => [unquote(value1)], "metrics" => [3]},+ %{"dimensions" => [unquote(value2)], "metrics" => [2]},+ %{"dimensions" => [unquote(blank_value)], "metrics" => [1]}+ ]+ end+ end++ test "breakdown by visit:os and visit:os_version", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:pageview, operating_system: "Mac", operating_system_version: "14"),+ build(:pageview, operating_system: "Mac", operating_system_version: "14"),+ build(:pageview, operating_system: "Mac", operating_system_version: "14"),+ build(:pageview, operating_system_version: "14"),+ build(:pageview,+ operating_system: "Windows",+ operating_system_version: "11"+ ),+ build(:pageview,+ operating_system: "Windows",+ operating_system_version: "11"+ )+ ])++ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "metrics" => ["visitors"],+ "date_range" => "all",+ "dimensions" => ["visit:os", "visit:os_version"]+ })++ %{"results" => results} = json_response(conn, 200)++ assert results == [+ %{"dimensions" => ["Mac", "14"], "metrics" => [3]},+ %{"dimensions" => ["Windows", "11"], "metrics" => [2]},+ %{"dimensions" => ["(not set)", "14"], "metrics" => [1]}+ ]+ end++ test "breakdown by visit:browser and visit:browser_version", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:pageview, browser: "Chrome", browser_version: "14"),+ build(:pageview, browser: "Chrome", browser_version: "14"),+ build(:pageview, browser: "Chrome", browser_version: "14"),+ build(:pageview, browser_version: "14"),+ build(:pageview,+ browser: "Firefox",+ browser_version: "11"+ ),+ build(:pageview,+ browser: "Firefox",+ browser_version: "11"+ )+ ])++ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "metrics" => ["visitors"],+ "date_range" => "all",+ "dimensions" => ["visit:browser", "visit:browser_version"]+ })++ %{"results" => results} = json_response(conn, 200)++ assert results == [+ %{"dimensions" => ["Chrome", "14"], "metrics" => [3]},+ %{"dimensions" => ["Firefox", "11"], "metrics" => [2]},+ %{"dimensions" => ["(not set)", "14"], "metrics" => [1]}+ ]+ end++ test "explicit order_by", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:pageview, operating_system: "Windows", browser: "Chrome"),+ build(:pageview, operating_system: "Windows", browser: "Firefox"),+ build(:pageview, operating_system: "Linux", browser: "Firefox"),+ build(:pageview, operating_system: "Linux", browser: "Firefox"),+ build(:pageview, operating_system: "Mac", browser: "Chrome")+ ])++ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "metrics" => ["visitors"],+ "date_range" => "all",+ "dimensions" => ["visit:os", "visit:browser"],+ "order_by" => [["visitors", "asc"], ["visit:browser", "desc"], ["visit:os", "asc"]]+ })++ %{"results" => results} = json_response(conn, 200)++ assert results == [+ %{"dimensions" => ["Windows", "Firefox"], "metrics" => [1]},+ %{"dimensions" => ["Mac", "Chrome"], "metrics" => [1]},+ %{"dimensions" => ["Windows", "Chrome"], "metrics" => [1]},+ %{"dimensions" => ["Linux", "Firefox"], "metrics" => [2]}+ ]+ end++ test "breaks down all metrics by visit:utm_source with imported data", %{conn: conn, site: site} do+ site_import =+ insert(:site_import,+ site: site,+ start_date: ~D[2005-01-01],+ end_date: Timex.today(),+ source: :universal_analytics+ )++ populate_stats(site, site_import.id, [+ build(:pageview, utm_source: "SomeUTMSource", timestamp: ~N[2021-01-01 00:00:00]),+ build(:pageview, utm_source: "SomeUTMSource-1", timestamp: ~N[2021-01-01 00:00:00]),+ build(:imported_sources,+ utm_source: "SomeUTMSource",+ date: ~D[2021-01-01],+ visitors: 2,+ visits: 2,+ pageviews: 2,+ bounces: 1,+ visit_duration: 120+ ),+ build(:imported_sources,+ utm_source: "SomeUTMSource-2",+ date: ~D[2021-01-01],+ visitors: 2,+ visits: 2,+ pageviews: 2,+ bounces: 2,+ visit_duration: 0+ ),+ build(:imported_sources,+ date: ~D[2021-01-01],+ visitors: 10,+ visits: 11,+ pageviews: 50,+ bounces: 0,+ visit_duration: 1100+ )+ ])++ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "metrics" => ["visitors", "visits", "pageviews", "bounce_rate", "visit_duration"],+ "date_range" => "all",+ "dimensions" => ["visit:utm_source"],+ "include" => %{"imports" => true}+ })++ %{"results" => results} = json_response(conn, 200)++ assert results == [+ %{"dimensions" => ["SomeUTMSource"], "metrics" => [3, 3, 3, 67.0, 40.0]},+ %{"dimensions" => ["SomeUTMSource-2"], "metrics" => [2, 2, 2, 100.0, 0.0]},+ %{"dimensions" => ["SomeUTMSource-1"], "metrics" => [1, 1, 1, 100.0, 0.0]}+ ]+ end++ # test "pageviews breakdown by event:page - imported data having pageviews=0 and visitors=n should be bypassed",+ # %{conn: conn, site: site} do+ # site_import =+ # insert(:site_import,+ # site: site,+ # start_date: ~D[2005-01-01],+ # end_date: Timex.today(),+ # source: :universal_analytics+ # )++ # populate_stats(site, site_import.id, [+ # build(:pageview, pathname: "/", timestamp: ~N[2021-01-01 00:00:00]),+ # build(:pageview, pathname: "/", timestamp: ~N[2021-01-01 00:25:00]),+ # build(:pageview,+ # pathname: "/plausible.io",+ # timestamp: ~N[2021-01-01 00:00:00]+ # ),+ # build(:imported_pages,+ # page: "/skip-me",+ # date: ~D[2021-01-01],+ # visitors: 1,+ # pageviews: 0+ # ),+ # build(:imported_pages,+ # page: "/include-me",+ # date: ~D[2021-01-01],+ # visitors: 1,+ # pageviews: 1+ # )+ # ])++ # conn =+ # get(conn, "/api/v1/stats/breakdown", %{+ # "site_id" => site.domain,+ # "period" => "day",+ # "date" => "2021-01-01",+ # "property" => "event:page",+ # "with_imported" => "true",+ # "metrics" => "pageviews"+ # })++ # assert json_response(conn, 200) == %{+ # "results" => [+ # %{"page" => "/", "pageviews" => 2},+ # %{"page" => "/plausible.io", "pageviews" => 1},+ # %{"page" => "/include-me", "pageviews" => 1}+ # ]+ # }+ # end++ # test "breakdown by event:page", %{conn: conn, site: site} do+ # populate_stats(site, [+ # build(:pageview, pathname: "/", timestamp: ~N[2021-01-01 00:00:00]),+ # build(:pageview, pathname: "/", timestamp: ~N[2021-01-01 00:25:00]),+ # build(:pageview,+ # pathname: "/plausible.io",+ # timestamp: ~N[2021-01-01 00:00:00]+ # )+ # ])++ # conn =+ # get(conn, "/api/v1/stats/breakdown", %{+ # "site_id" => site.domain,+ # "period" => "day",+ # "date" => "2021-01-01",+ # "property" => "event:page"+ # })++ # assert json_response(conn, 200) == %{+ # "results" => [+ # %{"page" => "/", "visitors" => 2},+ # %{"page" => "/plausible.io", "visitors" => 1}+ # ]+ # }+ # end++ # test "breakdown by event:page when there are no events in the second page", %{+ # conn: conn,+ # site: site+ # } do+ # populate_stats(site, [+ # build(:pageview, pathname: "/", timestamp: ~N[2021-01-01 00:00:00]),+ # build(:pageview, pathname: "/", timestamp: ~N[2021-01-01 00:25:00]),+ # build(:pageview,+ # pathname: "/plausible.io",+ # timestamp: ~N[2021-01-01 00:00:00]+ # )+ # ])++ # conn =+ # get(conn, "/api/v1/stats/breakdown", %{+ # "site_id" => site.domain,+ # "period" => "day",+ # "date" => "2021-01-01",+ # "property" => "event:page",+ # "metrics" => "visitors,bounce_rate",+ # "page" => 2,+ # "limit" => 2+ # })++ # assert json_response(conn, 200) == %{"results" => []}+ # end++ # test "attempting to breakdown by event:hostname returns an error", %{conn: conn, site: site} do+ # conn =+ # get(conn, "/api/v1/stats/breakdown", %{+ # "site_id" => site.domain,+ # "period" => "day",+ # "date" => "2021-01-01",+ # "property" => "event:hostname",+ # "with_imported" => "true"+ # })++ # assert %{+ # "error" => error+ # } = json_response(conn, 400)++ # assert error =~ "Property 'event:hostname' is currently not supported for breakdowns."+ # end++ describe "breakdown by visit:exit_page" do+ setup %{site: site} do+ site_import = insert(:site_import, site: site)++ populate_stats(site, site_import.id, [+ build(:pageview,+ pathname: "/a",+ timestamp: ~N[2021-01-01 00:00:00]+ ),+ build(:pageview,+ user_id: @user_id,+ pathname: "/a",+ timestamp: ~N[2021-01-01 00:25:00]+ ),+ build(:pageview,+ user_id: @user_id,+ pathname: "/b",+ timestamp: ~N[2021-01-01 00:35:00]+ ),+ build(:imported_exit_pages,+ exit_page: "/b",+ exits: 3,+ visitors: 2,+ pageviews: 5,+ date: ~D[2021-01-01]+ )+ ])+ end++ test "can query with visit:exit_page dimension", %{conn: conn, site: site} do+ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "metrics" => ["visits"],+ "date_range" => "all",+ "dimensions" => ["visit:exit_page"],+ "include" => %{"imports" => true}+ })++ %{"results" => results} = json_response(conn, 200)++ assert results == [+ %{"dimensions" => ["/b"], "metrics" => [4]},+ %{"dimensions" => ["/a"], "metrics" => [1]}+ ]+ end+ end++ describe "custom events" do+ test "can breakdown by event:name", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:event,+ name: "Signup",+ timestamp: ~N[2021-01-01 00:00:00]+ ),+ build(:event,+ name: "Signup",+ timestamp: ~N[2021-01-01 00:00:00]+ ),+ build(:pageview,+ timestamp: ~N[2021-01-01 00:25:00]+ )+ ])++ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "metrics" => ["visitors"],+ "date_range" => "all",+ "dimensions" => ["event:name"]+ })++ %{"results" => results} = json_response(conn, 200)++ assert results == [+ %{"dimensions" => ["Signup"], "metrics" => [2]},+ %{"dimensions" => ["pageview"], "metrics" => [1]}+ ]+ end++ test "can breakdown by event:name with visitors and events metrics", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:pageview,+ pathname: "/non-existing",+ user_id: @user_id,+ timestamp: ~N[2021-01-01 00:00:00]+ ),+ build(:event,+ name: "404",+ pathname: "/non-existing",+ user_id: @user_id,+ timestamp: ~N[2021-01-01 00:00:00]+ ),+ build(:pageview,+ pathname: "/non-existing",+ timestamp: ~N[2021-01-01 00:00:01]+ ),+ build(:pageview,+ pathname: "/non-existing",+ user_id: @user_id,+ timestamp: ~N[2021-01-01 00:00:02]+ ),+ build(:event,+ name: "404",+ pathname: "/non-existing",+ user_id: @user_id,+ timestamp: ~N[2021-01-01 00:00:02]+ ),+ build(:pageview,+ pathname: "/non-existing",+ user_id: @user_id,+ timestamp: ~N[2021-01-01 00:00:03]+ )+ ])++ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "metrics" => ["visitors", "events"],+ "date_range" => "all",+ "dimensions" => ["event:name"]+ })++ %{"results" => results} = json_response(conn, 200)++ assert results == [+ %{"dimensions" => ["pageview"], "metrics" => [2, 4]},+ %{"dimensions" => ["404"], "metrics" => [1, 2]}+ ]+ end++ test "can breakdown by event:name while filtering for something", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:event,+ name: "Signup",+ pathname: "/pageA",+ browser: "Chrome",+ timestamp: ~N[2021-01-01 00:00:00]+ ),+ build(:event,+ name: "Signup",+ pathname: "/pageA",+ browser: "Chrome",+ timestamp: ~N[2021-01-01 00:00:00]+ ),+ build(:event,+ name: "Signup",+ pathname: "/pageA",+ browser: "Safari",+ timestamp: ~N[2021-01-01 00:00:00]+ ),+ build(:event,+ name: "Signup",+ pathname: "/pageB",+ browser: "Chrome",+ timestamp: ~N[2021-01-01 00:00:00]+ ),+ build(:pageview,+ pathname: "/pageA",+ browser: "Chrome",+ timestamp: ~N[2021-01-01 00:25:00]+ )+ ])++ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "metrics" => ["visitors"],+ "date_range" => "all",+ "dimensions" => ["event:name"],+ "filters" => [+ ["is", "event:page", ["/pageA"]],+ ["is", "visit:browser", ["Chrome"]]+ ]+ })++ %{"results" => results} = json_response(conn, 200)++ assert results == [+ %{"dimensions" => ["Signup"], "metrics" => [2]},+ %{"dimensions" => ["pageview"], "metrics" => [1]}+ ]+ end++ test "can breakdown by a visit:property when filtering by event:name", %{+ conn: conn,+ site: site+ } do+ populate_stats(site, [+ build(:pageview,+ referrer_source: "Google",+ user_id: @user_id,+ timestamp: ~N[2021-01-01 00:00:00]+ ),+ build(:event,+ name: "Signup",+ user_id: @user_id,+ timestamp: ~N[2021-01-01 00:00:00]+ ),+ build(:pageview,+ referrer_source: "Twitter",+ timestamp: ~N[2021-01-01 00:25:00]+ )+ ])++ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "metrics" => ["visitors"],+ "date_range" => "all",+ "dimensions" => ["visit:source"],+ "filters" => [+ ["is", "event:name", ["Signup"]]+ ]+ })++ %{"results" => results} = json_response(conn, 200)++ assert results == [+ %{"dimensions" => ["Google"], "metrics" => [1]}+ ]+ end++ test "can breakdown by event:name when filtering by event:page", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:pageview,+ pathname: "/pageA",+ timestamp: ~N[2021-01-01 00:00:00]+ ),+ build(:pageview,+ pathname: "/pageA",+ timestamp: ~N[2021-01-01 00:00:00]+ ),+ build(:event,+ pathname: "/pageA",+ name: "Signup",+ timestamp: ~N[2021-01-01 00:00:00]+ ),+ build(:pageview,+ pathname: "/pageB",+ referrer_source: "Twitter",+ timestamp: ~N[2021-01-01 00:25:00]+ )+ ])++ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "metrics" => ["visitors"],+ "date_range" => "all",+ "dimensions" => ["event:name"],+ "filters" => [+ ["matches", "event:page", ["/pageA"]]+ ]+ })++ %{"results" => results} = json_response(conn, 200)++ assert results == [+ %{"dimensions" => ["pageview"], "metrics" => [2]},+ %{"dimensions" => ["Signup"], "metrics" => [1]}+ ]+ end++ test "can breakdown by event:page when filtering by event:name", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:event,+ name: "Signup",+ pathname: "/pageA",+ timestamp: ~N[2021-01-01 00:00:00]+ ),+ build(:event,+ name: "Signup",+ pathname: "/pageA",+ timestamp: ~N[2021-01-01 00:00:00]+ ),+ build(:event,+ name: "Signup",+ pathname: "/pageB",+ timestamp: ~N[2021-01-01 00:00:00]+ ),+ build(:pageview,+ pathname: "/pageB",+ referrer_source: "Twitter",+ timestamp: ~N[2021-01-01 00:25:00]+ )+ ])++ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "metrics" => ["visitors"],+ "date_range" => "all",+ "dimensions" => ["event:page"],+ "filters" => [+ ["is", "event:name", ["Signup"]]+ ]+ })++ %{"results" => results} = json_response(conn, 200)++ assert results == [+ %{"dimensions" => ["/pageA"], "metrics" => [2]},+ %{"dimensions" => ["/pageB"], "metrics" => [1]}+ ]+ end++ test "can filter event:page with a wildcard", %{+ conn: conn,+ site: site+ } do+ populate_stats(site, [+ build(:pageview, pathname: "/en/page1"),+ build(:pageview, pathname: "/en/page2"),+ build(:pageview, pathname: "/en/page2"),+ build(:pageview, pathname: "/pl/page1")+ ])++ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "metrics" => ["visitors"],+ "date_range" => "all",+ "dimensions" => ["event:page"],+ "filters" => [+ ["matches", "event:page", ["/en/**"]]+ ]+ })++ %{"results" => results} = json_response(conn, 200)++ assert results == [+ %{"dimensions" => ["/en/page2"], "metrics" => [2]},+ %{"dimensions" => ["/en/page1"], "metrics" => [1]}+ ]+ end++ test "can filter event:hostname with a wildcard", %{+ conn: conn,+ site: site+ } do+ populate_stats(site, [+ build(:pageview, hostname: "alice.example.com", pathname: "/a"),+ build(:pageview, hostname: "anna.example.com", pathname: "/a"),+ build(:pageview, hostname: "adam.example.com", pathname: "/a"),+ build(:pageview, hostname: "bob.example.com", pathname: "/b")+ ])++ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "metrics" => ["visitors"],+ "date_range" => "all",+ "dimensions" => ["event:page"],+ "filters" => [+ ["matches", "event:hostname", ["a*.example.com"]]+ ]+ })++ %{"results" => results} = json_response(conn, 200)++ assert results == [+ %{"dimensions" => ["/a"], "metrics" => [3]}+ ]+ end++ test "breakdown by custom event property", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:event,+ name: "Purchase",+ "meta.key": ["package"],+ "meta.value": ["business"],+ timestamp: ~N[2021-01-01 00:00:00]+ ),+ build(:event,+ name: "Purchase",+ "meta.key": ["package"],+ "meta.value": ["personal"],+ timestamp: ~N[2021-01-01 00:00:00]+ ),+ build(:event,+ name: "Purchase",+ "meta.key": ["package"],+ "meta.value": ["business"],+ timestamp: ~N[2021-01-01 00:25:00]+ ),+ build(:event,+ name: "Some other event",+ "meta.key": ["package"],+ "meta.value": ["business"],+ timestamp: ~N[2021-01-01 00:25:00]+ )+ ])++ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "metrics" => ["visitors"],+ "date_range" => "all",+ "dimensions" => ["event:props:package"],+ "filters" => [+ ["is", "event:name", ["Purchase"]]+ ]+ })++ %{"results" => results} = json_response(conn, 200)++ assert results == [+ %{"dimensions" => ["business"], "metrics" => [2]},+ %{"dimensions" => ["personal"], "metrics" => [1]}+ ]+ end++ test "breakdown by custom event property, with pageviews metric", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:pageview,+ "meta.key": ["package"],+ "meta.value": ["business"],+ timestamp: ~N[2021-01-01 00:00:00]+ ),+ build(:pageview,+ "meta.key": ["package"],+ "meta.value": ["personal"],+ timestamp: ~N[2021-01-01 00:00:00]+ ),+ build(:pageview,+ "meta.key": ["package"],+ "meta.value": ["business"],+ timestamp: ~N[2021-01-01 00:25:00]+ )+ ])++ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "metrics" => ["pageviews"],+ "date_range" => "all",+ "dimensions" => ["event:props:package"],+ "filters" => []+ })++ %{"results" => results} = json_response(conn, 200)++ assert results == [+ %{"dimensions" => ["business"], "metrics" => [2]},+ %{"dimensions" => ["personal"], "metrics" => [1]}+ ]+ end++ test "breakdown by custom event property, with (none)", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:event,+ name: "Purchase",+ "meta.key": ["cost"],+ "meta.value": ["16"],+ timestamp: ~N[2021-01-01 00:00:00]+ ),+ build(:event,+ name: "Purchase",+ "meta.key": ["cost"],+ "meta.value": ["16"],+ timestamp: ~N[2021-01-01 00:00:00]+ ),+ build(:event,+ name: "Purchase",+ "meta.key": ["cost"],+ "meta.value": ["16"],+ timestamp: ~N[2021-01-01 00:00:00]+ ),+ build(:event,+ name: "Purchase",+ timestamp: ~N[2021-01-01 00:25:00]+ ),+ build(:event,+ name: "Purchase",+ "meta.key": ["cost"],+ "meta.value": ["14"],+ timestamp: ~N[2021-01-01 00:25:00]+ ),+ build(:event,+ name: "Purchase",+ "meta.key": ["cost"],+ "meta.value": ["14"],+ timestamp: ~N[2021-01-01 00:26:00]+ )+ ])++ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "metrics" => ["visitors"],+ "date_range" => "all",+ "dimensions" => ["event:props:cost"],+ "filters" => [["is", "event:name", ["Purchase"]]]+ })++ %{"results" => results} = json_response(conn, 200)++ assert results == [+ %{"dimensions" => ["16"], "metrics" => [3]},+ %{"dimensions" => ["14"], "metrics" => [2]},+ %{"dimensions" => ["(none)"], "metrics" => [1]}+ ]+ end++ # test "breakdown by custom event property, limited", %{conn: conn, site: site} do+ # populate_stats(site, [+ # build(:event,+ # name: "Purchase",+ # "meta.key": ["cost"],+ # "meta.value": ["16"],+ # timestamp: ~N[2021-01-01 00:00:00]+ # ),+ # build(:event,+ # name: "Purchase",+ # "meta.key": ["cost"],+ # "meta.value": ["18"],+ # timestamp: ~N[2021-01-01 00:25:00]+ # ),+ # build(:event,+ # name: "Purchase",+ # "meta.key": ["cost"],+ # "meta.value": ["14"],+ # timestamp: ~N[2021-01-01 00:25:00]+ # ),+ # build(:event,+ # name: "Purchase",+ # "meta.key": ["cost"],+ # "meta.value": ["14"],+ # timestamp: ~N[2021-01-01 00:26:00]+ # )+ # ])++ # conn =+ # get(conn, "/api/v1/stats/breakdown", %{+ # "site_id" => site.domain,+ # "period" => "day",+ # "date" => "2021-01-01",+ # "property" => "event:props:cost",+ # "filters" => "event:name==Purchase",+ # "limit" => 2+ # })++ # assert json_response(conn, 200) == %{+ # "results" => [+ # %{"cost" => "14", "visitors" => 2},+ # %{"cost" => "16", "visitors" => 1}+ # ]+ # }+ # end++ # test "breakdown by custom event property, paginated", %{conn: conn, site: site} do+ # populate_stats(site, [+ # build(:event,+ # name: "Purchase",+ # "meta.key": ["cost"],+ # "meta.value": ["16"],+ # timestamp: ~N[2021-01-01 00:00:00]+ # ),+ # build(:event,+ # name: "Purchase",+ # "meta.key": ["cost"],+ # "meta.value": ["16"],+ # timestamp: ~N[2021-01-01 00:00:00]+ # ),+ # build(:event,+ # name: "Purchase",+ # "meta.key": ["cost"],+ # "meta.value": ["18"],+ # timestamp: ~N[2021-01-01 00:25:00]+ # ),+ # build(:event,+ # name: "Purchase",+ # "meta.key": ["cost"],+ # "meta.value": ["14"],+ # timestamp: ~N[2021-01-01 00:25:00]+ # ),+ # build(:event,+ # name: "Purchase",+ # "meta.key": ["cost"],+ # "meta.value": ["14"],+ # timestamp: ~N[2021-01-01 00:26:00]+ # )+ # ])++ # conn =+ # get(conn, "/api/v1/stats/breakdown", %{+ # "site_id" => site.domain,+ # "period" => "day",+ # "date" => "2021-01-01",+ # "property" => "event:props:cost",+ # "filters" => "event:name==Purchase",+ # "limit" => 2,+ # "page" => 2+ # })++ # assert json_response(conn, 200) == %{+ # "results" => [+ # %{"cost" => "18", "visitors" => 1}+ # ]+ # }+ # end+ end++ # describe "breakdown by event:goal" do+ # test "returns custom event goals and pageview goals", %{conn: conn, site: site} do+ # insert(:goal, %{site: site, event_name: "Purchase"})+ # insert(:goal, %{site: site, page_path: "/test"})++ # populate_stats(site, [+ # build(:pageview,+ # timestamp: ~N[2021-01-01 00:00:01],+ # pathname: "/test"+ # ),+ # build(:event,+ # name: "Purchase",+ # timestamp: ~N[2021-01-01 00:00:03]+ # ),+ # build(:event,+ # name: "Purchase",+ # timestamp: ~N[2021-01-01 00:00:03]+ # )+ # ])++ # conn =+ # get(conn, "/api/v1/stats/breakdown", %{+ # "site_id" => site.domain,+ # "period" => "day",+ # "date" => "2021-01-01",+ # "property" => "event:goal"+ # })++ # assert [+ # %{"goal" => "Purchase", "visitors" => 2},+ # %{"goal" => "Visit /test", "visitors" => 1}+ # ] = json_response(conn, 200)["results"]+ # end++ # test "returns pageview goals containing wildcards", %{conn: conn, site: site} do+ # insert(:goal, %{site: site, page_path: "/**/post"})+ # insert(:goal, %{site: site, page_path: "/blog**"})++ # populate_stats(site, [+ # build(:pageview, pathname: "/blog", user_id: @user_id),+ # build(:pageview, pathname: "/blog/post-1", user_id: @user_id),+ # build(:pageview, pathname: "/blog/post-2", user_id: @user_id),+ # build(:pageview, pathname: "/blog/something/post"),+ # build(:pageview, pathname: "/different/page/post")+ # ])++ # conn =+ # get(conn, "/api/v1/stats/breakdown", %{+ # "site_id" => site.domain,+ # "metrics" => "visitors,pageviews",+ # "property" => "event:goal"+ # })++ # assert [+ # %{"goal" => "Visit /**/post", "visitors" => 2, "pageviews" => 2},+ # %{"goal" => "Visit /blog**", "visitors" => 2, "pageviews" => 4}+ # ] = json_response(conn, 200)["results"]+ # end++ # test "does not return goals that are not configured for the site", %{conn: conn, site: site} do+ # populate_stats(site, [+ # build(:pageview, pathname: "/register"),+ # build(:event, name: "Signup")+ # ])++ # conn =+ # get(conn, "/api/v1/stats/breakdown", %{+ # "site_id" => site.domain,+ # "metrics" => "visitors,pageviews",+ # "property" => "event:goal"+ # })++ # assert [] = json_response(conn, 200)["results"]+ # end+ # end++ # describe "filtering" do+ test "event:goal filter returns 400 when goal not configured", %{conn: conn, site: site} do+ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "metrics" => ["visitors"],+ "date_range" => "all",+ "dimensions" => ["visit:browser"],+ "filters" => [+ ["is", "event:goal", ["Register"]]+ ]+ })++ assert %{"error" => msg} = json_response(conn, 400)+ assert msg =~ "The goal `Register` is not configured for this site. Find out how"+ end++ test "validates that filters are valid", %{conn: conn, site: site} do+ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "metrics" => ["visitors"],+ "date_range" => "all",+ "dimensions" => ["visit:browser"],+ "filters" => [+ ["is", "badproperty", ["bar"]]+ ]+ })++ assert %{"error" => msg} = json_response(conn, 400)+ assert msg =~ "Invalid filter"+ end++ test "event:page filter for breakdown by session props", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:pageview,+ pathname: "/ignore",+ browser: "Chrome",+ timestamp: ~N[2021-01-01 00:00:00]+ ),+ build(:pageview,+ pathname: "/plausible.io",+ browser: "Chrome",+ timestamp: ~N[2021-01-01 00:00:00]+ ),+ build(:pageview,+ pathname: "/plausible.io",+ browser: "Chrome",+ timestamp: ~N[2021-01-01 00:25:00]+ ),+ build(:pageview,+ browser: "Safari",+ pathname: "/plausible.io",+ timestamp: ~N[2021-01-01 00:00:00]+ )+ ])++ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "metrics" => ["visitors"],+ "date_range" => "all",+ "dimensions" => ["visit:browser"],+ "filters" => [+ ["is", "event:page", ["/plausible.io"]]+ ]+ })++ %{"results" => results} = json_response(conn, 200)++ assert results == [+ %{"dimensions" => ["Chrome"], "metrics" => [2]},+ %{"dimensions" => ["Safari"], "metrics" => [1]}+ ]+ end++ test "event:page filter shows sources of sessions that have visited that page", %{+ conn: conn,+ site: site+ } do+ populate_stats(site, [+ build(:pageview,+ pathname: "/",+ referrer_source: "Twitter",+ utm_medium: "Twitter",+ utm_source: "Twitter",+ utm_campaign: "Twitter",+ user_id: @user_id+ ),+ build(:pageview,+ pathname: "/plausible.io",+ user_id: @user_id+ ),+ build(:pageview,+ pathname: "/plausible.io",+ referrer_source: "Google",+ utm_medium: "Google",+ utm_source: "Google",+ utm_campaign: "Google"+ ),+ build(:pageview,+ pathname: "/plausible.io",+ referrer_source: "Google",+ utm_medium: "Google",+ utm_source: "Google",+ utm_campaign: "Google"+ )+ ])++ for dimension <- [+ "visit:source",+ "visit:utm_medium",+ "visit:utm_source",+ "visit:utm_campaign"+ ] do+ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "metrics" => ["visitors"],+ "date_range" => "all",+ "dimensions" => [dimension],+ "filters" => [+ ["is", "event:page", ["/plausible.io"]]+ ]+ })++ %{"results" => results} = json_response(conn, 200)++ assert results == [+ %{"dimensions" => ["Google"], "metrics" => [2]},+ %{"dimensions" => ["Twitter"], "metrics" => [1]}+ ]+ end+ end++ # test "top sources for a custom goal and filtered by hostname", %{conn: conn, site: site} do+ # populate_stats(site, [+ # build(:pageview,+ # hostname: "blog.example.com",+ # referrer_source: "Facebook",+ # user_id: @user_id+ # ),+ # build(:pageview,+ # hostname: "app.example.com",+ # pathname: "/register",+ # user_id: @user_id+ # ),+ # build(:event,+ # name: "Signup",+ # hostname: "app.example.com",+ # pathname: "/register",+ # user_id: @user_id+ # )+ # ])++ # conn =+ # post(conn, "/api/v2/query", %{+ # "site_id" => site.domain,+ # "metrics" => ["visitors"],+ # "date_range" => "all",+ # "dimensions" => ["visit:source"],+ # "filters" => [+ # ["is", "event:hostname", ["app.example.com"]]+ # ]+ # })++ # %{"results" => results} = json_response(conn, 200)++ # assert results == []+ # end++ # test "top sources for a custom goal and filtered by hostname (2)", %{conn: conn, site: site} do+ # populate_stats(site, [+ # build(:pageview,+ # hostname: "app.example.com",+ # referrer_source: "Facebook",+ # pathname: "/register",+ # user_id: @user_id+ # ),+ # build(:event,+ # name: "Signup",+ # hostname: "app.example.com",+ # pathname: "/register",+ # user_id: @user_id+ # )+ # ])++ # conn =+ # get(conn, "/api/v1/stats/breakdown", %{+ # "site_id" => site.domain,+ # "period" => "day",+ # "property" => "visit:source",+ # "filters" => "event:hostname==app.example.com"+ # })++ # assert json_response(conn, 200) == %{+ # "results" => [%{"source" => "Facebook", "visitors" => 1}]+ # }+ # end++ # test "event:page filter is interpreted as entry_page filter only for bounce_rate", %{+ # conn: conn,+ # site: site+ # } do+ # populate_stats(site, [+ # build(:pageview,+ # browser: "Chrome",+ # user_id: @user_id,+ # pathname: "/ignore",+ # timestamp: ~N[2021-01-01 00:00:00]+ # ),+ # build(:pageview,+ # browser: "Chrome",+ # user_id: @user_id,+ # pathname: "/plausible.io",+ # timestamp: ~N[2021-01-01 00:01:00]+ # ),+ # build(:pageview,+ # browser: "Chrome",+ # user_id: 456,+ # pathname: "/important-page",+ # timestamp: ~N[2021-01-01 00:00:00]+ # ),+ # build(:pageview,+ # browser: "Chrome",+ # user_id: 456,+ # pathname: "/",+ # timestamp: ~N[2021-01-01 00:01:00]+ # ),+ # build(:pageview,+ # browser: "Chrome",+ # pathname: "/plausible.io",+ # timestamp: ~N[2021-01-01 00:01:00]+ # )+ # ])++ # conn =+ # get(conn, "/api/v1/stats/breakdown", %{+ # "site_id" => site.domain,+ # "period" => "day",+ # "date" => "2021-01-01",+ # "metrics" => "visitors,bounce_rate",+ # "property" => "visit:browser",+ # "filters" => "event:page == /plausible.io|/important-page"+ # })++ # assert json_response(conn, 200) == %{+ # "results" => [+ # %{+ # "browser" => "Chrome",+ # "bounce_rate" => 50,+ # "visitors" => 3+ # }+ # ]+ # }+ # end++ test "event:goal pageview filter for breakdown by visit source", %{conn: conn, site: site} do+ insert(:goal, %{site: site, page_path: "/plausible.io"})++ populate_stats(site, [+ build(:pageview,+ referrer_source: "Bing",+ timestamp: ~N[2021-01-01 00:00:00]+ ),+ build(:pageview,+ referrer_source: "Google",+ user_id: @user_id,+ timestamp: ~N[2021-01-01 00:00:00]+ ),+ build(:pageview,+ pathname: "/plausible.io",+ user_id: @user_id,+ timestamp: ~N[2021-01-01 00:00:00]+ )+ ])++ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "metrics" => ["visitors"],+ "date_range" => "all",+ "dimensions" => ["visit:source"],+ "filters" => [+ ["is", "event:goal", ["Visit /plausible.io"]]+ ]+ })++ %{"results" => results} = json_response(conn, 200)++ assert results == [+ %{"dimensions" => ["Google"], "metrics" => [1]}+ ]+ end++ test "event:goal custom event filter for breakdown by visit source", %{conn: conn, site: site} do+ insert(:goal, %{site: site, event_name: "Register"})++ populate_stats(site, [+ build(:pageview,+ referrer_source: "Bing",+ timestamp: ~N[2021-01-01 00:00:00]+ ),+ build(:pageview,+ referrer_source: "Google",+ user_id: @user_id,+ timestamp: ~N[2021-01-01 00:00:00]+ ),+ build(:event,+ name: "Register",+ user_id: @user_id,+ timestamp: ~N[2021-01-01 00:00:00]+ )+ ])++ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "metrics" => ["visitors"],+ "date_range" => "all",+ "dimensions" => ["visit:source"],+ "filters" => [+ ["is", "event:goal", ["Register"]]+ ]+ })++ %{"results" => results} = json_response(conn, 200)++ assert results == [+ %{"dimensions" => ["Google"], "metrics" => [1]}+ ]+ end++ test "wildcard pageview goal filter for breakdown by event:page", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:pageview, pathname: "/en/register"),+ build(:pageview, pathname: "/en/register", user_id: @user_id),+ build(:pageview, pathname: "/en/register", user_id: @user_id),+ build(:pageview, pathname: "/123/it/register"),+ build(:pageview, pathname: "/should-not-appear")+ ])++ insert(:goal, %{site: site, page_path: "/**register"})++ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "metrics" => ["visitors", "pageviews"],+ "date_range" => "all",+ "dimensions" => ["event:page"],+ "filters" => [+ ["matches", "event:goal", ["Visit /**register"]]+ ]+ })++ %{"results" => results} = json_response(conn, 200)++ assert results == [+ %{"dimensions" => ["/en/register"], "metrics" => [2, 3]},+ %{"dimensions" => ["/123/it/register"], "metrics" => [1, 1]}+ ]+ end++ test "mixed multi-goal filter for breakdown by visit:country", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:pageview, country_code: "EE", pathname: "/en/register"),+ build(:event, country_code: "EE", name: "Signup", pathname: "/en/register"),+ build(:pageview, country_code: "US", pathname: "/123/it/register"),+ build(:pageview, country_code: "US", pathname: "/different")+ ])++ insert(:goal, %{site: site, page_path: "/**register"})+ insert(:goal, %{site: site, event_name: "Signup"})++ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "metrics" => ["visitors", "pageviews", "events"],+ "date_range" => "all",+ "dimensions" => ["visit:country"],+ "filters" => [+ ["matches", "event:goal", ["Signup", "Visit /**register"]]+ ]+ })++ %{"results" => results} = json_response(conn, 200)++ assert results == [+ %{"dimensions" => ["EE"], "metrics" => [2, 1, 2]},+ %{"dimensions" => ["US"], "metrics" => [1, 1, 1]}+ ]+ end++ test "event:goal custom event filter for breakdown by event page", %{conn: conn, site: site} do+ insert(:goal, %{site: site, event_name: "Register"})++ populate_stats(site, [+ build(:event,+ pathname: "/en/register",+ name: "Register"+ ),+ build(:event,+ pathname: "/en/register",+ name: "Register"+ ),+ build(:event,+ pathname: "/it/register",+ name: "Register"+ )+ ])++ conn =+ get(conn, "/api/v1/stats/breakdown", %{+ "site_id" => site.domain,+ "period" => "day",+ "property" => "event:page",+ "filters" => "event:goal == Register"+ })++ assert json_response(conn, 200) == %{+ "results" => [+ %{"page" => "/en/register", "visitors" => 2},+ %{"page" => "/it/register", "visitors" => 1}+ ]+ }++ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "metrics" => ["visitors"],+ "date_range" => "all",+ "dimensions" => ["event:page"],+ "filters" => [+ ["is", "event:goal", ["Register"]]+ ]+ })++ %{"results" => results} = json_response(conn, 200)++ assert results == [+ %{"dimensions" => ["/en/register"], "metrics" => [2]},+ %{"dimensions" => ["/it/register"], "metrics" => [1]}+ ]+ end++ test "IN filter for event:page", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:pageview,+ pathname: "/ignore",+ timestamp: ~N[2021-01-01 00:00:00]+ ),+ build(:pageview,+ pathname: "/plausible.io",+ timestamp: ~N[2021-01-01 00:00:00]+ ),+ build(:pageview,+ pathname: "/plausible.io",+ timestamp: ~N[2021-01-01 00:00:00]+ ),+ build(:pageview,+ pathname: "/important-page",+ timestamp: ~N[2021-01-01 00:00:00]+ )+ ])++ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "metrics" => ["visitors"],+ "date_range" => "all",+ "dimensions" => ["event:page"],+ "filters" => [+ ["is", "event:page", ["/plausible.io", "/important-page"]]+ ]+ })++ %{"results" => results} = json_response(conn, 200)++ assert results == [+ %{"dimensions" => ["/plausible.io"], "metrics" => [2]},+ %{"dimensions" => ["/important-page"], "metrics" => [1]}+ ]+ end++ test "IN filter for visit:browser", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:pageview,+ pathname: "/ignore",+ browser: "Firefox",+ timestamp: ~N[2021-01-01 00:00:00]+ ),+ build(:pageview,+ pathname: "/plausible.io",+ browser: "Chrome",+ timestamp: ~N[2021-01-01 00:00:00]+ ),+ build(:pageview,+ pathname: "/plausible.io",+ browser: "Safari",+ timestamp: ~N[2021-01-01 00:00:00]+ ),+ build(:pageview,+ pathname: "/important-page",+ browser: "Safari",+ timestamp: ~N[2021-01-01 00:00:00]+ )+ ])++ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "metrics" => ["visitors"],+ "date_range" => "all",+ "dimensions" => ["event:page"],+ "filters" => [+ ["is", "visit:browser", ["Chrome", "Safari"]]+ ]+ })++ %{"results" => results} = json_response(conn, 200)++ assert results == [+ %{"dimensions" => ["/plausible.io"], "metrics" => [2]},+ %{"dimensions" => ["/important-page"], "metrics" => [1]}+ ]+ end++ # test "IN filter for visit:entry_page", %{conn: conn, site: site} do+ # populate_stats(site, [+ # build(:pageview,+ # pathname: "/ignore",+ # timestamp: ~N[2021-01-01 00:00:00]+ # ),+ # build(:pageview,+ # pathname: "/plausible.io",+ # timestamp: ~N[2021-01-01 00:00:00]+ # ),+ # build(:pageview,+ # pathname: "/plausible.io",+ # timestamp: ~N[2021-01-01 00:00:00]+ # ),+ # build(:pageview,+ # pathname: "/important-page",+ # timestamp: ~N[2021-01-01 00:00:00]+ # )+ # ])++ # conn =+ # post(conn, "/api/v2/query", %{+ # "site_id" => site.domain,+ # "metrics" => ["bounce_rate"],+ # "date_range" => "all",+ # "dimensions" => ["event:page"],+ # "filters" => [+ # ["is", "event:page", ["/plausible.io", "/important-page"]]+ # ]+ # })++ # %{"results" => results} = json_response(conn, 200)++ # assert results == [+ # %{"dimensions" => ["plausible.io"], "metrics" => [100]},+ # %{"dimensions" => ["/important-page"], "metrics" => [100]},+ # ]+ # end++ test "IN filter for event:name", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:event,+ name: "Signup",+ timestamp: ~N[2021-01-01 00:00:00]+ ),+ build(:event,+ name: "Signup",+ timestamp: ~N[2021-01-01 00:00:00]+ ),+ build(:event,+ name: "Login",+ timestamp: ~N[2021-01-01 00:00:00]+ ),+ build(:event,+ name: "Irrelevant",+ timestamp: ~N[2021-01-01 00:00:00]+ )+ ])++ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "metrics" => ["visitors"],+ "date_range" => "all",+ "dimensions" => ["event:name"],+ "filters" => [+ ["is", "event:name", ["Signup", "Login"]]+ ]+ })++ %{"results" => results} = json_response(conn, 200)++ assert results == [+ %{"dimensions" => ["Signup"], "metrics" => [2]},+ %{"dimensions" => ["Login"], "metrics" => [1]}+ ]+ end++ test "IN filter for event:props:*", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:pageview,+ browser: "Chrome",+ "meta.key": ["browser"],+ "meta.value": ["Chrome"],+ timestamp: ~N[2021-01-01 00:00:00]+ ),+ build(:pageview,+ browser: "Chrome",+ "meta.key": ["browser"],+ "meta.value": ["Chrome"],+ timestamp: ~N[2021-01-01 00:00:00]+ ),+ build(:pageview,+ browser: "Safari",+ "meta.key": ["browser"],+ "meta.value": ["Safari"],+ timestamp: ~N[2021-01-01 00:00:00]+ ),+ build(:pageview,+ browser: "Firefox",+ "meta.key": ["browser"],+ "meta.value": ["Firefox"],+ timestamp: ~N[2021-01-01 00:00:00]+ )+ ])++ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "metrics" => ["visitors"],+ "date_range" => "all",+ "dimensions" => ["visit:browser"],+ "filters" => [+ ["is", "event:props:browser", ["Chrome", "Safari"]]+ ]+ })++ %{"results" => results} = json_response(conn, 200)++ assert results == [+ %{"dimensions" => ["Chrome"], "metrics" => [2]},+ %{"dimensions" => ["Safari"], "metrics" => [1]}+ ]+ end++ test "Multiple event:props:* filters", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:pageview,+ browser: "Chrome",+ "meta.key": ["browser"],+ "meta.value": ["Chrome"],+ timestamp: ~N[2021-01-01 00:00:00]+ ),+ build(:pageview,+ browser: "Chrome",+ "meta.key": ["browser", "prop"],+ "meta.value": ["Chrome", "xyz"],+ timestamp: ~N[2021-01-01 00:00:00]+ ),+ build(:pageview,+ browser: "Safari",+ "meta.key": ["browser", "prop"],+ "meta.value": ["Safari", "target_value"],+ timestamp: ~N[2021-01-01 00:00:00]+ ),+ build(:pageview,+ browser: "Firefox",+ "meta.key": ["browser", "prop"],+ "meta.value": ["Firefox", "target_value"],+ timestamp: ~N[2021-01-01 00:00:00]+ )+ ])++ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "metrics" => ["visitors"],+ "date_range" => "all",+ "dimensions" => ["visit:browser"],+ "filters" => [+ ["is", "event:props:browser", ["Chrome", "Safari"]],+ ["is", "event:props:prop", ["target_value"]]+ ]+ })++ %{"results" => results} = json_response(conn, 200)++ assert results == [+ %{"dimensions" => ["Safari"], "metrics" => [1]}+ ]+ end++ test "IN filter for event:props:* including (none) value", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:pageview,+ browser: "Chrome",+ "meta.key": ["browser"],+ "meta.value": ["Chrome"],+ timestamp: ~N[2021-01-01 00:00:00]+ ),+ build(:pageview,+ browser: "Chrome",+ "meta.key": ["browser"],+ "meta.value": ["Chrome"],+ timestamp: ~N[2021-01-01 00:00:00]+ ),+ build(:pageview,+ browser: "Safari",+ timestamp: ~N[2021-01-01 00:00:00]+ ),+ build(:pageview,+ browser: "Firefox",+ "meta.key": ["browser"],+ "meta.value": ["Firefox"],+ timestamp: ~N[2021-01-01 00:00:00]+ )+ ])++ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "metrics" => ["visitors"],+ "date_range" => "all",+ "dimensions" => ["visit:browser"],+ "filters" => [["is", "event:props:browser", ["Chrome", "(none)"]]]+ })++ %{"results" => results} = json_response(conn, 200)++ assert results == [+ %{"dimensions" => ["Chrome"], "metrics" => [2]},+ %{"dimensions" => ["Safari"], "metrics" => [1]}+ ]+ end++ test "can use a is_not filter", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:pageview, browser: "Chrome"),+ build(:pageview, browser: "Safari"),+ build(:pageview, browser: "Safari"),+ build(:pageview, browser: "Edge")+ ])++ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "metrics" => ["visitors"],+ "date_range" => "all",+ "dimensions" => ["visit:browser"],+ "filters" => [+ ["is_not", "visit:browser", ["Chrome"]]+ ]+ })++ %{"results" => results} = json_response(conn, 200)++ assert results == [+ %{"dimensions" => ["Safari"], "metrics" => [2]},+ %{"dimensions" => ["Edge"], "metrics" => [1]}+ ]+ end++ # describe "metrics" do+ # test "returns conversion_rate in an event:goal breakdown", %{conn: conn, site: site} do+ # populate_stats(site, [+ # build(:event, name: "Signup", user_id: 1),+ # build(:event, name: "Signup", user_id: 1),+ # build(:pageview, pathname: "/blog"),+ # build(:pageview, pathname: "/blog/post"),+ # build(:pageview)+ # ])++ # insert(:goal, %{site: site, event_name: "Signup"})+ # insert(:goal, %{site: site, page_path: "/blog**"})++ # conn =+ # get(conn, "/api/v1/stats/breakdown", %{+ # "site_id" => site.domain,+ # "period" => "day",+ # "property" => "event:goal",+ # "metrics" => "visitors,events,conversion_rate"+ # })++ # assert json_response(conn, 200) == %{+ # "results" => [+ # %{+ # "goal" => "Visit /blog**",+ # "visitors" => 2,+ # "events" => 2,+ # "conversion_rate" => 50+ # },+ # %{+ # "goal" => "Signup",+ # "visitors" => 1,+ # "events" => 2,+ # "conversion_rate" => 25+ # }+ # ]+ # }+ # end++ # test "returns conversion_rate alone in an event:goal breakdown", %{conn: conn, site: site} do+ # populate_stats(site, [+ # build(:event, name: "Signup", user_id: 1),+ # build(:pageview)+ # ])++ # insert(:goal, %{site: site, event_name: "Signup"})++ # conn =+ # get(conn, "/api/v1/stats/breakdown", %{+ # "site_id" => site.domain,+ # "period" => "day",+ # "property" => "event:goal",+ # "metrics" => "conversion_rate"+ # })++ # assert json_response(conn, 200) == %{+ # "results" => [+ # %{+ # "goal" => "Signup",+ # "conversion_rate" => 50+ # }+ # ]+ # }+ # end++ # test "returns conversion_rate in a goal filtered custom prop breakdown", %{+ # conn: conn,+ # site: site+ # } do+ # populate_stats(site, [+ # build(:pageview, pathname: "/blog/1", "meta.key": ["author"], "meta.value": ["Uku"]),+ # build(:pageview, pathname: "/blog/2", "meta.key": ["author"], "meta.value": ["Uku"]),+ # build(:pageview, pathname: "/blog/3", "meta.key": ["author"], "meta.value": ["Uku"]),+ # build(:pageview, pathname: "/blog/1", "meta.key": ["author"], "meta.value": ["Marko"]),+ # build(:pageview,+ # pathname: "/blog/2",+ # "meta.key": ["author"],+ # "meta.value": ["Marko"],+ # user_id: 1+ # ),+ # build(:pageview,+ # pathname: "/blog/3",+ # "meta.key": ["author"],+ # "meta.value": ["Marko"],+ # user_id: 1+ # ),+ # build(:pageview, pathname: "/blog"),+ # build(:pageview, "meta.key": ["author"], "meta.value": ["Marko"]),+ # build(:pageview)+ # ])++ # insert(:goal, %{site: site, page_path: "/blog**"})++ # conn =+ # get(conn, "/api/v1/stats/breakdown", %{+ # "site_id" => site.domain,+ # "period" => "day",+ # "property" => "event:props:author",+ # "filters" => "event:goal==Visit /blog**",+ # "metrics" => "visitors,events,conversion_rate"+ # })++ # assert json_response(conn, 200) == %{+ # "results" => [+ # %{+ # "author" => "Uku",+ # "visitors" => 3,+ # "events" => 3,+ # "conversion_rate" => 37.5+ # },+ # %{+ # "author" => "Marko",+ # "visitors" => 2,+ # "events" => 3,+ # "conversion_rate" => 25+ # },+ # %{+ # "author" => "(none)",+ # "visitors" => 1,+ # "events" => 1,+ # "conversion_rate" => 12.5+ # }+ # ]+ # }+ # end++ test "returns conversion_rate alone in a goal filtered custom prop breakdown", %{+ conn: conn,+ site: site+ } do+ populate_stats(site, [+ build(:pageview, pathname: "/blog/1", "meta.key": ["author"], "meta.value": ["Uku"]),+ build(:pageview)+ ])++ insert(:goal, %{site: site, page_path: "/blog**"})++ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "metrics" => ["conversion_rate"],+ "date_range" => "all",+ "dimensions" => ["event:props:author"],+ "filters" => [["matches", "event:goal", ["Visit /blog**"]]]+ })++ %{"results" => results} = json_response(conn, 200)++ assert results == [+ %{"dimensions" => ["Uku"], "metrics" => [50]}+ ]+ end++ # test "returns conversion_rate in a goal filtered event:page breakdown", %{+ # conn: conn,+ # site: site+ # } do+ # populate_stats(site, [+ # build(:event, pathname: "/en/register", name: "pageview"),+ # build(:event, pathname: "/en/register", name: "Signup"),+ # build(:event, pathname: "/en/register", name: "Signup"),+ # build(:event, pathname: "/it/register", name: "Signup", user_id: 1),+ # build(:event, pathname: "/it/register", name: "Signup", user_id: 1),+ # build(:event, pathname: "/it/register", name: "pageview")+ # ])++ # insert(:goal, %{site: site, event_name: "Signup"})++ # conn =+ # get(conn, "/api/v1/stats/breakdown", %{+ # "site_id" => site.domain,+ # "period" => "day",+ # "property" => "event:page",+ # "filters" => "event:goal==Signup",+ # "metrics" => "visitors,events,conversion_rate"+ # })++ # assert json_response(conn, 200) == %{+ # "results" => [+ # %{+ # "page" => "/en/register",+ # "visitors" => 2,+ # "events" => 2,+ # "conversion_rate" => 66.7+ # },+ # %{+ # "page" => "/it/register",+ # "visitors" => 1,+ # "events" => 2,+ # "conversion_rate" => 50+ # }+ # ]+ # }+ # end++ # test "returns conversion_rate alone in a goal filtered event:page breakdown", %{+ # conn: conn,+ # site: site+ # } do+ # populate_stats(site, [+ # build(:event, pathname: "/en/register", name: "pageview"),+ # build(:event, pathname: "/en/register", name: "Signup")+ # ])++ # insert(:goal, %{site: site, event_name: "Signup"})++ # conn =+ # get(conn, "/api/v1/stats/breakdown", %{+ # "site_id" => site.domain,+ # "period" => "day",+ # "property" => "event:page",+ # "filters" => "event:goal==Signup",+ # "metrics" => "conversion_rate"+ # })++ # assert json_response(conn, 200) == %{+ # "results" => [+ # %{+ # "page" => "/en/register",+ # "conversion_rate" => 50+ # }+ # ]+ # }+ # end++ # test "returns conversion_rate in a multi-goal filtered visit:screen_size breakdown", %{+ # conn: conn,+ # site: site+ # } do+ # populate_stats(site, [+ # build(:event, screen_size: "Mobile", name: "pageview"),+ # build(:event, screen_size: "Mobile", name: "AddToCart"),+ # build(:event, screen_size: "Mobile", name: "AddToCart"),+ # build(:event, screen_size: "Desktop", name: "AddToCart", user_id: 1),+ # build(:event, screen_size: "Desktop", name: "Purchase", user_id: 1),+ # build(:event, screen_size: "Desktop", name: "pageview")+ # ])++ # # Make sure that revenue goals are treated the same+ # # way as regular custom event goals+ # insert(:goal, %{site: site, event_name: "Purchase", currency: :EUR})+ # insert(:goal, %{site: site, event_name: "AddToCart"})++ # conn =+ # post(conn, "/api/v2/query", %{+ # "site_id" => site.domain,+ # "metrics" => ["visitors", "events", "conversion_rate"],+ # "date_range" => "all",+ # "dimensions" => ["visit:device"],+ # "filters" => [["is", "event:goal", ["AddToCart", "Purchase"]]]+ # })++ # %{"results" => results} = json_response(conn, 200)++ # assert results == [+ # %{"dimensions" => ["Mobile"], "metrics" => [2, 2, 66.7]},+ # %{"dimensions" => ["Desktop"], "metrics" => [1, 2, 50]},+ # ]+ # end++ test "returns conversion_rate alone in a goal filtered visit:screen_size breakdown", %{+ conn: conn,+ site: site+ } do+ populate_stats(site, [+ build(:event, screen_size: "Mobile", name: "pageview"),+ build(:event, screen_size: "Mobile", name: "AddToCart")+ ])++ insert(:goal, %{site: site, event_name: "AddToCart"})++ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "metrics" => ["conversion_rate"],+ "date_range" => "all",+ "dimensions" => ["visit:device"],+ "filters" => [["is", "event:goal", ["AddToCart"]]]+ })++ %{"results" => results} = json_response(conn, 200)++ assert results == [+ %{"dimensions" => ["Mobile"], "metrics" => [50]}+ ]+ end++ # test "returns conversion_rate for a browser_version breakdown with pagination limit", %{+ # site: site,+ # conn: conn+ # } do+ # populate_stats(site, [+ # build(:pageview, browser: "Firefox", browser_version: "110"),+ # build(:pageview, browser: "Firefox", browser_version: "110"),+ # build(:pageview, browser: "Chrome", browser_version: "110"),+ # build(:pageview, browser: "Chrome", browser_version: "110"),+ # build(:pageview, browser: "Avast Secure Browser", browser_version: "110"),+ # build(:pageview, browser: "Avast Secure Browser", browser_version: "110"),+ # build(:event, name: "Signup", browser: "Edge", browser_version: "110")+ # ])++ # insert(:goal, site: site, event_name: "Signup")++ # conn =+ # get(conn, "/api/v1/stats/breakdown", %{+ # "site_id" => site.domain,+ # "period" => "day",+ # "property" => "visit:browser_version",+ # "filters" => "event:goal==Signup",+ # "metrics" => "visitors,conversion_rate",+ # "page" => 1,+ # "limit" => 1+ # })++ # assert json_response(conn, 200) == %{+ # "results" => [+ # %{+ # "browser" => "Edge",+ # "browser_version" => "110",+ # "visitors" => 1,+ # "conversion_rate" => 100.0+ # }+ # ]+ # }+ # end++ test "all metrics for breakdown by visit prop", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:pageview,+ user_id: 1,+ referrer_source: "Google",+ timestamp: ~N[2021-01-01 00:00:00]+ ),+ build(:event,+ name: "signup",+ user_id: 1,+ referrer_source: "Google",+ timestamp: ~N[2021-01-01 00:05:00]+ ),+ build(:pageview,+ user_id: 1,+ referrer_source: "Google",+ timestamp: ~N[2021-01-01 00:10:00]+ ),+ build(:pageview,+ referrer_source: "Google",+ timestamp: ~N[2021-01-01 00:25:00]+ ),+ build(:pageview,+ referrer_source: "Twitter",+ timestamp: ~N[2021-01-01 00:00:00]+ )+ ])++ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "metrics" => [+ "visitors",+ "visits",+ "pageviews",+ "events",+ "bounce_rate",+ "visit_duration"+ ],+ "date_range" => "all",+ "dimensions" => ["visit:source"],+ "filters" => []+ })++ %{"results" => results} = json_response(conn, 200)++ assert results == [+ %{"dimensions" => ["Google"], "metrics" => [2, 2, 3, 4, 50, 300]},+ %{"dimensions" => ["Twitter"], "metrics" => [1, 1, 1, 1, 100, 0]}+ ]+ end++ test "metrics=bounce_rate does not add visits to the response", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:pageview,+ user_id: 1,+ pathname: "/entry-page-1",+ timestamp: ~N[2021-01-01 00:00:00]+ ),+ build(:pageview,+ user_id: 1,+ pathname: "/some-page",+ timestamp: ~N[2021-01-01 00:10:00]+ ),+ build(:pageview,+ user_id: 2,+ pathname: "/entry-page-2",+ referrer_source: "Google",+ timestamp: ~N[2021-01-01 00:05:00]+ )+ ])++ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "metrics" => ["bounce_rate"],+ "date_range" => "all",+ "dimensions" => ["visit:entry_page"],+ "order_by" => [["visit:entry_page", "asc"]]+ })++ %{"results" => results} = json_response(conn, 200)++ assert results == [+ %{"dimensions" => ["/entry-page-1"], "metrics" => [0]},+ %{"dimensions" => ["/entry-page-2"], "metrics" => [100]}+ ]+ end++ test "filter by custom event property", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:event,+ name: "Purchase",+ "meta.key": ["package"],+ "meta.value": ["business"],+ browser: "Chrome",+ timestamp: ~N[2021-01-01 00:00:00]+ ),+ build(:event,+ name: "Purchase",+ "meta.key": ["package"],+ "meta.value": ["business"],+ browser: "Safari",+ timestamp: ~N[2021-01-01 00:00:00]+ ),+ build(:event,+ name: "Purchase",+ "meta.key": ["package"],+ "meta.value": ["business"],+ browser: "Safari",+ timestamp: ~N[2021-01-01 00:25:00]+ ),+ build(:event,+ name: "Purchase",+ "meta.key": ["package"],+ "meta.value": ["personal"],+ browser: "IE",+ timestamp: ~N[2021-01-01 00:25:00]+ )+ ])++ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "metrics" => ["visitors"],+ "date_range" => "all",+ "dimensions" => ["visit:browser"],+ "filters" => [+ ["is", "event:name", ["Purchase"]],+ ["is", "event:props:package", ["business"]]+ ]+ })++ %{"results" => results} = json_response(conn, 200)++ assert results == [+ %{"dimensions" => ["Safari"], "metrics" => [2]},+ %{"dimensions" => ["Chrome"], "metrics" => [1]}+ ]+ end++ # test "all metrics for breakdown by event prop", %{conn: conn, site: site} do+ # populate_stats(site, [+ # build(:pageview,+ # user_id: 1,+ # pathname: "/",+ # timestamp: ~N[2021-01-01 00:00:00]+ # ),+ # build(:pageview,+ # user_id: 1,+ # pathname: "/plausible.io",+ # timestamp: ~N[2021-01-01 00:10:00]+ # ),+ # build(:pageview, pathname: "/", timestamp: ~N[2021-01-01 00:25:00]),+ # build(:pageview,+ # pathname: "/plausible.io",+ # timestamp: ~N[2021-01-01 00:00:00]+ # )+ # ])++ # conn =+ # post(conn, "/api/v2/query", %{+ # "site_id" => site.domain,+ # "metrics" => [+ # "visitors",+ # "visits",+ # "pageviews",+ # "events",+ # "bounce_rate",+ # "visit_duration"+ # ],+ # "date_range" => "all",+ # "dimensions" => ["event:page"],+ # })++ # %{"results" => results} = json_response(conn, 200)++ # assert results == [+ # %{"dimensions" => ["/"], "metrics" => [2, 2, 3, 4, 50, 300]},+ # %{"dimensions" => ["/plausible.io"], "metrics" => [1, 1, 1, 1, 100, 0]}+ # ]+ # end+ # end++ describe "imported data" do+ test "returns screen sizes breakdown when filtering by screen size", %{conn: conn, site: site} do+ site_import = insert(:site_import, site: site)++ populate_stats(site, site_import.id, [+ build(:pageview,+ timestamp: ~N[2021-01-01 00:00:01],+ screen_size: "Mobile"+ ),+ build(:imported_devices,+ device: "Mobile",+ visitors: 3,+ pageviews: 5,+ date: ~D[2021-01-01]+ )+ ])++ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "metrics" => ["visitors", "pageviews"],+ "date_range" => "all",+ "dimensions" => ["visit:device"],+ "filters" => [+ ["is", "visit:device", ["Mobile"]]+ ],+ "include" => %{"imports" => true}+ })++ %{"results" => results} = json_response(conn, 200)++ assert results == [%{"dimensions" => ["Mobile"], "metrics" => [4, 6]}]+ end++ # test "returns custom event goals and pageview goals", %{conn: conn, site: site} do+ # insert(:goal, site: site, event_name: "Purchase")+ # insert(:goal, site: site, page_path: "/test")++ # site_import = insert(:site_import, site: site)++ # populate_stats(site, site_import.id, [+ # build(:pageview,+ # timestamp: ~N[2021-01-01 00:00:01],+ # pathname: "/test"+ # ),+ # build(:event,+ # name: "Purchase",+ # timestamp: ~N[2021-01-01 00:00:03]+ # ),+ # build(:event,+ # name: "Purchase",+ # timestamp: ~N[2021-01-01 00:00:03]+ # ),+ # build(:imported_custom_events,+ # name: "Purchase",+ # visitors: 3,+ # events: 5,+ # date: ~D[2021-01-01]+ # ),+ # build(:imported_pages,+ # page: "/test",+ # visitors: 2,+ # pageviews: 2,+ # date: ~D[2021-01-01]+ # ),+ # build(:imported_visitors, visitors: 5, date: ~D[2021-01-01])+ # ])++ # conn =+ # get(conn, "/api/v1/stats/breakdown", %{+ # "site_id" => site.domain,+ # "period" => "day",+ # "date" => "2021-01-01",+ # "property" => "event:goal",+ # "metrics" => "visitors,events,pageviews,conversion_rate",+ # "with_imported" => "true"+ # })++ # assert [+ # %{+ # "goal" => "Purchase",+ # "visitors" => 5,+ # "events" => 7,+ # "pageviews" => 0,+ # "conversion_rate" => 62.5+ # },+ # %{+ # "goal" => "Visit /test",+ # "visitors" => 3,+ # "events" => 3,+ # "pageviews" => 3,+ # "conversion_rate" => 37.5+ # }+ # ] = json_response(conn, 200)["results"]+ # end++ test "pageviews are returned as events for breakdown reports other than custom events", %{+ conn: conn,+ site: site+ } do+ site_import = insert(:site_import, site: site)++ populate_stats(site, site_import.id, [+ build(:imported_browsers, browser: "Chrome", pageviews: 1, date: ~D[2021-01-01]),+ build(:imported_devices, device: "Desktop", pageviews: 1, date: ~D[2021-01-01]),+ build(:imported_entry_pages, entry_page: "/test", pageviews: 1, date: ~D[2021-01-01]),+ build(:imported_exit_pages, exit_page: "/test", pageviews: 1, date: ~D[2021-01-01]),+ build(:imported_locations, country: "EE", pageviews: 1, date: ~D[2021-01-01]),+ build(:imported_operating_systems,+ operating_system: "Mac",+ pageviews: 1,+ date: ~D[2021-01-01]+ ),+ build(:imported_pages, page: "/test", pageviews: 1, date: ~D[2021-01-01]),+ build(:imported_sources, source: "Google", pageviews: 1, date: ~D[2021-01-01])+ ])++ breakdown_and_first = fn dimension ->+ conn+ |> post("/api/v2/query", %{+ "site_id" => site.domain,+ "metrics" => ["events"],+ "date_range" => ["2021-01-01", "2021-01-01"],+ "dimensions" => [dimension],+ "include" => %{"imports" => true}+ })+ |> json_response(200)+ |> Map.get("results")+ |> List.first()+ end++ assert %{"dimensions" => ["Chrome"], "metrics" => [1]} =+ breakdown_and_first.("visit:browser")++ assert %{"dimensions" => ["Desktop"], "metrics" => [1]} =+ breakdown_and_first.("visit:device")++ # assert %{"dimensions" => ["/test"], "metrics" => [1]} = breakdown_and_first.("visit:entry_page")+ # assert %{"dimensions" => ["/test"], "metrics" => [1]} = breakdown_and_first.("visit:exit_page")+ assert %{"dimensions" => ["EE"], "metrics" => [1]} = breakdown_and_first.("visit:country")+ assert %{"dimensions" => ["Mac"], "metrics" => [1]} = breakdown_and_first.("visit:os")+ assert %{"dimensions" => ["/test"], "metrics" => [1]} = breakdown_and_first.("event:page")++ assert %{"dimensions" => ["Google"], "metrics" => [1]} =+ breakdown_and_first.("visit:source")+ end++ for goal_name <- Plausible.Imported.goals_with_url() do+ test "returns url breakdown for #{goal_name} goal", %{conn: conn, site: site} do+ insert(:goal, event_name: unquote(goal_name), site: site)+ site_import = insert(:site_import, site: site)++ populate_stats(site, site_import.id, [+ build(:event,+ name: unquote(goal_name),+ "meta.key": ["url"],+ "meta.value": ["https://one.com"]+ ),+ build(:imported_custom_events,+ name: unquote(goal_name),+ visitors: 2,+ events: 5,+ link_url: "https://one.com"+ ),+ build(:imported_custom_events,+ name: unquote(goal_name),+ visitors: 5,+ events: 10,+ link_url: "https://two.com"+ ),+ build(:imported_custom_events,+ name: "some goal",+ visitors: 5,+ events: 10+ ),+ build(:imported_visitors, visitors: 9)+ ])++ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "metrics" => ["visitors", "events"],+ # "metrics" => ["visitors", "events", "conversion_rate"],+ "date_range" => "all",+ "dimensions" => ["event:props:url"],+ "filters" => [+ ["is", "event:goal", [unquote(goal_name)]]+ ],+ "include" => %{"imports" => true}+ })++ assert json_response(conn, 200)["results"] == [+ %{"dimensions" => ["https://two.com"], "metrics" => [5, 10]},+ %{"dimensions" => ["https://one.com"], "metrics" => [3, 6]}+ ]++ refute json_response(conn, 200)["meta"]["warning"]+ end+ end++ for goal_name <- Plausible.Imported.goals_with_path() do+ test "returns path breakdown for #{goal_name} goal", %{conn: conn, site: site} do+ insert(:goal, event_name: unquote(goal_name), site: site)+ site_import = insert(:site_import, site: site)++ populate_stats(site, site_import.id, [+ build(:event,+ name: unquote(goal_name),+ "meta.key": ["path"],+ "meta.value": ["/one"]+ ),+ build(:imported_custom_events,+ name: unquote(goal_name),+ visitors: 2,+ events: 5,+ path: "/one"+ ),+ build(:imported_custom_events,+ name: unquote(goal_name),+ visitors: 5,+ events: 10,+ path: "/two"+ ),+ build(:imported_custom_events,+ name: "some goal",+ visitors: 5,+ events: 10+ ),+ build(:imported_visitors, visitors: 9)+ ])++ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "metrics" => ["visitors", "events"],+ # "metrics" => ["visitors", "events", "conversion_rate"],+ "date_range" => "all",+ "dimensions" => ["event:props:path"],+ "filters" => [+ ["is", "event:goal", [unquote(goal_name)]]+ ],+ "include" => %{"imports" => true}+ })++ assert json_response(conn, 200)["results"] == [+ %{"dimensions" => ["/two"], "metrics" => [5, 10]},+ %{"dimensions" => ["/one"], "metrics" => [3, 6]}+ ]++ refute json_response(conn, 200)["meta"]["warning"]+ end+ end++ test "adds a warning when query params are not supported for imported data", %{+ conn: conn,+ site: site+ } do+ site_import = insert(:site_import, site: site)++ insert(:goal, event_name: "Signup", site: site)++ populate_stats(site, site_import.id, [+ build(:event,+ name: "Signup",+ "meta.key": ["package"],+ "meta.value": ["large"]+ ),+ build(:imported_visitors, visitors: 9)+ ])++ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "metrics" => ["visitors"],+ "date_range" => "all",+ "dimensions" => ["event:props:package"],+ "filters" => [+ ["is", "event:goal", ["Signup"]]+ ],+ "include" => %{"imports" => true}+ })++ assert json_response(conn, 200)["results"] == [+ %{"dimensions" => ["large"], "metrics" => [1]}+ ]++ assert json_response(conn, 200)["meta"]["warning"] =~+ "Imported stats are not included in the results because query parameters are not supported."+ end++ test "does not add a warning when there are no site imports", %{conn: conn, site: site} do+ insert(:goal, event_name: "Signup", site: site)++ populate_stats(site, [+ build(:event,+ name: "Signup",+ "meta.key": ["package"],+ "meta.value": ["large"]+ )+ ])++ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "metrics" => ["visitors"],+ "date_range" => "all",+ "dimensions" => ["event:props:package"],+ "filters" => [+ ["is", "event:goal", ["Signup"]]+ ],+ "include" => %{"imports" => true}+ })++ refute json_response(conn, 200)["meta"]["warning"]+ end++ test "does not add a warning when import is out of queried date range", %{+ conn: conn,+ site: site+ } do+ site_import = insert(:site_import, site: site, end_date: Date.add(Date.utc_today(), -3))++ insert(:goal, event_name: "Signup", site: site)++ populate_stats(site, site_import.id, [+ build(:event,+ name: "Signup",+ "meta.key": ["package"],+ "meta.value": ["large"]+ ),+ build(:imported_visitors, visitors: 9)+ ])++ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "metrics" => ["visitors"],+ "date_range" => "day",+ "dimensions" => ["event:props:package"],+ "filters" => [+ ["is", "event:goal", ["Signup"]]+ ],+ "include" => %{"imports" => true}+ })++ refute json_response(conn, 200)["meta"]["warning"]+ end++ test "applies multiple filters if the properties belong to the same table", %{+ conn: conn,+ site: site+ } do+ site_import = insert(:site_import, site: site)++ populate_stats(site, site_import.id, [+ build(:imported_sources, source: "Google", utm_medium: "organic", utm_term: "one"),+ build(:imported_sources, source: "Twitter", utm_medium: "organic", utm_term: "two"),+ build(:imported_sources,+ source: "Facebook",+ utm_medium: "something_else",+ utm_term: "one"+ )+ ])++ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "metrics" => ["visitors"],+ "date_range" => "day",+ "dimensions" => ["visit:source"],+ "filters" => [+ ["is", "visit:utm_medium", ["organic"]],+ ["is", "visit:utm_term", ["one"]]+ ],+ "include" => %{"imports" => true}+ })++ assert json_response(conn, 200)["results"] == [+ %{"dimensions" => ["Google"], "metrics" => [1]}+ ]+ end++ test "ignores imported data if filtered property belongs to a different table than the breakdown property",+ %{+ conn: conn,+ site: site+ } do+ site_import = insert(:site_import, site: site)++ populate_stats(site, site_import.id, [+ build(:imported_sources, source: "Google"),+ build(:imported_devices, device: "Desktop")+ ])++ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "metrics" => ["visitors"],+ "date_range" => "day",+ "dimensions" => ["visit:source"],+ "filters" => [+ ["is", "visit:device", ["Desktop"]]+ ],+ "include" => %{"imports" => true}+ })++ assert %{+ "results" => [],+ "meta" => meta+ } = json_response(conn, 200)++ assert meta["warning"] =~ "Imported stats are not included in the results"+ end+ end+end