Skip to content

Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.

Refactor internal Query schema and introduce WhereBuilder (#4082)

Karl-Aksel Puulmannbaa99652f612
diff --git a/extra/lib/plausible/stats/goal/revenue.ex b/extra/lib/plausible/stats/goal/revenue.ex
index 7b6e3368f5..8c4115c0aa 100644
--- a/extra/lib/plausible/stats/goal/revenue.ex
+++ b/extra/lib/plausible/stats/goal/revenue.ex
@@ -4,6 +4,8 @@ defmodule Plausible.Stats.Goal.Revenue do
"""
import Ecto.Query
+ alias Plausible.Stats.Query
+
@revenue_metrics [:average_revenue, :total_revenue]
def revenue_metrics() do
@@ -37,10 +39,10 @@ defmodule Plausible.Stats.Goal.Revenue do
"""
def get_revenue_tracking_currency(site, query, metrics) do
goal_filters =
- case query.filters do
- %{"event:goal" => {:is, {_, goal_name}}} -> [goal_name]
- %{"event:goal" => {:member, list}} -> Enum.map(list, fn {_, goal_name} -> goal_name end)
- _any -> []
+ case Query.get_filter(query, "event:goal") do
+ [:is, "event:goal", {_, goal_name}] -> [goal_name]
+ [:member, "event:goal", list] -> Enum.map(list, fn {_, goal_name} -> goal_name end)
+ _ -> []
end
requested_revenue_metrics? = Enum.any?(metrics, &(&1 in @revenue_metrics))
diff --git a/lib/plausible/google/api.ex b/lib/plausible/google/api.ex
index 3fd63c87f7..763981aea9 100644
--- a/lib/plausible/google/api.ex
+++ b/lib/plausible/google/api.ex
@@ -74,16 +74,16 @@ defmodule Plausible.Google.API do
end
end
- def fetch_stats(site, %{filters: %{} = filters, date_range: date_range}, limit) do
+ def fetch_stats(site, query, limit) do
with {:ok, site} <- ensure_search_console_property(site),
{:ok, access_token} <- maybe_refresh_token(site.google_auth),
{:ok, search_console_filters} <-
- SearchConsole.Filters.transform(site.google_auth.property, filters),
+ SearchConsole.Filters.transform(site.google_auth.property, query.filters),
{:ok, stats} <-
HTTP.list_stats(
access_token,
site.google_auth.property,
- date_range,
+ query.date_range,
limit,
search_console_filters
) do
diff --git a/lib/plausible/google/search_console/filters.ex b/lib/plausible/google/search_console/filters.ex
index 4e0e00d875..484ed84390 100644
--- a/lib/plausible/google/search_console/filters.ex
+++ b/lib/plausible/google/search_console/filters.ex
@@ -3,12 +3,11 @@ defmodule Plausible.Google.SearchConsole.Filters do
import Plausible.Stats.Base, only: [page_regex: 1]
def transform(property, plausible_filters) do
- plausible_filters = Map.drop(plausible_filters, ["visit:source"])
-
search_console_filters =
Enum.reduce_while(plausible_filters, [], fn plausible_filter, search_console_filters ->
case transform_filter(property, plausible_filter) do
:unsupported -> {:halt, :unsupported_filters}
+ :ignore -> {:cont, search_console_filters}
search_console_filter -> {:cont, [search_console_filter | search_console_filters]}
end
end)
@@ -20,27 +19,27 @@ defmodule Plausible.Google.SearchConsole.Filters do
end
end
- defp transform_filter(property, {"event:page", filter}) do
- transform_filter(property, {"visit:entry_page", filter})
+ defp transform_filter(property, [op, "event:page" | rest]) do
+ transform_filter(property, [op, "visit:entry_page" | rest])
end
- defp transform_filter(property, {"visit:entry_page", {:is, page}}) when is_binary(page) do
+ defp transform_filter(property, [:is, "visit:entry_page", page]) when is_binary(page) do
%{dimension: "page", expression: property_url(property, page)}
end
- defp transform_filter(property, {"visit:entry_page", {:member, pages}}) when is_list(pages) do
+ defp transform_filter(property, [:member, "visit:entry_page", pages]) when is_list(pages) do
expression =
Enum.map_join(pages, "|", fn page -> property_url(property, Regex.escape(page)) end)
%{dimension: "page", operator: "includingRegex", expression: expression}
end
- defp transform_filter(property, {"visit:entry_page", {:matches, page}}) when is_binary(page) do
+ defp transform_filter(property, [:matches, "visit:entry_page", page]) when is_binary(page) do
page = page_regex(property_url(property, page))
%{dimension: "page", operator: "includingRegex", expression: page}
end
- defp transform_filter(property, {"visit:entry_page", {:matches_member, pages}})
+ defp transform_filter(property, [:matches_member, "visit:entry_page", pages])
when is_list(pages) do
expression =
Enum.map_join(pages, "|", fn page -> page_regex(property_url(property, page)) end)
@@ -48,25 +47,27 @@ defmodule Plausible.Google.SearchConsole.Filters do
%{dimension: "page", operator: "includingRegex", expression: expression}
end
- defp transform_filter(_property, {"visit:screen", {:is, device}}) when is_binary(device) do
+ defp transform_filter(_property, [:is, "visit:screen", device]) when is_binary(device) do
%{dimension: "device", expression: search_console_device(device)}
end
- defp transform_filter(_property, {"visit:screen", {:member, devices}}) when is_list(devices) do
+ defp transform_filter(_property, [:member, "visit:screen", devices]) when is_list(devices) do
expression = devices |> Enum.join("|")
%{dimension: "device", operator: "includingRegex", expression: expression}
end
- defp transform_filter(_property, {"visit:country", {:is, country}}) when is_binary(country) do
+ defp transform_filter(_property, [:is, "visit:country", country]) when is_binary(country) do
%{dimension: "country", expression: search_console_country(country)}
end
- defp transform_filter(_property, {"visit:country", {:member, countries}})
+ defp transform_filter(_property, [:member, "visit:country", countries])
when is_list(countries) do
expression = Enum.map_join(countries, "|", &search_console_country/1)
%{dimension: "country", operator: "includingRegex", expression: expression}
end
+ defp transform_filter(_, [_, "visit:source" | _rest]), do: :ignore
+
defp transform_filter(_, _filter), do: :unsupported
defp property_url("sc-domain:" <> domain, page), do: "https://" <> domain <> page
diff --git a/lib/plausible/stats/aggregate.ex b/lib/plausible/stats/aggregate.ex
index 991e80431e..73d33f0b24 100644
--- a/lib/plausible/stats/aggregate.ex
+++ b/lib/plausible/stats/aggregate.ex
@@ -70,10 +70,7 @@ defmodule Plausible.Stats.Aggregate do
defp neighbor_aggregate_time_on_page(site, query) do
q =
from(
- e in base_event_query(site, %Query{
- query
- | filters: Map.delete(query.filters, "event:page")
- }),
+ e in base_event_query(site, Query.remove_filters(query, ["event:page"])),
select: {
fragment("? as p", e.pathname),
fragment("? as t", e.timestamp),
@@ -86,32 +83,32 @@ defmodule Plausible.Stats.Aggregate do
where_param_idx = length(base_query_raw_params)
{where_clause, where_arg} =
- case query.filters["event:page"] do
- {:is, page} ->
+ case Query.get_filter(query, "event:page") do
+ [:is, _, page] ->
{"p = {$#{where_param_idx}:String}", page}
- {:is_not, page} ->
+ [:is_not, _, page] ->
{"p != {$#{where_param_idx}:String}", page}
- {:member, page} ->
+ [:member, _, page] ->
{"p IN {$#{where_param_idx}:Array(String)}", page}
- {:not_member, page} ->
+ [:not_member, _, page] ->
{"p NOT IN {$#{where_param_idx}:Array(String)}", page}
- {:matches, expr} ->
+ [:matches, _, expr] ->
regex = page_regex(expr)
{"match(p, {$#{where_param_idx}:String})", regex}
- {:matches_member, exprs} ->
+ [:matches_member, _, exprs] ->
page_regexes = Enum.map(exprs, &page_regex/1)
{"multiMatchAny(p, {$#{where_param_idx}:Array(String)})", page_regexes}
- {:not_matches_member, exprs} ->
+ [:not_matches_member, _, exprs] ->
page_regexes = Enum.map(exprs, &page_regex/1)
{"not(multiMatchAny(p, {$#{where_param_idx}:Array(String)}))", page_regexes}
- {:does_not_match, expr} ->
+ [:does_not_match, _, expr] ->
regex = page_regex(expr)
{"not(match(p, {$#{where_param_idx}:String}))", regex}
end
@@ -148,29 +145,29 @@ defmodule Plausible.Stats.Aggregate do
defp window_aggregate_time_on_page(site, query) do
windowed_pages_q =
- from e in base_event_query(site, %Query{
- query
- | filters: Map.delete(query.filters, "event:page")
- }),
- select: %{
- next_timestamp: over(fragment("leadInFrame(?)", e.timestamp), :event_horizon),
- next_pathname: over(fragment("leadInFrame(?)", e.pathname), :event_horizon),
- timestamp: e.timestamp,
- pathname: e.pathname,
- session_id: e.session_id
- },
- windows: [
- event_horizon: [
- partition_by: e.session_id,
- order_by: e.timestamp,
- frame: fragment("ROWS BETWEEN CURRENT ROW AND 1 FOLLOWING")
- ]
- ]
+ from e in base_event_query(site, Query.remove_filters(query, ["event:page"])),
+ select: %{
+ next_timestamp: over(fragment("leadInFrame(?)", e.timestamp), :event_horizon),
+ next_pathname: over(fragment("leadInFrame(?)", e.pathname), :event_horizon),
+ timestamp: e.timestamp,
+ pathname: e.pathname,
+ session_id: e.session_id
+ },
+ windows: [
+ event_horizon: [
+ partition_by: e.session_id,
+ order_by: e.timestamp,
+ frame: fragment("ROWS BETWEEN CURRENT ROW AND 1 FOLLOWING")
+ ]
+ ]
+
+ event_page_filter = Query.get_filter(query, "event:page")
timed_page_transitions_q =
from e in Ecto.Query.subquery(windowed_pages_q),
group_by: [e.pathname, e.next_pathname, e.session_id],
- where: ^Plausible.Stats.Base.dynamic_filter_condition(query, "event:page", :pathname),
+ where:
+ ^Plausible.Stats.Filters.WhereBuilder.build_condition(:pathname, event_page_filter),
where: e.next_timestamp != 0,
select: %{
pathname: e.pathname,
diff --git a/lib/plausible/stats/base.ex b/lib/plausible/stats/base.ex
index fe74fb426b..07d14a7871 100644
--- a/lib/plausible/stats/base.ex
+++ b/lib/plausible/stats/base.ex
@@ -7,9 +7,6 @@ defmodule Plausible.Stats.Base do
alias Plausible.Timezones
import Ecto.Query
- @no_ref "Direct / None"
- @not_set "(not set)"
-
@uniq_users_expression "toUInt64(round(uniq(?) * any(_sample_factor)))"
def base_event_query(site, query) do
@@ -30,180 +27,30 @@ defmodule Plausible.Stats.Base do
on: e.session_id == sq.session_id
)
else
- if query.experimental_reduced_joins? do
- events_q |> filter_by_visit_props(Filters.event_table_visit_props(), query)
- else
- events_q
- end
+ events_q
end
end
defp query_events(site, query) do
- {first_datetime, last_datetime} = utc_boundaries(query, site)
-
- q =
- from(
- e in "events_v2",
- where: e.site_id == ^site.id,
- where: e.timestamp >= ^first_datetime and e.timestamp < ^last_datetime
- )
+ q = from(e in "events_v2", where: ^Filters.WhereBuilder.build(:events, site, query))
on_ee do
q = Plausible.Stats.Sampling.add_query_hint(q, query)
end
- q =
- q
- |> where([e], ^dynamic_filter_condition(query, "event:page", :pathname))
- |> where([e], ^dynamic_filter_condition(query, "event:hostname", :hostname))
-
- q =
- case query.filters["event:name"] do
- {:is, name} ->
- from(e in q, where: e.name == ^name)
-
- {:member, list} ->
- from(e in q, where: e.name in ^list)
-
- nil ->
- q
- end
-
- q =
- case query.filters["event:goal"] do
- {:is, {:page, path}} ->
- from(e in q, where: e.pathname == ^path and e.name == "pageview")
-
- {:matches, {:page, expr}} ->
- regex = page_regex(expr)
-
- from(e in q,
- where: fragment("match(?, ?)", e.pathname, ^regex) and e.name == "pageview"
- )
-
- {:is, {:event, event}} ->
- from(e in q, where: e.name == ^event)
-
- {:member, clauses} ->
- {events, pages} = split_goals(clauses)
-
- from(e in q,
- where: (e.pathname in ^pages and e.name == "pageview") or e.name in ^events
- )
-
- {:matches_member, clauses} ->
- {events, pages} = split_goals(clauses, &page_regex/1)
-
- event_clause =
- if Enum.any?(events) do
- dynamic([x], fragment("multiMatchAny(?, ?)", x.name, ^events))
- else
- dynamic([x], false)
- end
-
- page_clause =
- if Enum.any?(pages) do
- dynamic(
- [x],
- fragment("multiMatchAny(?, ?)", x.pathname, ^pages) and x.name == "pageview"
- )
- else
- dynamic([x], false)
- end
-
- where_clause = dynamic([], ^event_clause or ^page_clause)
-
- from(e in q, where: ^where_clause)
-
- nil ->
- q
- end
-
- q =
- Enum.reduce(
- Query.get_all_filters_by_prefix(query, "event:props"),
- q,
- &filter_by_custom_prop/2
- )
-
q
end
def query_sessions(site, query) do
- {first_datetime, last_datetime} =
- utc_boundaries(query, site)
-
- q = from(s in "sessions_v2", where: s.site_id == ^site.id)
-
- sessions_q =
- if query.experimental_session_count? do
- from s in q, where: s.timestamp >= ^first_datetime and s.start < ^last_datetime
- else
- from s in q, where: s.start >= ^first_datetime and s.start < ^last_datetime
- end
+ q = from(s in "sessions_v2", where: ^Filters.WhereBuilder.build(:sessions, site, query))
on_ee do
- sessions_q = Plausible.Stats.Sampling.add_query_hint(sessions_q, query)
- end
-
- filter_by_entry_props(sessions_q, query)
- |> filter_by_visit_props(Filters.visit_props(), query)
- end
-
- defp filter_by_visit_props(q, visit_props, query) do
- Enum.reduce(visit_props, q, fn prop_name, sessions_q ->
- filter_key = "visit:" <> prop_name
- db_field = String.to_existing_atom(prop_name)
-
- from(s in sessions_q,
- where: ^dynamic_filter_condition(query, filter_key, db_field)
- )
- end)
- end
-
- def filter_by_entry_props(sessions_q, query) do
- case Query.get_filter_by_prefix(query, "visit:entry_props:") do
- nil ->
- sessions_q
-
- {"visit:entry_props:" <> prop_name, filter_value} ->
- apply_entry_prop_filter(sessions_q, prop_name, filter_value)
+ q = Plausible.Stats.Sampling.add_query_hint(q, query)
end
- end
-
- def apply_entry_prop_filter(sessions_q, prop_name, {:is, "(none)"}) do
- from(
- s in sessions_q,
- where: not has_key(s, :entry_meta, ^prop_name)
- )
- end
-
- def apply_entry_prop_filter(sessions_q, prop_name, {:is, value}) do
- from(
- s in sessions_q,
- where:
- has_key(s, :entry_meta, ^prop_name) and get_by_key(s, :entry_meta, ^prop_name) == ^value
- )
- end
- def apply_entry_prop_filter(sessions_q, prop_name, {:is_not, "(none)"}) do
- from(
- s in sessions_q,
- where: has_key(s, :entry_meta, ^prop_name)
- )
- end
-
- def apply_entry_prop_filter(sessions_q, prop_name, {:is_not, value}) do
- from(
- s in sessions_q,
- where:
- not has_key(s, :entry_meta, ^prop_name) or
- get_by_key(s, :entry_meta, ^prop_name) != ^value
- )
+ q
end
- def apply_entry_prop_filter(sessions_q, _, _), do: sessions_q
-
def select_event_metrics(metrics) do
metrics
|> Enum.map(&select_event_metric/1)
@@ -272,7 +119,9 @@ defmodule Plausible.Stats.Base do
end
defp select_session_metric(:bounce_rate, query) do
- condition = dynamic_filter_condition(query, "event:page", :entry_page)
+ # :TRICKY: If page is passed to query, we only count bounce rate where users _entered_ at page.
+ event_page_filter = Query.get_filter(query, "event:page")
+ condition = Filters.WhereBuilder.build_condition(:entry_page, event_page_filter)
%{
bounce_rate:
@@ -358,45 +207,6 @@ defmodule Plausible.Stats.Base do
defp select_session_metric(:percentage, _query), do: %{}
- def dynamic_filter_condition(query, filter_key, db_field) do
- case query && query.filters && query.filters[filter_key] do
- {:is, value} ->
- value = db_field_val(db_field, value)
- dynamic([x], field(x, ^db_field) == ^value)
-
- {:is_not, value} ->
- value = db_field_val(db_field, value)
- dynamic([x], field(x, ^db_field) != ^value)
-
- {:matches_member, glob_exprs} ->
- page_regexes = Enum.map(glob_exprs, &page_regex/1)
- dynamic([x], fragment("multiMatchAny(?, ?)", field(x, ^db_field), ^page_regexes))
-
- {:not_matches_member, glob_exprs} ->
- page_regexes = Enum.map(glob_exprs, &page_regex/1)
- dynamic([x], fragment("not(multiMatchAny(?, ?))", field(x, ^db_field), ^page_regexes))
-
- {:matches, glob_expr} ->
- regex = page_regex(glob_expr)
- dynamic([x], fragment("match(?, ?)", field(x, ^db_field), ^regex))
-
- {:does_not_match, glob_expr} ->
- regex = page_regex(glob_expr)
- dynamic([x], fragment("not(match(?, ?))", field(x, ^db_field), ^regex))
-
- {:member, list} ->
- list = Enum.map(list, &db_field_val(db_field, &1))
- dynamic([x], field(x, ^db_field) in ^list)
-
- {:not_member, list} ->
- list = Enum.map(list, &db_field_val(db_field, &1))
- dynamic([x], field(x, ^db_field) not in ^list)
-
- nil ->
- true
- end
- end
-
def filter_converted_sessions(db_query, site, query) do
if Query.has_event_filters?(query) do
converted_sessions =
@@ -416,16 +226,6 @@ defmodule Plausible.Stats.Base do
end
end
- defp db_field_val(:source, @no_ref), do: ""
- defp db_field_val(:referrer, @no_ref), do: ""
- defp db_field_val(:utm_medium, @no_ref), do: ""
- defp db_field_val(:utm_source, @no_ref), do: ""
- defp db_field_val(:utm_campaign, @no_ref), do: ""
- defp db_field_val(:utm_content, @no_ref), do: ""
- defp db_field_val(:utm_term, @no_ref), do: ""
- defp db_field_val(_, @not_set), do: ""
- defp db_field_val(_, val), do: val
-
defp beginning_of_time(candidate, native_stats_start_at) do
if Timex.after?(native_stats_start_at, candidate) do
native_stats_start_at
@@ -486,92 +286,6 @@ defmodule Plausible.Stats.Base do
"^#{escaped}$"
end
- defp split_goals(clauses, map_fn \\ &Function.identity/1) do
- groups =
- Enum.group_by(clauses, fn {goal_type, _v} -> goal_type end, fn {_k, val} -> map_fn.(val) end)
-
- {
- Map.get(groups, :event, []),
- Map.get(groups, :page, [])
- }
- end
-
- defp filter_by_custom_prop({"event:props:" <> prop_name, {:is, "(none)"}}, q) do
- from(
- e in q,
- where: not has_key(e, :meta, ^prop_name)
- )
- end
-
- defp filter_by_custom_prop({"event:props:" <> prop_name, {:is, value}}, q) do
- from(
- e in q,
- where: has_key(e, :meta, ^prop_name) and get_by_key(e, :meta, ^prop_name) == ^value
- )
- end
-
- defp filter_by_custom_prop({"event:props:" <> prop_name, {:is_not, "(none)"}}, q) do
- from(
- e in q,
- where: has_key(e, :meta, ^prop_name)
- )
- end
-
- defp filter_by_custom_prop({"event:props:" <> prop_name, {:is_not, value}}, q) do
- from(
- e in q,
- where: not has_key(e, :meta, ^prop_name) or get_by_key(e, :meta, ^prop_name) != ^value
- )
- end
-
- defp filter_by_custom_prop({"event:props:" <> prop_name, {:matches, value}}, q) do
- regex = page_regex(value)
-
- from(
- e in q,
- where:
- has_key(e, :meta, ^prop_name) and
- fragment("match(?, ?)", get_by_key(e, :meta, ^prop_name), ^regex)
- )
- end
-
- defp filter_by_custom_prop({"event:props:" <> prop_name, {:member, values}}, q) do
- none_value_included = Enum.member?(values, "(none)")
-
- from(
- e in q,
- where:
- (has_key(e, :meta, ^prop_name) and get_by_key(e, :meta, ^prop_name) in ^values) or
- (^none_value_included and not has_key(e, :meta, ^prop_name))
- )
- end
-
- defp filter_by_custom_prop({"event:props:" <> prop_name, {:not_member, values}}, q) do
- none_value_included = Enum.member?(values, "(none)")
-
- from(
- e in q,
- where:
- (has_key(e, :meta, ^prop_name) and
- get_by_key(e, :meta, ^prop_name) not in ^values) or
- (^none_value_included and
- has_key(e, :meta, ^prop_name) and
- get_by_key(e, :meta, ^prop_name) not in ^values) or
- (not (^none_value_included) and not has_key(e, :meta, ^prop_name))
- )
- end
-
- defp filter_by_custom_prop({"event:props:" <> prop_name, {:matches_member, clauses}}, q) do
- regexes = Enum.map(clauses, &page_regex/1)
-
- from(
- e in q,
- where:
- has_key(e, :meta, ^prop_name) and
- fragment("arrayExists(k -> match(?, k), ?)", get_by_key(e, :meta, ^prop_name), ^regexes)
- )
- end
-
defp total_visitors(site, query) do
base_event_query(site, query)
|> select([e], total_visitors: fragment(@uniq_users_expression, e.user_id))
@@ -615,7 +329,7 @@ defmodule Plausible.Stats.Base do
# filters.
def maybe_add_conversion_rate(q, site, query, metrics) do
if :conversion_rate in metrics do
- total_query = query |> Query.remove_event_filters([:goal, :props])
+ total_query = query |> Query.remove_filters(["event:goal", "event:props"])
# :TRICKY: Subquery is used due to event:goal breakdown above doing an UNION ALL
subquery(q)
diff --git a/lib/plausible/stats/breakdown.ex b/lib/plausible/stats/breakdown.ex
index 566da8a071..305d80838b 100644
--- a/lib/plausible/stats/breakdown.ex
+++ b/lib/plausible/stats/breakdown.ex
@@ -39,7 +39,7 @@ defmodule Plausible.Stats.Breakdown do
event_query = %Query{
query
- | filters: Map.put(query.filters, "event:name", {:member, events}),
+ | filters: query.filters ++ [[:member, "event:name", events]],
property: "event:name"
}
@@ -180,7 +180,7 @@ defmodule Plausible.Stats.Breakdown do
pages ->
query
- |> Query.put_filter("visit:entry_page", {:member, Enum.map(pages, & &1[:page])})
+ |> Query.put_filter([:member, "visit:entry_page", Enum.map(pages, & &1[:page])])
|> struct!(property: "visit:entry_page")
end
@@ -256,12 +256,12 @@ defmodule Plausible.Stats.Breakdown do
end
defp update_hostname(query, visit_prop) do
- case query.filters["event:hostname"] do
+ case Query.get_filter(query, "event:hostname") do
nil ->
query
- some ->
- Plausible.Stats.Query.put_filter(query, visit_prop, some)
+ [op, "event:hostname", value] ->
+ Plausible.Stats.Query.put_filter(query, [op, visit_prop, value])
end
end
@@ -353,7 +353,7 @@ defmodule Plausible.Stats.Breakdown do
import Ecto.Query
windowed_pages_q =
- from e in base_event_query(site, Query.remove_event_filters(query, [:page, :props])),
+ from e in base_event_query(site, Query.remove_filters(query, ["event:page", "event:props"])),
select: %{
next_timestamp: over(fragment("leadInFrame(?)", e.timestamp), :event_horizon),
next_pathname: over(fragment("leadInFrame(?)", e.pathname), :event_horizon),
@@ -719,7 +719,8 @@ defmodule Plausible.Stats.Breakdown do
metrics
) do
if :conversion_rate in metrics do
- breakdown_total_visitors_query = query |> Query.remove_event_filters([:goal, :props])
+ breakdown_total_visitors_query =
+ query |> Query.remove_filters(["event:goal", "event:props"])
breakdown_total_visitors_q =
breakdown_fn.(site, breakdown_total_visitors_query, [:visitors])
diff --git a/lib/plausible/stats/custom_props.ex b/lib/plausible/stats/custom_props.ex
index 8fe69b9d44..5c6df48eb4 100644
--- a/lib/plausible/stats/custom_props.ex
+++ b/lib/plausible/stats/custom_props.ex
@@ -9,7 +9,7 @@ defmodule Plausible.Stats.CustomProps do
def fetch_prop_names(site, query) do
case Query.get_filter_by_prefix(query, "event:props:") do
- {"event:props:" <> key, _} ->
+ [_op, "event:props:" <> key | _rest] ->
[key]
_ ->
diff --git a/lib/plausible/stats/email_report.ex b/lib/plausible/stats/email_report.ex
index e446e45326..63809e3af8 100644
--- a/lib/plausible/stats/email_report.ex
+++ b/lib/plausible/stats/email_report.ex
@@ -48,7 +48,7 @@ defmodule Plausible.Stats.EmailReport do
defp put_top_5_sources(stats, site, query) do
query =
query
- |> Query.put_filter("visit:source", {:is_not, "Direct / None"})
+ |> Query.put_filter([:is_not, "visit:source", "Direct / None"])
|> struct!(property: "visit:source")
sources = Stats.breakdown(site, query, [:visitors], {5, 1})
diff --git a/lib/plausible/stats/filter_suggestions.ex b/lib/plausible/stats/filter_suggestions.ex
index ee46392dae..6e38044f99 100644
--- a/lib/plausible/stats/filter_suggestions.ex
+++ b/lib/plausible/stats/filter_suggestions.ex
@@ -152,10 +152,10 @@ defmodule Plausible.Stats.FilterSuggestions do
def filter_suggestions(site, query, "prop_value", filter_search) do
filter_query = if filter_search == nil, do: "%", else: "%#{filter_search}%"
- {"event:props:" <> key, _filter} = Query.get_filter_by_prefix(query, "event:props")
+ [_op, "event:props:" <> key | _rest] = Query.get_filter_by_prefix(query, "event:props")
none_q =
- from(e in base_event_query(site, Query.remove_event_filters(query, [:props])),
+ from(e in base_event_query(site, Query.remove_filters(query, ["event:props"])),
select: "(none)",
where: not has_key(e, :meta, ^key),
limit: 1
diff --git a/lib/plausible/stats/filters/dashboard_filter_parser.ex b/lib/plausible/stats/filters/dashboard_filter_parser.ex
index e1f80811c1..e57d1d49cb 100644
--- a/lib/plausible/stats/filters/dashboard_filter_parser.ex
+++ b/lib/plausible/stats/filters/dashboard_filter_parser.ex
@@ -9,52 +9,76 @@ defmodule Plausible.Stats.Filters.DashboardFilterParser do
the internal React dashboard API
"""
def parse_and_prefix(filters_map) do
- Enum.reduce(filters_map, %{}, fn {name, val}, new_filters ->
+ Enum.flat_map(filters_map, fn {name, val} ->
cond do
name in Filters.visit_props() ->
- Map.put(new_filters, "visit:" <> name, filter_value(name, val))
+ [filter_value("visit:" <> name, val)]
name in Filters.event_props() ->
- Map.put(new_filters, "event:" <> name, filter_value(name, val))
+ [filter_value("event:" <> name, val)]
name == "props" ->
- put_parsed_props(new_filters, name, val)
+ parse_props(val)
true ->
- new_filters
+ []
end
end)
end
- @spec filter_value(String.t(), String.t()) :: {atom(), String.t() | [String.t()]}
def filter_value(key, val) do
{is_negated, val} = parse_negated_prefix(val)
{is_contains, val} = parse_contains_prefix(val)
is_list = list_expression?(val)
is_wildcard = String.contains?(key, ["page", "goal", "hostname"]) && wildcard_expression?(val)
val = if is_list, do: parse_member_list(val), else: remove_escape_chars(val)
- val = if key == "goal", do: wrap_goal_value(val), else: val
+ val = if key == "event:goal", do: wrap_goal_value(val), else: val
cond do
- is_negated && is_wildcard && is_list -> {:not_matches_member, val}
- is_negated && is_contains && is_list -> {:not_matches_member, Enum.map(val, &"**#{&1}**")}
- is_wildcard && is_list -> {:matches_member, val}
- is_negated && is_wildcard -> {:does_not_match, val}
- is_negated && is_list -> {:not_member, val}
- is_negated && is_contains -> {:does_not_match, "**" <> val <> "**"}
- is_contains && is_list -> {:matches_member, Enum.map(val, &"**#{&1}**")}
- is_wildcard && is_list -> {:matches_member, val}
- is_negated -> {:is_not, val}
- is_list -> {:member, val}
- is_contains -> {:matches, "**" <> val <> "**"}
- is_wildcard -> {:matches, val}
- true -> {:is, val}
+ is_negated && is_wildcard && is_list ->
+ [:not_matches_member, key, val]
+
+ is_negated && is_contains && is_list ->
+ [:not_matches_member, key, Enum.map(val, &"**#{&1}**")]
+
+ is_wildcard && is_list ->
+ [:matches_member, key, val]
+
+ is_negated && is_wildcard ->
+ [:does_not_match, key, val]
+
+ is_negated && is_list ->
+ [:not_member, key, val]
+
+ is_negated && is_contains ->
+ [:does_not_match, key, "**" <> val <> "**"]
+
+ is_contains && is_list ->
+ [:matches_member, key, Enum.map(val, &"**#{&1}**")]
+
+ is_wildcard && is_list ->
+ [:matches_member, key, val]
+
+ is_negated ->
+ [:is_not, key, val]
+
+ is_list ->
+ [:member, key, val]
+
+ is_contains ->
+ [:matches, key, "**" <> val <> "**"]
+
+ is_wildcard ->
+ [:matches, key, val]
+
+ true ->
+ [:is, key, val]
end
end
- defp put_parsed_props(new_filters, name, val) do
- Enum.reduce(val, new_filters, fn {prop_key, prop_val}, new_filters ->
- Map.put(new_filters, "event:props:" <> prop_key, filter_value(name, prop_val))
+ defp parse_props(val) do
+ Enum.map(val, fn {prop_key, prop_val} ->
+ filter_value("event:props:" <> prop_key, prop_val)
end)
end
diff --git a/lib/plausible/stats/filters/filters.ex b/lib/plausible/stats/filters/filters.ex
index 7323696864..3775e73435 100644
--- a/lib/plausible/stats/filters/filters.ex
+++ b/lib/plausible/stats/filters/filters.ex
@@ -56,24 +56,24 @@ defmodule Plausible.Stats.Filters do
### Examples:
iex> Filters.parse("{\\"page\\":\\"/blog/**\\"}")
- %{"event:page" => {:matches, "/blog/**"}}
+ [[:matches, "event:page", "/blog/**"]]
iex> Filters.parse("visit:browser!=Chrome")
- %{"visit:browser" => {:is_not, "Chrome"}}
+ [[:is_not, "visit:browser", "Chrome"]]
iex> Filters.parse(nil)
- %{}
+ []
"""
def parse(filters) when is_binary(filters) do
case Jason.decode(filters) do
{:ok, filters} when is_map(filters) -> DashboardFilterParser.parse_and_prefix(filters)
- {:ok, _} -> %{}
+ {:ok, _} -> []
{:error, err} -> StatsAPIFilterParser.parse_filter_expression(err.data)
end
end
def parse(filters) when is_map(filters), do: DashboardFilterParser.parse_and_prefix(filters)
- def parse(_), do: %{}
+ def parse(_), do: []
def without_prefix(property) do
property
diff --git a/lib/plausible/stats/filters/stats_api_filter_parser.ex b/lib/plausible/stats/filters/stats_api_filter_parser.ex
index b946e5950a..fc58168993 100644
--- a/lib/plausible/stats/filters/stats_api_filter_parser.ex
+++ b/lib/plausible/stats/filters/stats_api_filter_parser.ex
@@ -12,13 +12,12 @@ defmodule Plausible.Stats.Filters.StatsAPIFilterParser do
Enum.map(filters, &parse_single_filter/1)
|> Enum.reject(fn parsed -> parsed == :error end)
- |> Enum.into(%{})
end
defp parse_single_filter(str) do
case to_kv(str) do
["event:goal" = key, raw_value] ->
- {key, parse_goal_filter(raw_value)}
+ parse_goal_filter(key, raw_value)
[key, raw_value] ->
is_negated? = String.contains?(str, "!=")
@@ -28,11 +27,11 @@ defmodule Plausible.Stats.Filters.StatsAPIFilterParser do
final_value = remove_escape_chars(raw_value)
cond do
- is_wildcard? && is_negated? -> {key, {:does_not_match, raw_value}}
- is_wildcard? -> {key, {:matches, raw_value}}
- is_list? -> {key, {:member, parse_member_list(raw_value)}}
- is_negated? -> {key, {:is_not, final_value}}
- true -> {key, {:is, final_value}}
+ is_wildcard? && is_negated? -> [:does_not_match, key, raw_value]
+ is_wildcard? -> [:matches, key, raw_value]
+ is_list? -> [:member, key, parse_member_list(raw_value)]
+ is_negated? -> [:is_not, key, final_value]
+ true -> [:is, key, final_value]
end
|> reject_invalid_country_codes()
@@ -41,7 +40,7 @@ defmodule Plausible.Stats.Filters.StatsAPIFilterParser do
end
end
- defp reject_invalid_country_codes({"visit:country", {_, code_or_codes}} = filter) do
+ defp reject_invalid_country_codes([_op, "visit:country", code_or_codes] = filter) do
code_or_codes
|> List.wrap()
|> Enum.reduce_while(filter, fn
@@ -59,7 +58,7 @@ defmodule Plausible.Stats.Filters.StatsAPIFilterParser do
|> Enum.map(&String.trim/1)
end
- defp parse_goal_filter(value) do
+ defp parse_goal_filter(key, value) do
is_list? = list_expression?(value)
is_wildcard? = wildcard_expression?(value)
@@ -72,10 +71,10 @@ defmodule Plausible.Stats.Filters.StatsAPIFilterParser do
|> wrap_goal_value()
cond do
- is_list? && is_wildcard? -> {:matches_member, value}
- is_list? -> {:member, value}
- is_wildcard? -> {:matches, value}
- true -> {:is, value}
+ is_list? && is_wildcard? -> [:matches_member, key, value]
+ is_list? -> [:member, key, value]
+ is_wildcard? -> [:matches, key, value]
+ true -> [:is, key, value]
end
end
end
diff --git a/lib/plausible/stats/filters/where_builder.ex b/lib/plausible/stats/filters/where_builder.ex
new file mode 100644
index 0000000000..04f66c8363
--- /dev/null
+++ b/lib/plausible/stats/filters/where_builder.ex
@@ -0,0 +1,304 @@
+defmodule Plausible.Stats.Filters.WhereBuilder do
+ @moduledoc """
+ A module for building am ecto where clause of a query out of a query.
+ """
+
+ import Ecto.Query
+ import Plausible.Stats.Base, only: [page_regex: 1, utc_boundaries: 2]
+
+ alias Plausible.Stats.Query
+
+ use Plausible.Stats.Fragments
+
+ @sessions_only_visit_fields [
+ :entry_page,
+ :exit_page,
+ :entry_page_hostname,
+ :exit_page_hostname
+ ]
+
+ @doc "Builds WHERE clause for a given Query against sessions or events table"
+ def build(table, site, query) do
+ base_condition = filter_site_time_range(table, site, query)
+
+ query.filters
+ |> Enum.map(&add_filter(table, query, &1))
+ |> Enum.reduce(base_condition, fn condition, acc -> dynamic([], ^acc and ^condition) end)
+ end
+
+ @doc """
+ Builds WHERE clause condition based off of a filter and a custom column name
+ Used for special business logic cases
+
+ Accepts nil as the `filter` parameter, in which case the condition is a no-op (WHERE TRUE).
+ """
+ def build_condition(db_field, filter) do
+ if filter do
+ filter_field(db_field, filter)
+ else
+ true
+ end
+ end
+
+ defp filter_site_time_range(:events, site, query) do
+ {first_datetime, last_datetime} = utc_boundaries(query, site)
+
+ dynamic(
+ [e],
+ e.site_id == ^site.id and e.timestamp >= ^first_datetime and e.timestamp < ^last_datetime
+ )
+ end
+
+ defp filter_site_time_range(:sessions, site, %Query{experimental_session_count?: true} = query) do
+ {first_datetime, last_datetime} = utc_boundaries(query, site)
+
+ # Counts each _active_ session in time range even if they started before
+ dynamic(
+ [s],
+ s.site_id == ^site.id and s.timestamp >= ^first_datetime and s.start < ^last_datetime
+ )
+ end
+
+ defp filter_site_time_range(:sessions, site, query) do
+ {first_datetime, last_datetime} = utc_boundaries(query, site)
+
+ dynamic(
+ [s],
+ s.site_id == ^site.id and s.start >= ^first_datetime and s.start < ^last_datetime
+ )
+ end
+
+ defp add_filter(:events, _query, [:is, "event:name", name]) do
+ dynamic([e], e.name == ^name)
+ end
+
+ defp add_filter(:events, _query, [:member, "event:name", list]) do
+ dynamic([e], e.name in ^list)
+ end
+
+ defp add_filter(:events, _query, [:is, "event:goal", {:page, path}]) do
+ dynamic([e], e.pathname == ^path and e.name == "pageview")
+ end
+
+ defp add_filter(:events, _query, [:matches, "event:goal", {:page, expr}]) do
+ regex = page_regex(expr)
+
+ dynamic([e], fragment("match(?, ?)", e.pathname, ^regex) and e.name == "pageview")
+ end
+
+ defp add_filter(:events, _query, [:is, "event:goal", {:event, event}]) do
+ dynamic([e], e.name == ^event)
+ end
+
+ defp add_filter(:events, _query, [:member, "event:goal", clauses]) do
+ {events, pages} = split_goals(clauses)
+
+ dynamic([e], (e.pathname in ^pages and e.name == "pageview") or e.name in ^events)
+ end
+
+ defp add_filter(:events, _query, [:matches_member, "event:goal", clauses]) do
+ {events, pages} = split_goals(clauses, &page_regex/1)
+
+ event_clause =
+ if Enum.any?(events) do
+ dynamic([x], fragment("multiMatchAny(?, ?)", x.name, ^events))
+ else
+ dynamic([x], false)
+ end
+
+ page_clause =
+ if Enum.any?(pages) do
+ dynamic(
+ [x],
+ fragment("multiMatchAny(?, ?)", x.pathname, ^pages) and x.name == "pageview"
+ )
+ else
+ dynamic([x], false)
+ end
+
+ where_clause = dynamic([], ^event_clause or ^page_clause)
+
+ dynamic([e], ^where_clause)
+ end
+
+ defp add_filter(:events, _query, [_, "event:page" | _rest] = filter) do
+ filter_field(:pathname, filter)
+ end
+
+ defp add_filter(:events, _query, [_, "event:hostname" | _rest] = filter) do
+ filter_field(:hostname, filter)
+ end
+
+ defp add_filter(:events, _query, [_, "event:props:" <> prop_name | _rest] = filter) do
+ filter_custom_prop(prop_name, :meta, filter)
+ end
+
+ defp add_filter(:events, _query, [_, "visit:entry_props:" <> _prop_name | _rest]) do
+ true
+ end
+
+ defp add_filter(
+ :events,
+ %Query{experimental_reduced_joins?: true},
+ [_, "visit:" <> key | _rest] = filter
+ ) do
+ # Filter events query if experimental_reduced_joins? is true
+ field_name = String.to_existing_atom(key)
+
+ if Enum.member?(@sessions_only_visit_fields, field_name) do
+ true
+ else
+ filter_field(field_name, filter)
+ end
+ end
+
+ defp add_filter(:events, _query, [_, "visit:" <> _key | _rest]) do
+ true
+ end
+
+ defp add_filter(:sessions, _query, [_, "visit:entry_props:" <> prop_name | _rest] = filter) do
+ filter_custom_prop(prop_name, :entry_meta, filter)
+ end
+
+ defp add_filter(:sessions, _query, [_, "visit:" <> key | _rest] = filter) do
+ filter_field(String.to_existing_atom(key), filter)
+ end
+
+ defp add_filter(:sessions, _query, [_, "event:" <> _ | _rest]) do
+ # Cannot apply sessions filters directly on session query where clause.
+ true
+ end
+
+ defp filter_custom_prop(prop_name, column_name, [:is, _, "(none)"]) do
+ dynamic([t], not has_key(t, column_name, ^prop_name))
+ end
+
+ defp filter_custom_prop(prop_name, column_name, [:is, _, value]) do
+ dynamic(
+ [t],
+ has_key(t, column_name, ^prop_name) and get_by_key(t, column_name, ^prop_name) == ^value
+ )
+ end
+
+ defp filter_custom_prop(prop_name, column_name, [:is_not, _, "(none)"]) do
+ dynamic([t], has_key(t, column_name, ^prop_name))
+ end
+
+ defp filter_custom_prop(prop_name, column_name, [:is_not, _, value]) do
+ dynamic(
+ [t],
+ not has_key(t, column_name, ^prop_name) or get_by_key(t, column_name, ^prop_name) != ^value
+ )
+ end
+
+ defp filter_custom_prop(prop_name, column_name, [:matches, _, value]) do
+ regex = page_regex(value)
+
+ dynamic(
+ [t],
+ has_key(t, column_name, ^prop_name) and
+ fragment("match(?, ?)", get_by_key(t, column_name, ^prop_name), ^regex)
+ )
+ end
+
+ defp filter_custom_prop(prop_name, column_name, [:member, _, values]) do
+ none_value_included = Enum.member?(values, "(none)")
+
+ dynamic(
+ [t],
+ (has_key(t, column_name, ^prop_name) and get_by_key(t, column_name, ^prop_name) in ^values) or
+ (^none_value_included and not has_key(t, column_name, ^prop_name))
+ )
+ end
+
+ defp filter_custom_prop(prop_name, column_name, [:not_member, _, values]) do
+ none_value_included = Enum.member?(values, "(none)")
+
+ dynamic(
+ [t],
+ (has_key(t, column_name, ^prop_name) and
+ get_by_key(t, column_name, ^prop_name) not in ^values) or
+ (^none_value_included and
+ has_key(t, column_name, ^prop_name) and
+ get_by_key(t, column_name, ^prop_name) not in ^values) or
+ (not (^none_value_included) and not has_key(t, column_name, ^prop_name))
+ )
+ end
+
+ defp filter_custom_prop(prop_name, column_name, [:matches_member, _, clauses]) do
+ regexes = Enum.map(clauses, &page_regex/1)
+
+ dynamic(
+ [t],
+ has_key(t, column_name, ^prop_name) and
+ fragment(
+ "arrayExists(k -> match(?, k), ?)",
+ get_by_key(t, column_name, ^prop_name),
+ ^regexes
+ )
+ )
+ end
+
+ defp filter_field(db_field, [:is, _key, value]) do
+ value = db_field_val(db_field, value)
+ dynamic([x], field(x, ^db_field) == ^value)
+ end
+
+ defp filter_field(db_field, [:is_not, _key, value]) do
+ value = db_field_val(db_field, value)
+ dynamic([x], field(x, ^db_field) != ^value)
+ end
+
+ defp filter_field(db_field, [:matches_member, _key, glob_exprs]) do
+ page_regexes = Enum.map(glob_exprs, &page_regex/1)
+ dynamic([x], fragment("multiMatchAny(?, ?)", field(x, ^db_field), ^page_regexes))
+ end
+
+ defp filter_field(db_field, [:not_matches_member, _key, glob_exprs]) do
+ page_regexes = Enum.map(glob_exprs, &page_regex/1)
+ dynamic([x], fragment("not(multiMatchAny(?, ?))", field(x, ^db_field), ^page_regexes))
+ end
+
+ defp filter_field(db_field, [:matches, _key, glob_expr]) do
+ regex = page_regex(glob_expr)
+ dynamic([x], fragment("match(?, ?)", field(x, ^db_field), ^regex))
+ end
+
+ defp filter_field(db_field, [:does_not_match, _key, glob_expr]) do
+ regex = page_regex(glob_expr)
+ dynamic([x], fragment("not(match(?, ?))", field(x, ^db_field), ^regex))
+ end
+
+ defp filter_field(db_field, [:member, _key, list]) do
+ list = Enum.map(list, &db_field_val(db_field, &1))
+ dynamic([x], field(x, ^db_field) in ^list)
+ end
+
+ defp filter_field(db_field, [:not_member, _key, list]) do
+ list = Enum.map(list, &db_field_val(db_field, &1))
+ dynamic([x], field(x, ^db_field) not in ^list)
+ end
+
+ @no_ref "Direct / None"
+ @not_set "(not set)"
+
+ defp db_field_val(:source, @no_ref), do: ""
+ defp db_field_val(:referrer, @no_ref), do: ""
+ defp db_field_val(:utm_medium, @no_ref), do: ""
+ defp db_field_val(:utm_source, @no_ref), do: ""
+ defp db_field_val(:utm_campaign, @no_ref), do: ""
+ defp db_field_val(:utm_content, @no_ref), do: ""
+ defp db_field_val(:utm_term, @no_ref), do: ""
+ defp db_field_val(_, @not_set), do: ""
+ defp db_field_val(_, val), do: val
+
+ defp split_goals(clauses, map_fn \\ &Function.identity/1) do
+ groups =
+ Enum.group_by(clauses, fn {goal_type, _v} -> goal_type end, fn {_k, val} -> map_fn.(val) end)
+
+ {
+ Map.get(groups, :event, []),
+ Map.get(groups, :page, [])
+ }
+ end
+end
diff --git a/lib/plausible/stats/fragments.ex b/lib/plausible/stats/fragments.ex
index ae5ef4e44c..27f49839f5 100644
--- a/lib/plausible/stats/fragments.ex
+++ b/lib/plausible/stats/fragments.ex
@@ -110,7 +110,7 @@ defmodule Plausible.Stats.Fragments do
quote do
fragment(
"has(?, ?)",
- field(unquote(table), unquote(meta_key_column(meta_column))),
+ field(unquote(table), ^meta_key_column(unquote(meta_column))),
unquote(key)
)
end
@@ -130,18 +130,18 @@ defmodule Plausible.Stats.Fragments do
quote do
fragment(
"?[indexOf(?, ?)]",
- field(unquote(table), unquote(meta_value_column(meta_column))),
- field(unquote(table), unquote(meta_key_column(meta_column))),
+ field(unquote(table), ^meta_value_column(unquote(meta_column))),
+ field(unquote(table), ^meta_key_column(unquote(meta_column))),
unquote(key)
)
end
end
- defp meta_key_column(:meta), do: :"meta.key"
- defp meta_key_column(:entry_meta), do: :"entry_meta.key"
+ def meta_key_column(:meta), do: :"meta.key"
+ def meta_key_column(:entry_meta), do: :"entry_meta.key"
- defp meta_value_column(:meta), do: :"meta.value"
- defp meta_value_column(:entry_meta), do: :"entry_meta.value"
+ def meta_value_column(:meta), do: :"meta.value"
+ def meta_value_column(:entry_meta), do: :"entry_meta.value"
defmacro __using__(_) do
quote do
diff --git a/lib/plausible/stats/imported.ex b/lib/plausible/stats/imported.ex
index 9ab3dd703e..3c43c17f85 100644
--- a/lib/plausible/stats/imported.ex
+++ b/lib/plausible/stats/imported.ex
@@ -33,27 +33,29 @@ defmodule Plausible.Stats.Imported do
@imported_properties Map.keys(@property_to_table_mappings)
+ @goals_with_url ["Outbound Link: Click", "Cloaked Link: Click", "File Download"]
+
+ @doc """
+ Returns a boolean indicating whether the combination of filters and
+ breakdown property is possible to query from the imported tables.
+
+ Usually, when no filters are used, the imported schema supports the
+ query. There is one exception though - breakdown by a custom property.
+ We are currently importing only two custom properties - `url` and `path.
+ Both these properties can only be used with their special goal filter
+ (see `@goals_with_url` and `@goals_with_path`).
+ """
def schema_supports_query?(query) do
- filter_count = length(Map.keys(query.filters))
+ filter_count = length(query.filters)
case {filter_count, query.property} do
{0, "event:props:" <> _} -> false
{0, _} -> true
- {1, _} -> supports_single_filter?(query)
+ {1, "event:props:url"} -> has_special_goal_filter?(query, @goals_with_url)
{_, _} -> false
end
end
- defp supports_single_filter?(%Query{
- filters: %{"event:goal" => {:is, {:event, event}}},
- property: "event:props:url"
- })
- when event in ["Outbound Link: Click", "File Download"] do
- true
- end
-
- defp supports_single_filter?(_query), do: false
-
def merge_imported_timeseries(native_q, _, %Plausible.Stats.Query{include_imported: false}, _),
do: native_q
@@ -118,7 +120,7 @@ defmodule Plausible.Stats.Imported do
where: i.visitors > 0,
select: %{}
)
- |> maybe_apply_filter(query.filters, property, dim)
+ |> maybe_apply_filter(query, property, dim)
|> group_imported_by(dim)
|> select_imported_metrics(metrics)
@@ -214,23 +216,35 @@ defmodule Plausible.Stats.Imported do
)
end
- defp maybe_apply_filter(
- q,
- %{"event:goal" => {:is, {:event, event_name}}},
- "event:props:url",
- _dim
- )
- when event_name in ["Outbound Link: Click", "File Download"] do
- where(q, [i], i.name == ^event_name)
+ defp maybe_apply_filter(q, query, "event:props:url", _) do
+ if name = find_special_goal_filter(query, @goals_with_url) do
+ where(q, [i], i.name == ^name)
+ else
+ q
+ end
end
- defp maybe_apply_filter(q, filters, property, dim) do
- case filters[property] do
- {:member, list} -> where(q, [i], field(i, ^dim) in ^list)
+ defp maybe_apply_filter(q, query, property, dim) do
+ case Query.get_filter(query, property) do
+ [:member, _, list] -> where(q, [i], field(i, ^dim) in ^list)
_ -> q
end
end
+ defp has_special_goal_filter?(query, event_names) do
+ not is_nil(find_special_goal_filter(query, event_names))
+ end
+
+ defp find_special_goal_filter(query, event_names) do
+ case Query.get_filter(query, "event:goal") do
+ [:is, "event:goal", {:event, name}] ->
+ if name in event_names, do: name, else: nil
+
+ _ ->
+ nil
+ end
+ end
+
defp select_imported_metrics(q, []), do: q
defp select_imported_metrics(q, [:visitors | rest]) do
diff --git a/lib/plausible/stats/query.ex b/lib/plausible/stats/query.ex
index 5af5ede7f4..4fa8f4a94f 100644
--- a/lib/plausible/stats/query.ex
+++ b/lib/plausible/stats/query.ex
@@ -5,7 +5,7 @@ defmodule Plausible.Stats.Query do
interval: nil,
period: nil,
property: nil,
- filters: %{},
+ filters: [],
sample_threshold: 20_000_000,
imported_data_requested: false,
include_imported: false,
@@ -200,50 +200,36 @@ defmodule Plausible.Stats.Query do
struct!(query, filters: Filters.parse(params["filters"]))
end
- def put_filter(query, key, val) do
- parsed_val =
- if is_binary(val) do
- Filters.DashboardFilterParser.filter_value(key, val)
- else
- val
- end
-
+ def put_filter(query, filter) do
struct!(query,
- filters: Map.put(query.filters, key, parsed_val)
+ filters: query.filters ++ [filter]
)
end
- def remove_event_filters(query, opts) do
+ def remove_filters(query, prefixes) do
new_filters =
- Enum.filter(query.filters, fn {filter_key, _} ->
- cond do
- :page in opts && filter_key == "event:page" -> false
- :goal in opts && filter_key == "event:goal" -> false
- :props in opts && filter_key && String.starts_with?(filter_key, "event:props:") -> false
- true -> true
- end
+ Enum.reject(query.filters, fn [_, filter_key | _rest] ->
+ Enum.any?(prefixes, &String.starts_with?(filter_key, &1))
end)
- |> Enum.into(%{})
struct!(query, filters: new_filters)
end
def has_event_filters?(query) do
- Enum.any?(query.filters, fn
- {"event:" <> _, _} -> true
- _ -> false
+ Enum.any?(query.filters, fn [_op, prop | _rest] ->
+ String.starts_with?(prop, "event:")
end)
end
def get_filter_by_prefix(query, prefix) do
- Enum.find(query.filters, fn {prop, _value} ->
+ Enum.find(query.filters, fn [_op, prop | _rest] ->
String.starts_with?(prop, prefix)
end)
end
- def get_all_filters_by_prefix(query, prefix) do
- Enum.filter(query.filters, fn {prop, _value} ->
- String.starts_with?(prop, prefix)
+ def get_filter(query, name) do
+ Enum.find(query.filters, fn [_op, prop | _rest] ->
+ prop == name
end)
end
@@ -281,7 +267,12 @@ defmodule Plausible.Stats.Query do
@spec trace(%__MODULE__{}, [atom()]) :: %__MODULE__{}
def trace(%__MODULE__{} = query, metrics) do
- filter_keys = Map.keys(query.filters) |> Enum.sort() |> Enum.join(";")
+ filter_keys =
+ query.filters
+ |> Enum.map(fn [_op, prop | _rest] -> prop end)
+ |> Enum.sort()
+ |> Enum.join(";")
+
metrics = metrics |> Enum.sort() |> Enum.join(";")
Tracer.set_attributes([
diff --git a/lib/plausible/stats/table_decider.ex b/lib/plausible/stats/table_decider.ex
index ef976a0604..8b77ca5928 100644
--- a/lib/plausible/stats/table_decider.ex
+++ b/lib/plausible/stats/table_decider.ex
@@ -23,15 +23,15 @@ defmodule Plausible.Stats.TableDecider do
partition(metrics, query, &metric_partitioner/2)
# Treat breakdown property as yet another filter
- filters =
+ query =
if breakdown_property do
- Map.put(query.filters, breakdown_property, nil)
+ Query.put_filter(query, [:is, breakdown_property, []])
else
- query.filters
+ query
end
%{event: event_only_filters, session: session_only_filters} =
- partition(filters, query, &filters_partitioner/2)
+ partition(query.filters, query, &filters_partitioner/2)
cond do
# Only one table needs to be queried
@@ -83,16 +83,16 @@ defmodule Plausible.Stats.TableDecider do
defp metric_partitioner(_, _), do: :either
- defp filters_partitioner(_, {"event:" <> _, _}), do: :event
- defp filters_partitioner(_, {"visit:entry_page", _}), do: :session
- defp filters_partitioner(_, {"visit:entry_page_hostname", _}), do: :session
- defp filters_partitioner(_, {"visit:exit_page", _}), do: :session
- defp filters_partitioner(_, {"visit:exit_page_hostname", _}), do: :session
+ defp filters_partitioner(_, [_, "event:" <> _ | _rest]), do: :event
+ defp filters_partitioner(_, [_, "visit:entry_page" | _rest]), do: :session
+ defp filters_partitioner(_, [_, "visit:entry_page_hostname" | _rest]), do: :session
+ defp filters_partitioner(_, [_, "visit:exit_page" | _rest]), do: :session
+ defp filters_partitioner(_, [_, "visit:exit_page_hostname" | _rest]), do: :session
- defp filters_partitioner(%Query{experimental_reduced_joins?: true}, {"visit:" <> _, _}),
+ defp filters_partitioner(%Query{experimental_reduced_joins?: true}, [_, "visit:" <> _ | _rest]),
do: :either
- defp filters_partitioner(_, {"visit:" <> _, _}),
+ defp filters_partitioner(_, [_, "visit:" <> _ | _rest]),
do: :session
defp filters_partitioner(%Query{experimental_reduced_joins?: false}, {unknown, _}) do
diff --git a/lib/plausible/stats/timeseries.ex b/lib/plausible/stats/timeseries.ex
index 0ca145330b..08dfaf18bf 100644
--- a/lib/plausible/stats/timeseries.ex
+++ b/lib/plausible/stats/timeseries.ex
@@ -314,7 +314,7 @@ defmodule Plausible.Stats.Timeseries do
defp maybe_add_timeseries_conversion_rate(q, site, query, metrics) do
if :conversion_rate in metrics do
- totals_query = query |> Query.remove_event_filters([:goal, :props])
+ totals_query = query |> Query.remove_filters(["event:goal", "event:props"])
totals_timeseries_q =
from(e in base_event_query(site, totals_query),
diff --git a/lib/plausible_web/controllers/api/external_stats_controller.ex b/lib/plausible_web/controllers/api/external_stats_controller.ex
index 0623873993..a14abf6cd7 100644
--- a/lib/plausible_web/controllers/api/external_stats_controller.ex
+++ b/lib/plausible_web/controllers/api/external_stats_controller.ex
@@ -119,7 +119,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController do
{_, _, :all} ->
true
- {{"event:props:" <> prop, _}, _property, allowed_props} ->
+ {[_, "event:props:" <> prop | _], _property, allowed_props} ->
prop in allowed_props
{_filter, "event:props:" <> prop, allowed_props} ->
@@ -156,10 +156,10 @@ defmodule PlausibleWeb.Api.ExternalStatsController do
defp validate_metric("time_on_page" = metric, query) do
cond do
- query.filters["event:goal"] ->
+ Query.get_filter(query, "event:goal") ->
{:error, "Metric `#{metric}` cannot be queried when filtering by `event:goal`"}
- query.filters["event:name"] ->
+ Query.get_filter(query, "event:name") ->
{:error, "Metric `#{metric}` cannot be queried when filtering by `event:name`"}
query.property == "event:page" ->
@@ -169,7 +169,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController do
{:error,
"Metric `#{metric}` is not supported in breakdown queries (except `event:page` breakdown)"}
- query.filters["event:page"] ->
+ Query.get_filter(query, "event:page") ->
{:ok, metric}
true ->
@@ -183,7 +183,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController do
query.property == "event:goal" ->
{:ok, metric}
- query.filters["event:goal"] ->
+ Query.get_filter(query, "event:goal") ->
{:ok, metric}
true ->
@@ -198,7 +198,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController do
defp validate_metric("views_per_visit" = metric, query) do
cond do
- query.filters["event:page"] ->
+ Query.get_filter(query, "event:page") ->
{:error, "Metric `#{metric}` cannot be queried with a filter on `event:page`."}
query.property != nil ->
@@ -235,7 +235,9 @@ defmodule PlausibleWeb.Api.ExternalStatsController do
end
defp find_event_only_filter(query) do
- Map.keys(query.filters) |> Enum.find(&event_only_property?/1)
+ query.filters
+ |> Enum.map(fn [_op, prop | _] -> prop end)
+ |> Enum.find(&event_only_property?/1)
end
defp event_only_property?("event:name"), do: true
@@ -327,7 +329,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController do
end)
end
- defp validate_filter(site, {"event:goal", {_type, goal_filter}}) do
+ defp validate_filter(site, [_type, "event:goal", goal_filter]) do
configured_goals =
Plausible.Goals.for_site(site)
|> Enum.map(fn
@@ -350,7 +352,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController do
end
end
- defp validate_filter(_site, {property, _}) do
+ defp validate_filter(_site, [_, property | _]) do
if Plausible.Stats.Props.valid_prop?(property) do
:ok
else
diff --git a/lib/plausible_web/controllers/api/stats_controller.ex b/lib/plausible_web/controllers/api/stats_controller.ex
index dc2a562cdd..bc9086ca84 100644
--- a/lib/plausible_web/controllers/api/stats_controller.ex
+++ b/lib/plausible_web/controllers/api/stats_controller.ex
@@ -6,6 +6,7 @@ defmodule PlausibleWeb.Api.StatsController do
alias Plausible.Stats
alias Plausible.Stats.{Query, Comparisons}
+ alias Plausible.Stats.Filters.DashboardFilterParser
alias PlausibleWeb.Api.Helpers, as: H
require Logger
@@ -276,11 +277,25 @@ defmodule PlausibleWeb.Api.StatsController do
end
end
- defp fetch_top_stats(
- site,
- %Query{period: "realtime", filters: %{"event:goal" => _goal}} = query,
- _comparison_query
- ) do
+ defp fetch_top_stats(site, query, comparison_query) do
+ goal_filter = Query.get_filter(query, "event:goal")
+
+ cond do
+ query.period == "realtime" && goal_filter ->
+ fetch_goal_realtime_top_stats(site, query, comparison_query)
+
+ query.period == "realtime" ->
+ fetch_realtime_top_stats(site, query, comparison_query)
+
+ goal_filter ->
+ fetch_goal_top_stats(site, query, comparison_query)
+
+ true ->
+ fetch_other_top_stats(site, query, comparison_query)
+ end
+ end
+
+ defp fetch_goal_realtime_top_stats(site, query, _comparison_query) do
query_30m = %Query{query | period: "30m"}
%{
@@ -308,7 +323,7 @@ defmodule PlausibleWeb.Api.StatsController do
{stats, 100}
end
- defp fetch_top_stats(site, %Query{period: "realtime"} = query, _comparison_query) do
+ defp fetch_realtime_top_stats(site, query, _comparison_query) do
query_30m = %Query{query | period: "30m"}
%{
@@ -336,7 +351,7 @@ defmodule PlausibleWeb.Api.StatsController do
{stats, 100}
end
- defp fetch_top_stats(site, %Query{filters: %{"event:goal" => _}} = query, comparison_query) do
+ defp fetch_goal_top_stats(site, query, comparison_query) do
metrics =
[:total_visitors, :visitors, :events, :conversion_rate] ++ @revenue_metrics
@@ -365,9 +380,9 @@ defmodule PlausibleWeb.Api.StatsController do
|> then(&{&1, 100})
end
- defp fetch_top_stats(site, query, comparison_query) do
+ defp fetch_other_top_stats(site, query, comparison_query) do
metrics =
- if query.filters["event:page"] do
+ if Query.get_filter(query, "event:page") do
[
:visitors,
:visits,
@@ -469,7 +484,7 @@ defmodule PlausibleWeb.Api.StatsController do
|> transform_keys(%{source: :name})
if params["csv"] do
- if Map.has_key?(query.filters, "event:goal") do
+ if Query.get_filter(query, "event:goal") do
res
|> transform_keys(%{visitors: :conversions})
|> to_csv([:name, :conversions, :conversion_rate])
@@ -519,17 +534,17 @@ defmodule PlausibleWeb.Api.StatsController do
end
defp validate_funnel_query(query) do
- case query do
- _ when is_map_key(query.filters, "event:goal") ->
+ cond do
+ Query.get_filter(query, "event:goal") ->
{:error, {:invalid_funnel_query, "goals"}}
- _ when is_map_key(query.filters, "event:page") ->
+ Query.get_filter(query, "event:page") ->
{:error, {:invalid_funnel_query, "pages"}}
- _ when query.period == "realtime" ->
+ query.period == "realtime" ->
{:error, {:invalid_funnel_query, "realtime period"}}
- _ ->
+ true ->
:ok
end
end
@@ -547,7 +562,7 @@ defmodule PlausibleWeb.Api.StatsController do
|> transform_keys(%{utm_medium: :name})
if params["csv"] do
- if Map.has_key?(query.filters, "event:goal") do
+ if Query.get_filter(query, "event:goal") do
res
|> transform_keys(%{visitors: :conversions})
|> to_csv([:name, :conversions, :conversion_rate])
@@ -571,7 +586,7 @@ defmodule PlausibleWeb.Api.StatsController do
|> transform_keys(%{utm_campaign: :name})
if params["csv"] do
- if Map.has_key?(query.filters, "event:goal") do
+ if Query.get_filter(query, "event:goal") do
res
|> transform_keys(%{visitors: :conversions})
|> to_csv([:name, :conversions, :conversion_rate])
@@ -595,7 +610,7 @@ defmodule PlausibleWeb.Api.StatsController do
|> transform_keys(%{utm_content: :name})
if params["csv"] do
- if Map.has_key?(query.filters, "event:goal") do
+ if Query.get_filter(query, "event:goal") do
res
|> transform_keys(%{visitors: :conversions})
|> to_csv([:name, :conversions, :conversion_rate])
@@ -619,7 +634,7 @@ defmodule PlausibleWeb.Api.StatsController do
|> transform_keys(%{utm_term: :name})
if params["csv"] do
- if Map.has_key?(query.filters, "event:goal") do
+ if Query.get_filter(query, "event:goal") do
res
|> transform_keys(%{visitors: :conversions})
|> to_csv([:name, :conversions, :conversion_rate])
@@ -643,7 +658,7 @@ defmodule PlausibleWeb.Api.StatsController do
|> transform_keys(%{utm_source: :name})
if params["csv"] do
- if Map.has_key?(query.filters, "event:goal") do
+ if Query.get_filter(query, "event:goal") do
res
|> transform_keys(%{visitors: :conversions})
|> to_csv([:name, :conversions, :conversion_rate])
@@ -667,7 +682,7 @@ defmodule PlausibleWeb.Api.StatsController do
|> transform_keys(%{referrer: :name})
if params["csv"] do
- if Map.has_key?(query.filters, "event:goal") do
+ if Query.get_filter(query, "event:goal") do
res
|> transform_keys(%{visitors: :conversions})
|> to_csv([:name, :conversions, :conversion_rate])
@@ -711,9 +726,11 @@ defmodule PlausibleWeb.Api.StatsController do
site = conn.assigns[:site]
params = Map.put(params, "property", "visit:referrer")
+ referrer_filter = DashboardFilterParser.filter_value("visit:source", referrer)
+
query =
Query.from(site, params)
- |> Query.put_filter("visit:source", referrer)
+ |> Query.put_filter(referrer_filter)
pagination = parse_pagination(params)
@@ -747,7 +764,7 @@ defmodule PlausibleWeb.Api.StatsController do
|> transform_keys(%{page: :name})
if params["csv"] do
- if Map.has_key?(query.filters, "event:goal") do
+ if Query.get_filter(query, "event:goal") do
pages
|> transform_keys(%{visitors: :conversions})
|> to_csv([:name, :conversions, :conversion_rate])
@@ -771,7 +788,7 @@ defmodule PlausibleWeb.Api.StatsController do
|> transform_keys(%{entry_page: :name})
if params["csv"] do
- if Map.has_key?(query.filters, "event:goal") do
+ if Query.get_filter(query, "event:goal") do
to_csv(entry_pages, [:name, :visitors, :conversion_rate], [
:name,
:conversions,
@@ -803,7 +820,7 @@ defmodule PlausibleWeb.Api.StatsController do
|> transform_keys(%{exit_page: :name})
if params["csv"] do
- if Map.has_key?(query.filters, "event:goal") do
+ if Query.get_filter(query, "event:goal") do
to_csv(exit_pages, [:name, :visitors, :conversion_rate], [
:name,
:conversions,
@@ -829,8 +846,9 @@ defmodule PlausibleWeb.Api.StatsController do
pages = Enum.map(breakdown_results, & &1[:exit_page])
total_visits_query =
- Query.put_filter(query, "event:page", {:member, pages})
- |> Query.put_filter("event:name", {:is, "pageview"})
+ query
+ |> Query.put_filter([:member, "event:page", pages])
+ |> Query.put_filter([:is, "event:name", "pageview"])
|> struct!(property: "event:page")
total_pageviews =
@@ -870,7 +888,7 @@ defmodule PlausibleWeb.Api.StatsController do
Map.put(country, :name, country_info.name)
end)
- if Map.has_key?(query.filters, "event:goal") do
+ if Query.get_filter(query, "event:goal") do
countries
|> transform_keys(%{visitors: :conversions})
|> to_csv([:name, :conversions, :conversion_rate])
@@ -926,7 +944,7 @@ defmodule PlausibleWeb.Api.StatsController do
end)
if params["csv"] do
- if Map.has_key?(query.filters, "event:goal") do
+ if Query.get_filter(query, "event:goal") do
regions
|> transform_keys(%{visitors: :conversions})
|> to_csv([:name, :conversions, :conversion_rate])
@@ -966,7 +984,7 @@ defmodule PlausibleWeb.Api.StatsController do
end)
if params["csv"] do
- if Map.has_key?(query.filters, "event:goal") do
+ if Query.get_filter(query, "event:goal") do
cities
|> transform_keys(%{visitors: :conversions})
|> to_csv([:name, :conversions, :conversion_rate])
@@ -990,7 +1008,7 @@ defmodule PlausibleWeb.Api.StatsController do
|> transform_keys(%{browser: :name})
if params["csv"] do
- if Map.has_key?(query.filters, "event:goal") do
+ if Query.get_filter(query, "event:goal") do
browsers
|> transform_keys(%{visitors: :conversions})
|> to_csv([:name, :conversions, :conversion_rate])
@@ -1014,7 +1032,7 @@ defmodule PlausibleWeb.Api.StatsController do
|> transform_keys(%{browser_version: :name})
if params["csv"] do
- if Map.has_key?(query.filters, "event:goal") do
+ if Query.get_filter(query, "event:goal") do
versions
|> transform_keys(%{
name: :version,
@@ -1044,7 +1062,7 @@ defmodule PlausibleWeb.Api.StatsController do
|> transform_keys(%{os: :name})
if params["csv"] do
- if Map.has_key?(query.filters, "event:goal") do
+ if Query.get_filter(query, "event:goal") do
systems
|> transform_keys(%{visitors: :conversions})
|> to_csv([:name, :conversions, :conversion_rate])
@@ -1068,7 +1086,7 @@ defmodule PlausibleWeb.Api.StatsController do
|> transform_keys(%{os_version: :name})
if params["csv"] do
- if Map.has_key?(query.filters, "event:goal") do
+ if Query.get_filter(query, "event:goal") do
versions
|> transform_keys(%{name: :version, os: :name, visitors: :conversions})
|> to_csv([:name, :version, :conversions, :conversion_rate])
@@ -1094,7 +1112,7 @@ defmodule PlausibleWeb.Api.StatsController do
|> transform_keys(%{device: :name})
if params["csv"] do
- if Map.has_key?(query.filters, "event:goal") do
+ if Query.get_filter(query, "event:goal") do
sizes
|> transform_keys(%{visitors: :conversions})
|> to_csv([:name, :conversions, :conversion_rate])
@@ -1183,7 +1201,7 @@ defmodule PlausibleWeb.Api.StatsController do
|> Enum.concat()
percent_or_cr =
- if query.filters["event:goal"],
+ if Query.get_filter(query, "event:goal"),
do: :conversion_rate,
else: :percentage
@@ -1200,7 +1218,7 @@ defmodule PlausibleWeb.Api.StatsController do
query = Query.from(site, params)
metrics =
- if query.filters["event:goal"] do
+ if Query.get_filter(query, "event:goal") do
[:visitors, :events, :conversion_rate] ++ @revenue_metrics
else
[:visitors, :events, :percentage] ++ @revenue_metrics
@@ -1359,7 +1377,7 @@ defmodule PlausibleWeb.Api.StatsController do
requires_goal_filter? = metric in [:conversion_rate, :events]
- if requires_goal_filter? and !query.filters["event:goal"] do
+ if requires_goal_filter? and !Query.get_filter(query, "event:goal") do
{:error, "Metric `#{metric}` can only be queried with a goal filter"}
else
{:ok, metric}
@@ -1401,7 +1419,7 @@ defmodule PlausibleWeb.Api.StatsController do
end
defp breakdown_metrics(query, extra_metrics \\ []) do
- if query.filters["event:goal"] do
+ if Query.get_filter(query, "event:goal") do
[:visitors, :conversion_rate, :total_visitors]
else
[:visitors] ++ extra_metrics
diff --git a/lib/plausible_web/controllers/stats_controller.ex b/lib/plausible_web/controllers/stats_controller.ex
index ca921c045f..d198647432 100644
--- a/lib/plausible_web/controllers/stats_controller.ex
+++ b/lib/plausible_web/controllers/stats_controller.ex
@@ -180,16 +180,28 @@ defmodule PlausibleWeb.StatsController do
|> Enum.join()
end
- defp csv_graph_metrics(%Query{filters: %{"event:goal" => _}}) do
- metrics = [:visitors, :events, :conversion_rate]
- column_headers = [:date, :unique_conversions, :total_conversions, :conversion_rate]
-
- {metrics, column_headers}
- end
-
- defp csv_graph_metrics(_) do
- metrics = [:visitors, :pageviews, :visits, :views_per_visit, :bounce_rate, :visit_duration]
- column_headers = [:date | metrics]
+ defp csv_graph_metrics(query) do
+ {metrics, column_headers} =
+ if Query.get_filter(query, "event:goal") do
+ {
+ [:visitors, :events, :conversion_rate],
+ [:date, :unique_conversions, :total_conversions, :conversion_rate]
+ }
+ else
+ metrics = [
+ :visitors,
+ :pageviews,
+ :visits,
+ :views_per_visit,
+ :bounce_rate,
+ :visit_duration
+ ]
+
+ {
+ metrics,
+ [:date | metrics]
+ }
+ end
{metrics, column_headers}
end
diff --git a/test/plausible/google/search_console/filters_test.exs b/test/plausible/google/search_console/filters_test.exs
index 7d73dacda8..c6efffa069 100644
--- a/test/plausible/google/search_console/filters_test.exs
+++ b/test/plausible/google/search_console/filters_test.exs
@@ -3,9 +3,9 @@ defmodule Plausible.Google.SearchConsole.FiltersTest do
use Plausible.DataCase, async: true
test "transforms simple page filter" do
- filters = %{
- "visit:entry_page" => {:is, "/page"}
- }
+ filters = [
+ [:is, "visit:entry_page", "/page"]
+ ]
{:ok, transformed} = Filters.transform("sc-domain:plausible.io", filters)
@@ -15,9 +15,9 @@ defmodule Plausible.Google.SearchConsole.FiltersTest do
end
test "transforms matches page filter" do
- filters = %{
- "visit:entry_page" => {:matches, "*page*"}
- }
+ filters = [
+ [:matches, "visit:entry_page", "*page*"]
+ ]
{:ok, transformed} = Filters.transform("sc-domain:plausible.io", filters)
@@ -35,9 +35,9 @@ defmodule Plausible.Google.SearchConsole.FiltersTest do
end
test "transforms member page filter" do
- filters = %{
- "visit:entry_page" => {:member, ["/pageA", "/pageB"]}
- }
+ filters = [
+ [:member, "visit:entry_page", ["/pageA", "/pageB"]]
+ ]
{:ok, transformed} = Filters.transform("sc-domain:plausible.io", filters)
@@ -55,9 +55,9 @@ defmodule Plausible.Google.SearchConsole.FiltersTest do
end
test "transforms matches_member page filter" do
- filters = %{
- "visit:entry_page" => {:matches_member, ["/pageA*", "/pageB*"]}
- }
+ filters = [
+ [:matches_member, "visit:entry_page", ["/pageA*", "/pageB*"]]
+ ]
{:ok, transformed} = Filters.transform("sc-domain:plausible.io", filters)
@@ -75,9 +75,9 @@ defmodule Plausible.Google.SearchConsole.FiltersTest do
end
test "transforms event:page exactly like visit:entry_page" do
- filters = %{
- "event:page" => {:matches_member, ["/pageA*", "/pageB*"]}
- }
+ filters = [
+ [:matches_member, "event:page", ["/pageA*", "/pageB*"]]
+ ]
{:ok, transformed} = Filters.transform("sc-domain:plausible.io", filters)
@@ -95,9 +95,9 @@ defmodule Plausible.Google.SearchConsole.FiltersTest do
end
test "transforms simple visit:screen filter" do
- filters = %{
- "visit:screen" => {:is, "Desktop"}
- }
+ filters = [
+ [:is, "visit:screen", "Desktop"]
+ ]
{:ok, transformed} = Filters.transform("sc-domain:plausible.io", filters)
@@ -105,9 +105,9 @@ defmodule Plausible.Google.SearchConsole.FiltersTest do
end
test "transforms member visit:screen filter" do
- filters = %{
- "visit:screen" => {:member, ["Mobile", "Tablet"]}
- }
+ filters = [
+ [:member, "visit:screen", ["Mobile", "Tablet"]]
+ ]
{:ok, transformed} = Filters.transform("sc-domain:plausible.io", filters)
@@ -121,9 +121,9 @@ defmodule Plausible.Google.SearchConsole.FiltersTest do
end
test "transforms simple visit:country filter to alpha3" do
- filters = %{
- "visit:country" => {:is, "EE"}
- }
+ filters = [
+ [:is, "visit:country", "EE"]
+ ]
{:ok, transformed} = Filters.transform("sc-domain:plausible.io", filters)
@@ -131,9 +131,9 @@ defmodule Plausible.Google.SearchConsole.FiltersTest do
end
test "transforms member visit:country filter" do
- filters = %{
- "visit:country" => {:member, ["EE", "PL"]}
- }
+ filters = [
+ [:member, "visit:country", ["EE", "PL"]]
+ ]
{:ok, transformed} = Filters.transform("sc-domain:plausible.io", filters)
@@ -147,11 +147,11 @@ defmodule Plausible.Google.SearchConsole.FiltersTest do
end
test "filters can be combined" do
- filters = %{
- "visit:entry_page" => {:matches, "*web-analytics*"},
- "visit:screen" => {:is, "Desktop"},
- "visit:country" => {:member, ["EE", "PL"]}
- }
+ filters = [
+ [:member, "visit:country", ["EE", "PL"]],
+ [:matches, "visit:entry_page", "*web-analytics*"],
+ [:is, "visit:screen", "Desktop"]
+ ]
{:ok, transformed} = Filters.transform("sc-domain:plausible.io", filters)
@@ -171,12 +171,12 @@ defmodule Plausible.Google.SearchConsole.FiltersTest do
end
test "when unsupported filter is included the whole set becomes invalid" do
- filters = %{
- "visit:entry_page" => {:matches, "*web-analytics*"},
- "visit:screen" => {:is, "Desktop"},
- "visit:country" => {:member, ["EE", "PL"]},
- "visit:utm_medium" => {:is, "facebook"}
- }
+ filters = [
+ [:matches, "visit:entry_page", "*web-analytics*"],
+ [:is, "visit:screen", "Desktop"],
+ [:member, "visit:country", ["EE", "PL"]],
+ [:is, "visit:utm_medium", "facebook"]
+ ]
assert :unsupported_filters = Filters.transform("sc-domain:plausible.io", filters)
end
diff --git a/test/plausible/stats/dashboard_filter_parser_test.exs b/test/plausible/stats/dashboard_filter_parser_test.exs
index e5469bdfda..1602380f37 100644
--- a/test/plausible/stats/dashboard_filter_parser_test.exs
+++ b/test/plausible/stats/dashboard_filter_parser_test.exs
@@ -9,199 +9,199 @@ defmodule Plausible.Stats.DashboardFilterParserTest do
describe "adding prefix" do
test "adds appropriate prefix to filter" do
%{"page" => "/"}
- |> assert_parsed(%{"event:page" => {:is, "/"}})
+ |> assert_parsed([[:is, "event:page", "/"]])
%{"goal" => "Signup"}
- |> assert_parsed(%{"event:goal" => {:is, {:event, "Signup"}}})
+ |> assert_parsed([[:is, "event:goal", {:event, "Signup"}]])
%{"goal" => "Visit /blog"}
- |> assert_parsed(%{"event:goal" => {:is, {:page, "/blog"}}})
+ |> assert_parsed([[:is, "event:goal", {:page, "/blog"}]])
%{"source" => "Google"}
- |> assert_parsed(%{"visit:source" => {:is, "Google"}})
+ |> assert_parsed([[:is, "visit:source", "Google"]])
%{"referrer" => "cnn.com"}
- |> assert_parsed(%{"visit:referrer" => {:is, "cnn.com"}})
+ |> assert_parsed([[:is, "visit:referrer", "cnn.com"]])
%{"utm_medium" => "search"}
- |> assert_parsed(%{"visit:utm_medium" => {:is, "search"}})
+ |> assert_parsed([[:is, "visit:utm_medium", "search"]])
%{"utm_source" => "bing"}
- |> assert_parsed(%{"visit:utm_source" => {:is, "bing"}})
+ |> assert_parsed([[:is, "visit:utm_source", "bing"]])
%{"utm_content" => "content"}
- |> assert_parsed(%{"visit:utm_content" => {:is, "content"}})
+ |> assert_parsed([[:is, "visit:utm_content", "content"]])
%{"utm_term" => "term"}
- |> assert_parsed(%{"visit:utm_term" => {:is, "term"}})
+ |> assert_parsed([[:is, "visit:utm_term", "term"]])
%{"screen" => "Desktop"}
- |> assert_parsed(%{"visit:screen" => {:is, "Desktop"}})
+ |> assert_parsed([[:is, "visit:screen", "Desktop"]])
%{"browser" => "Opera"}
- |> assert_parsed(%{"visit:browser" => {:is, "Opera"}})
+ |> assert_parsed([[:is, "visit:browser", "Opera"]])
%{"browser_version" => "10.1"}
- |> assert_parsed(%{"visit:browser_version" => {:is, "10.1"}})
+ |> assert_parsed([[:is, "visit:browser_version", "10.1"]])
%{"os" => "Linux"}
- |> assert_parsed(%{"visit:os" => {:is, "Linux"}})
+ |> assert_parsed([[:is, "visit:os", "Linux"]])
%{"os_version" => "13.0"}
- |> assert_parsed(%{"visit:os_version" => {:is, "13.0"}})
+ |> assert_parsed([[:is, "visit:os_version", "13.0"]])
%{"country" => "EE"}
- |> assert_parsed(%{"visit:country" => {:is, "EE"}})
+ |> assert_parsed([[:is, "visit:country", "EE"]])
%{"region" => "EE-12"}
- |> assert_parsed(%{"visit:region" => {:is, "EE-12"}})
+ |> assert_parsed([[:is, "visit:region", "EE-12"]])
%{"city" => "123"}
- |> assert_parsed(%{"visit:city" => {:is, "123"}})
+ |> assert_parsed([[:is, "visit:city", "123"]])
%{"entry_page" => "/blog"}
- |> assert_parsed(%{"visit:entry_page" => {:is, "/blog"}})
+ |> assert_parsed([[:is, "visit:entry_page", "/blog"]])
%{"exit_page" => "/blog"}
- |> assert_parsed(%{"visit:exit_page" => {:is, "/blog"}})
+ |> assert_parsed([[:is, "visit:exit_page", "/blog"]])
%{"props" => %{"cta" => "Top"}}
- |> assert_parsed(%{"event:props:cta" => {:is, "Top"}})
+ |> assert_parsed([[:is, "event:props:cta", "Top"]])
%{"hostname" => "dummy.site"}
- |> assert_parsed(%{"event:hostname" => {:is, "dummy.site"}})
+ |> assert_parsed([[:is, "event:hostname", "dummy.site"]])
end
end
describe "escaping pipe character" do
test "in simple is filter" do
%{"goal" => ~S(Foo \| Bar)}
- |> assert_parsed(%{"event:goal" => {:is, {:event, "Foo | Bar"}}})
+ |> assert_parsed([[:is, "event:goal", {:event, "Foo | Bar"}]])
end
test "in member filter" do
%{"page" => ~S(/|\|)}
- |> assert_parsed(%{"event:page" => {:member, ["/", "|"]}})
+ |> assert_parsed([[:member, "event:page", ["/", "|"]]])
end
end
describe "is not filter type" do
test "simple is not filter" do
%{"page" => "!/"}
- |> assert_parsed(%{"event:page" => {:is_not, "/"}})
+ |> assert_parsed([[:is_not, "event:page", "/"]])
%{"props" => %{"cta" => "!Top"}}
- |> assert_parsed(%{"event:props:cta" => {:is_not, "Top"}})
+ |> assert_parsed([[:is_not, "event:props:cta", "Top"]])
end
end
describe "member filter type" do
test "simple member filter" do
%{"page" => "/|/blog"}
- |> assert_parsed(%{"event:page" => {:member, ["/", "/blog"]}})
+ |> assert_parsed([[:member, "event:page", ["/", "/blog"]]])
end
test "escaping pipe character" do
%{"page" => "/|\\|"}
- |> assert_parsed(%{"event:page" => {:member, ["/", "|"]}})
+ |> assert_parsed([[:member, "event:page", ["/", "|"]]])
end
test "mixed goals" do
%{"goal" => "Signup|Visit /thank-you"}
- |> assert_parsed(%{"event:goal" => {:member, [{:event, "Signup"}, {:page, "/thank-you"}]}})
+ |> assert_parsed([[:member, "event:goal", [{:event, "Signup"}, {:page, "/thank-you"}]]])
%{"goal" => "Visit /thank-you|Signup"}
- |> assert_parsed(%{"event:goal" => {:member, [{:page, "/thank-you"}, {:event, "Signup"}]}})
+ |> assert_parsed([[:member, "event:goal", [{:page, "/thank-you"}, {:event, "Signup"}]]])
end
end
describe "matches_member filter type" do
test "parses matches_member filter type" do
%{"page" => "/|/blog**"}
- |> assert_parsed(%{"event:page" => {:matches_member, ["/", "/blog**"]}})
+ |> assert_parsed([[:matches_member, "event:page", ["/", "/blog**"]]])
end
test "parses not_matches_member filter type" do
%{"page" => "!/|/blog**"}
- |> assert_parsed(%{"event:page" => {:not_matches_member, ["/", "/blog**"]}})
+ |> assert_parsed([[:not_matches_member, "event:page", ["/", "/blog**"]]])
end
end
describe "contains filter type" do
test "single contains" do
%{"page" => "~blog"}
- |> assert_parsed(%{"event:page" => {:matches, "**blog**"}})
+ |> assert_parsed([[:matches, "event:page", "**blog**"]])
end
test "negated contains" do
%{"page" => "!~articles"}
- |> assert_parsed(%{"event:page" => {:does_not_match, "**articles**"}})
+ |> assert_parsed([[:does_not_match, "event:page", "**articles**"]])
end
test "contains member" do
%{"page" => "~articles|blog"}
- |> assert_parsed(%{"event:page" => {:matches_member, ["**articles**", "**blog**"]}})
+ |> assert_parsed([[:matches_member, "event:page", ["**articles**", "**blog**"]]])
end
test "not contains member" do
%{"page" => "!~articles|blog"}
- |> assert_parsed(%{"event:page" => {:not_matches_member, ["**articles**", "**blog**"]}})
+ |> assert_parsed([[:not_matches_member, "event:page", ["**articles**", "**blog**"]]])
end
end
describe "not_member filter type" do
test "simple not_member filter" do
%{"page" => "!/|/blog"}
- |> assert_parsed(%{"event:page" => {:not_member, ["/", "/blog"]}})
+ |> assert_parsed([[:not_member, "event:page", ["/", "/blog"]]])
end
test "mixed goals" do
%{"goal" => "!Signup|Visit /thank-you"}
- |> assert_parsed(%{
- "event:goal" => {:not_member, [{:event, "Signup"}, {:page, "/thank-you"}]}
- })
+ |> assert_parsed([
+ [:not_member, "event:goal", [{:event, "Signup"}, {:page, "/thank-you"}]]
+ ])
%{"goal" => "!Visit /thank-you|Signup"}
- |> assert_parsed(%{
- "event:goal" => {:not_member, [{:page, "/thank-you"}, {:event, "Signup"}]}
- })
+ |> assert_parsed([
+ [:not_member, "event:goal", [{:page, "/thank-you"}, {:event, "Signup"}]]
+ ])
end
end
describe "matches filter type" do
test "can be used with `goal` or `page` filters" do
%{"page" => "/blog/post-*"}
- |> assert_parsed(%{"event:page" => {:matches, "/blog/post-*"}})
+ |> assert_parsed([[:matches, "event:page", "/blog/post-*"]])
%{"goal" => "Visit /blog/post-*"}
- |> assert_parsed(%{"event:goal" => {:matches, {:page, "/blog/post-*"}}})
+ |> assert_parsed([[:matches, "event:goal", {:page, "/blog/post-*"}]])
end
test "other filters default to `is` even when wildcard is present" do
%{"country" => "Germa**"}
- |> assert_parsed(%{"visit:country" => {:is, "Germa**"}})
+ |> assert_parsed([[:is, "visit:country", "Germa**"]])
end
end
describe "does_not_match filter type" do
test "can be used with `page` filter" do
%{"page" => "!/blog/post-*"}
- |> assert_parsed(%{"event:page" => {:does_not_match, "/blog/post-*"}})
+ |> assert_parsed([[:does_not_match, "event:page", "/blog/post-*"]])
end
test "other filters default to is_not even when wildcard is present" do
%{"country" => "!Germa**"}
- |> assert_parsed(%{"visit:country" => {:is_not, "Germa**"}})
+ |> assert_parsed([[:is_not, "visit:country", "Germa**"]])
end
end
describe "contains prefix filter type" do
test "can be used with any filter" do
%{"page" => "~/blog/post"}
- |> assert_parsed(%{"event:page" => {:matches, "**/blog/post**"}})
+ |> assert_parsed([[:matches, "event:page", "**/blog/post**"]])
%{"source" => "~facebook"}
- |> assert_parsed(%{"visit:source" => {:matches, "**facebook**"}})
+ |> assert_parsed([[:matches, "visit:source", "**facebook**"]])
end
end
end
diff --git a/test/plausible/stats/filters_test.exs b/test/plausible/stats/filters_test.exs
index 4f87bd9957..aa5b77412f 100644
--- a/test/plausible/stats/filters_test.exs
+++ b/test/plausible/stats/filters_test.exs
@@ -12,90 +12,90 @@ defmodule Plausible.Stats.FiltersTest do
describe "parses filter expression" do
test "simple positive" do
"event:name==pageview"
- |> assert_parsed(%{"event:name" => {:is, "pageview"}})
+ |> assert_parsed([[:is, "event:name", "pageview"]])
end
test "simple negative" do
"event:name!=pageview"
- |> assert_parsed(%{"event:name" => {:is_not, "pageview"}})
+ |> assert_parsed([[:is_not, "event:name", "pageview"]])
end
test "whitespace is trimmed" do
" event:name == pageview "
- |> assert_parsed(%{"event:name" => {:is, "pageview"}})
+ |> assert_parsed([[:is, "event:name", "pageview"]])
end
test "wildcard" do
"event:page==/blog/post-*"
- |> assert_parsed(%{"event:page" => {:matches, "/blog/post-*"}})
+ |> assert_parsed([[:matches, "event:page", "/blog/post-*"]])
end
test "negative wildcard" do
"event:page!=/blog/post-*"
- |> assert_parsed(%{"event:page" => {:does_not_match, "/blog/post-*"}})
+ |> assert_parsed([[:does_not_match, "event:page", "/blog/post-*"]])
end
test "custom event goal" do
"event:goal==Signup"
- |> assert_parsed(%{"event:goal" => {:is, {:event, "Signup"}}})
+ |> assert_parsed([[:is, "event:goal", {:event, "Signup"}]])
end
test "pageview goal" do
"event:goal==Visit /blog"
- |> assert_parsed(%{"event:goal" => {:is, {:page, "/blog"}}})
+ |> assert_parsed([[:is, "event:goal", {:page, "/blog"}]])
end
test "member" do
"visit:country==FR|GB|DE"
- |> assert_parsed(%{"visit:country" => {:member, ["FR", "GB", "DE"]}})
+ |> assert_parsed([[:member, "visit:country", ["FR", "GB", "DE"]]])
end
test "member + wildcard" do
"event:page==/blog**|/newsletter|/*/"
- |> assert_parsed(%{"event:page" => {:matches, "/blog**|/newsletter|/*/"}})
+ |> assert_parsed([[:matches, "event:page", "/blog**|/newsletter|/*/"]])
end
test "combined with \";\"" do
"event:page==/blog**|/newsletter|/*/ ; visit:country==FR|GB|DE"
- |> assert_parsed(%{
- "event:page" => {:matches, "/blog**|/newsletter|/*/"},
- "visit:country" => {:member, ["FR", "GB", "DE"]}
- })
+ |> assert_parsed([
+ [:matches, "event:page", "/blog**|/newsletter|/*/"],
+ [:member, "visit:country", ["FR", "GB", "DE"]]
+ ])
end
test "escaping pipe character" do
"utm_campaign==campaign \\| 1"
- |> assert_parsed(%{"utm_campaign" => {:is, "campaign | 1"}})
+ |> assert_parsed([[:is, "utm_campaign", "campaign | 1"]])
end
test "escaping pipe character in member filter" do
"utm_campaign==campaign \\| 1|campaign \\| 2"
- |> assert_parsed(%{"utm_campaign" => {:member, ["campaign | 1", "campaign | 2"]}})
+ |> assert_parsed([[:member, "utm_campaign", ["campaign | 1", "campaign | 2"]]])
end
test "keeps escape characters in member + wildcard filter" do
"event:page==/**\\|page|/other/page"
- |> assert_parsed(%{"event:page" => {:matches, "/**\\|page|/other/page"}})
+ |> assert_parsed([[:matches, "event:page", "/**\\|page|/other/page"]])
end
test "gracefully fails to parse garbage" do
"bfg10309\uff1cs1\ufe65s2\u02bas3\u02b9hjl10309"
- |> assert_parsed(%{})
+ |> assert_parsed([])
end
test "gracefully fails to parse garbage with double quotes" do
"\";print(md5(31337));$a=\""
- |> assert_parsed(%{})
+ |> assert_parsed([])
end
test "gracefully fails to parse garbage country code" do
"visit:country==AKSJSDFKJSS"
- |> assert_parsed(%{})
+ |> assert_parsed([])
end
test "gracefully fails to parse garbage country code (with pipes)" do
"visit:country==ET'||DBMS_PIPE.RECEIVE_MESSAGE(CHR(98)||CHR(98)||CHR(98),15)||'"
- |> assert_parsed(%{})
+ |> assert_parsed([])
end
end
end
diff --git a/test/plausible/stats/query_test.exs b/test/plausible/stats/query_test.exs
index f568589549..b509b6914c 100644
--- a/test/plausible/stats/query_test.exs
+++ b/test/plausible/stats/query_test.exs
@@ -194,14 +194,14 @@ defmodule Plausible.Stats.QueryTest do
filters = Jason.encode!(%{"goal" => "Signup"})
q = Query.from(site, %{"period" => "6mo", "filters" => filters})
- assert q.filters["event:goal"] == {:is, {:event, "Signup"}}
+ assert q.filters == [[:is, "event:goal", {:event, "Signup"}]]
end
test "parses source filter", %{site: site} do
filters = Jason.encode!(%{"source" => "Twitter"})
q = Query.from(site, %{"period" => "6mo", "filters" => filters})
- assert q.filters["visit:source"] == {:is, "Twitter"}
+ assert q.filters == [[:is, "visit:source", "Twitter"]]
end
end