Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.
TypeScript analyticsanalytics-dashboardclickhouseelixirgoogle-analyticsmarketingopen-source-analyticsphoenixplausible-analyticspostgresqlprivacyprivacy-friendlyself-hostedsimple-analyticsstatisticstailwindcssweb-analyticswebsite-analyticswebsite-statswebsite-tracking
Refactor internal Query schema and introduce WhereBuilder (#4082)
Karl-Aksel Puulmannbaa99652f612
diff --git a/extra/lib/plausible/stats/goal/revenue.ex b/extra/lib/plausible/stats/goal/revenue.exindex 7b6e3368f5..8c4115c0aa 100644--- a/extra/lib/plausible/stats/goal/revenue.ex+++ b/extra/lib/plausible/stats/goal/revenue.ex@@ -4,6 +4,8 @@ defmodule Plausible.Stats.Goal.Revenue do"""import Ecto.Query+ alias Plausible.Stats.Query+@revenue_metrics [:average_revenue, :total_revenue]def revenue_metrics() do@@ -37,10 +39,10 @@ defmodule Plausible.Stats.Goal.Revenue do"""def get_revenue_tracking_currency(site, query, metrics) dogoal_filters =- case query.filters do- %{"event:goal" => {:is, {_, goal_name}}} -> [goal_name]- %{"event:goal" => {:member, list}} -> Enum.map(list, fn {_, goal_name} -> goal_name end)- _any -> []+ case Query.get_filter(query, "event:goal") do+ [:is, "event:goal", {_, goal_name}] -> [goal_name]+ [:member, "event:goal", list] -> Enum.map(list, fn {_, goal_name} -> goal_name end)+ _ -> []endrequested_revenue_metrics? = Enum.any?(metrics, &(&1 in @revenue_metrics))diff --git a/lib/plausible/google/api.ex b/lib/plausible/google/api.exindex 3fd63c87f7..763981aea9 100644--- a/lib/plausible/google/api.ex+++ b/lib/plausible/google/api.ex@@ -74,16 +74,16 @@ defmodule Plausible.Google.API doendend- def fetch_stats(site, %{filters: %{} = filters, date_range: date_range}, limit) do+ def fetch_stats(site, query, limit) dowith {:ok, site} <- ensure_search_console_property(site),{:ok, access_token} <- maybe_refresh_token(site.google_auth),{:ok, search_console_filters} <-- SearchConsole.Filters.transform(site.google_auth.property, filters),+ SearchConsole.Filters.transform(site.google_auth.property, query.filters),{:ok, stats} <-HTTP.list_stats(access_token,site.google_auth.property,- date_range,+ query.date_range,limit,search_console_filters) dodiff --git a/lib/plausible/google/search_console/filters.ex b/lib/plausible/google/search_console/filters.exindex 4e0e00d875..484ed84390 100644--- a/lib/plausible/google/search_console/filters.ex+++ b/lib/plausible/google/search_console/filters.ex@@ -3,12 +3,11 @@ defmodule Plausible.Google.SearchConsole.Filters doimport Plausible.Stats.Base, only: [page_regex: 1]def transform(property, plausible_filters) do- plausible_filters = Map.drop(plausible_filters, ["visit:source"])-search_console_filters =Enum.reduce_while(plausible_filters, [], fn plausible_filter, search_console_filters ->case transform_filter(property, plausible_filter) do:unsupported -> {:halt, :unsupported_filters}+ :ignore -> {:cont, search_console_filters}search_console_filter -> {:cont, [search_console_filter | search_console_filters]}endend)@@ -20,27 +19,27 @@ defmodule Plausible.Google.SearchConsole.Filters doendend- defp transform_filter(property, {"event:page", filter}) do- transform_filter(property, {"visit:entry_page", filter})+ defp transform_filter(property, [op, "event:page" | rest]) do+ transform_filter(property, [op, "visit:entry_page" | rest])end- defp transform_filter(property, {"visit:entry_page", {:is, page}}) when is_binary(page) do+ defp transform_filter(property, [:is, "visit:entry_page", page]) when is_binary(page) do%{dimension: "page", expression: property_url(property, page)}end- defp transform_filter(property, {"visit:entry_page", {:member, pages}}) when is_list(pages) do+ defp transform_filter(property, [:member, "visit:entry_page", pages]) when is_list(pages) doexpression =Enum.map_join(pages, "|", fn page -> property_url(property, Regex.escape(page)) end)%{dimension: "page", operator: "includingRegex", expression: expression}end- defp transform_filter(property, {"visit:entry_page", {:matches, page}}) when is_binary(page) do+ defp transform_filter(property, [:matches, "visit:entry_page", page]) when is_binary(page) dopage = page_regex(property_url(property, page))%{dimension: "page", operator: "includingRegex", expression: page}end- defp transform_filter(property, {"visit:entry_page", {:matches_member, pages}})+ defp transform_filter(property, [:matches_member, "visit:entry_page", pages])when is_list(pages) doexpression =Enum.map_join(pages, "|", fn page -> page_regex(property_url(property, page)) end)@@ -48,25 +47,27 @@ defmodule Plausible.Google.SearchConsole.Filters do%{dimension: "page", operator: "includingRegex", expression: expression}end- defp transform_filter(_property, {"visit:screen", {:is, device}}) when is_binary(device) do+ defp transform_filter(_property, [:is, "visit:screen", device]) when is_binary(device) do%{dimension: "device", expression: search_console_device(device)}end- defp transform_filter(_property, {"visit:screen", {:member, devices}}) when is_list(devices) do+ defp transform_filter(_property, [:member, "visit:screen", devices]) when is_list(devices) doexpression = devices |> Enum.join("|")%{dimension: "device", operator: "includingRegex", expression: expression}end- defp transform_filter(_property, {"visit:country", {:is, country}}) when is_binary(country) do+ defp transform_filter(_property, [:is, "visit:country", country]) when is_binary(country) do%{dimension: "country", expression: search_console_country(country)}end- defp transform_filter(_property, {"visit:country", {:member, countries}})+ defp transform_filter(_property, [:member, "visit:country", countries])when is_list(countries) doexpression = Enum.map_join(countries, "|", &search_console_country/1)%{dimension: "country", operator: "includingRegex", expression: expression}end+ defp transform_filter(_, [_, "visit:source" | _rest]), do: :ignore+defp transform_filter(_, _filter), do: :unsupporteddefp property_url("sc-domain:" <> domain, page), do: "https://" <> domain <> pagediff --git a/lib/plausible/stats/aggregate.ex b/lib/plausible/stats/aggregate.exindex 991e80431e..73d33f0b24 100644--- a/lib/plausible/stats/aggregate.ex+++ b/lib/plausible/stats/aggregate.ex@@ -70,10 +70,7 @@ defmodule Plausible.Stats.Aggregate dodefp neighbor_aggregate_time_on_page(site, query) doq =from(- e in base_event_query(site, %Query{- query- | filters: Map.delete(query.filters, "event:page")- }),+ e in base_event_query(site, Query.remove_filters(query, ["event:page"])),select: {fragment("? as p", e.pathname),fragment("? as t", e.timestamp),@@ -86,32 +83,32 @@ defmodule Plausible.Stats.Aggregate dowhere_param_idx = length(base_query_raw_params){where_clause, where_arg} =- case query.filters["event:page"] do- {:is, page} ->+ case Query.get_filter(query, "event:page") do+ [:is, _, page] ->{"p = {$#{where_param_idx}:String}", page}- {:is_not, page} ->+ [:is_not, _, page] ->{"p != {$#{where_param_idx}:String}", page}- {:member, page} ->+ [:member, _, page] ->{"p IN {$#{where_param_idx}:Array(String)}", page}- {:not_member, page} ->+ [:not_member, _, page] ->{"p NOT IN {$#{where_param_idx}:Array(String)}", page}- {:matches, expr} ->+ [:matches, _, expr] ->regex = page_regex(expr){"match(p, {$#{where_param_idx}:String})", regex}- {:matches_member, exprs} ->+ [:matches_member, _, exprs] ->page_regexes = Enum.map(exprs, &page_regex/1){"multiMatchAny(p, {$#{where_param_idx}:Array(String)})", page_regexes}- {:not_matches_member, exprs} ->+ [:not_matches_member, _, exprs] ->page_regexes = Enum.map(exprs, &page_regex/1){"not(multiMatchAny(p, {$#{where_param_idx}:Array(String)}))", page_regexes}- {:does_not_match, expr} ->+ [:does_not_match, _, expr] ->regex = page_regex(expr){"not(match(p, {$#{where_param_idx}:String}))", regex}end@@ -148,29 +145,29 @@ defmodule Plausible.Stats.Aggregate dodefp window_aggregate_time_on_page(site, query) dowindowed_pages_q =- from e in base_event_query(site, %Query{- query- | filters: Map.delete(query.filters, "event:page")- }),- select: %{- next_timestamp: over(fragment("leadInFrame(?)", e.timestamp), :event_horizon),- next_pathname: over(fragment("leadInFrame(?)", e.pathname), :event_horizon),- timestamp: e.timestamp,- pathname: e.pathname,- session_id: e.session_id- },- windows: [- event_horizon: [- partition_by: e.session_id,- order_by: e.timestamp,- frame: fragment("ROWS BETWEEN CURRENT ROW AND 1 FOLLOWING")- ]- ]+ from e in base_event_query(site, Query.remove_filters(query, ["event:page"])),+ select: %{+ next_timestamp: over(fragment("leadInFrame(?)", e.timestamp), :event_horizon),+ next_pathname: over(fragment("leadInFrame(?)", e.pathname), :event_horizon),+ timestamp: e.timestamp,+ pathname: e.pathname,+ session_id: e.session_id+ },+ windows: [+ event_horizon: [+ partition_by: e.session_id,+ order_by: e.timestamp,+ frame: fragment("ROWS BETWEEN CURRENT ROW AND 1 FOLLOWING")+ ]+ ]++ event_page_filter = Query.get_filter(query, "event:page")timed_page_transitions_q =from e in Ecto.Query.subquery(windowed_pages_q),group_by: [e.pathname, e.next_pathname, e.session_id],- where: ^Plausible.Stats.Base.dynamic_filter_condition(query, "event:page", :pathname),+ where:+ ^Plausible.Stats.Filters.WhereBuilder.build_condition(:pathname, event_page_filter),where: e.next_timestamp != 0,select: %{pathname: e.pathname,diff --git a/lib/plausible/stats/base.ex b/lib/plausible/stats/base.exindex fe74fb426b..07d14a7871 100644--- a/lib/plausible/stats/base.ex+++ b/lib/plausible/stats/base.ex@@ -7,9 +7,6 @@ defmodule Plausible.Stats.Base doalias Plausible.Timezonesimport Ecto.Query- @no_ref "Direct / None"- @not_set "(not set)"-@uniq_users_expression "toUInt64(round(uniq(?) * any(_sample_factor)))"def base_event_query(site, query) do@@ -30,180 +27,30 @@ defmodule Plausible.Stats.Base doon: e.session_id == sq.session_id)else- if query.experimental_reduced_joins? do- events_q |> filter_by_visit_props(Filters.event_table_visit_props(), query)- else- events_q- end+ events_qendenddefp query_events(site, query) do- {first_datetime, last_datetime} = utc_boundaries(query, site)-- q =- from(- e in "events_v2",- where: e.site_id == ^site.id,- where: e.timestamp >= ^first_datetime and e.timestamp < ^last_datetime- )+ q = from(e in "events_v2", where: ^Filters.WhereBuilder.build(:events, site, query))on_ee doq = Plausible.Stats.Sampling.add_query_hint(q, query)end- q =- q- |> where([e], ^dynamic_filter_condition(query, "event:page", :pathname))- |> where([e], ^dynamic_filter_condition(query, "event:hostname", :hostname))-- q =- case query.filters["event:name"] do- {:is, name} ->- from(e in q, where: e.name == ^name)-- {:member, list} ->- from(e in q, where: e.name in ^list)-- nil ->- q- end-- q =- case query.filters["event:goal"] do- {:is, {:page, path}} ->- from(e in q, where: e.pathname == ^path and e.name == "pageview")-- {:matches, {:page, expr}} ->- regex = page_regex(expr)-- from(e in q,- where: fragment("match(?, ?)", e.pathname, ^regex) and e.name == "pageview"- )-- {:is, {:event, event}} ->- from(e in q, where: e.name == ^event)-- {:member, clauses} ->- {events, pages} = split_goals(clauses)-- from(e in q,- where: (e.pathname in ^pages and e.name == "pageview") or e.name in ^events- )-- {:matches_member, clauses} ->- {events, pages} = split_goals(clauses, &page_regex/1)-- event_clause =- if Enum.any?(events) do- dynamic([x], fragment("multiMatchAny(?, ?)", x.name, ^events))- else- dynamic([x], false)- end-- page_clause =- if Enum.any?(pages) do- dynamic(- [x],- fragment("multiMatchAny(?, ?)", x.pathname, ^pages) and x.name == "pageview"- )- else- dynamic([x], false)- end-- where_clause = dynamic([], ^event_clause or ^page_clause)-- from(e in q, where: ^where_clause)-- nil ->- q- end-- q =- Enum.reduce(- Query.get_all_filters_by_prefix(query, "event:props"),- q,- &filter_by_custom_prop/2- )-qenddef query_sessions(site, query) do- {first_datetime, last_datetime} =- utc_boundaries(query, site)-- q = from(s in "sessions_v2", where: s.site_id == ^site.id)-- sessions_q =- if query.experimental_session_count? do- from s in q, where: s.timestamp >= ^first_datetime and s.start < ^last_datetime- else- from s in q, where: s.start >= ^first_datetime and s.start < ^last_datetime- end+ q = from(s in "sessions_v2", where: ^Filters.WhereBuilder.build(:sessions, site, query))on_ee do- sessions_q = Plausible.Stats.Sampling.add_query_hint(sessions_q, query)- end-- filter_by_entry_props(sessions_q, query)- |> filter_by_visit_props(Filters.visit_props(), query)- end-- defp filter_by_visit_props(q, visit_props, query) do- Enum.reduce(visit_props, q, fn prop_name, sessions_q ->- filter_key = "visit:" <> prop_name- db_field = String.to_existing_atom(prop_name)-- from(s in sessions_q,- where: ^dynamic_filter_condition(query, filter_key, db_field)- )- end)- end-- def filter_by_entry_props(sessions_q, query) do- case Query.get_filter_by_prefix(query, "visit:entry_props:") do- nil ->- sessions_q-- {"visit:entry_props:" <> prop_name, filter_value} ->- apply_entry_prop_filter(sessions_q, prop_name, filter_value)+ q = Plausible.Stats.Sampling.add_query_hint(q, query)end- end-- def apply_entry_prop_filter(sessions_q, prop_name, {:is, "(none)"}) do- from(- s in sessions_q,- where: not has_key(s, :entry_meta, ^prop_name)- )- end-- def apply_entry_prop_filter(sessions_q, prop_name, {:is, value}) do- from(- s in sessions_q,- where:- has_key(s, :entry_meta, ^prop_name) and get_by_key(s, :entry_meta, ^prop_name) == ^value- )- end- def apply_entry_prop_filter(sessions_q, prop_name, {:is_not, "(none)"}) do- from(- s in sessions_q,- where: has_key(s, :entry_meta, ^prop_name)- )- end-- def apply_entry_prop_filter(sessions_q, prop_name, {:is_not, value}) do- from(- s in sessions_q,- where:- not has_key(s, :entry_meta, ^prop_name) or- get_by_key(s, :entry_meta, ^prop_name) != ^value- )+ qend- def apply_entry_prop_filter(sessions_q, _, _), do: sessions_q-def select_event_metrics(metrics) dometrics|> Enum.map(&select_event_metric/1)@@ -272,7 +119,9 @@ defmodule Plausible.Stats.Base doenddefp select_session_metric(:bounce_rate, query) do- condition = dynamic_filter_condition(query, "event:page", :entry_page)+ # :TRICKY: If page is passed to query, we only count bounce rate where users _entered_ at page.+ event_page_filter = Query.get_filter(query, "event:page")+ condition = Filters.WhereBuilder.build_condition(:entry_page, event_page_filter)%{bounce_rate:@@ -358,45 +207,6 @@ defmodule Plausible.Stats.Base dodefp select_session_metric(:percentage, _query), do: %{}- def dynamic_filter_condition(query, filter_key, db_field) do- case query && query.filters && query.filters[filter_key] do- {:is, value} ->- value = db_field_val(db_field, value)- dynamic([x], field(x, ^db_field) == ^value)-- {:is_not, value} ->- value = db_field_val(db_field, value)- dynamic([x], field(x, ^db_field) != ^value)-- {:matches_member, glob_exprs} ->- page_regexes = Enum.map(glob_exprs, &page_regex/1)- dynamic([x], fragment("multiMatchAny(?, ?)", field(x, ^db_field), ^page_regexes))-- {:not_matches_member, glob_exprs} ->- page_regexes = Enum.map(glob_exprs, &page_regex/1)- dynamic([x], fragment("not(multiMatchAny(?, ?))", field(x, ^db_field), ^page_regexes))-- {:matches, glob_expr} ->- regex = page_regex(glob_expr)- dynamic([x], fragment("match(?, ?)", field(x, ^db_field), ^regex))-- {:does_not_match, glob_expr} ->- regex = page_regex(glob_expr)- dynamic([x], fragment("not(match(?, ?))", field(x, ^db_field), ^regex))-- {:member, list} ->- list = Enum.map(list, &db_field_val(db_field, &1))- dynamic([x], field(x, ^db_field) in ^list)-- {:not_member, list} ->- list = Enum.map(list, &db_field_val(db_field, &1))- dynamic([x], field(x, ^db_field) not in ^list)-- nil ->- true- end- end-def filter_converted_sessions(db_query, site, query) doif Query.has_event_filters?(query) doconverted_sessions =@@ -416,16 +226,6 @@ defmodule Plausible.Stats.Base doendend- defp db_field_val(:source, @no_ref), do: ""- defp db_field_val(:referrer, @no_ref), do: ""- defp db_field_val(:utm_medium, @no_ref), do: ""- defp db_field_val(:utm_source, @no_ref), do: ""- defp db_field_val(:utm_campaign, @no_ref), do: ""- defp db_field_val(:utm_content, @no_ref), do: ""- defp db_field_val(:utm_term, @no_ref), do: ""- defp db_field_val(_, @not_set), do: ""- defp db_field_val(_, val), do: val-defp beginning_of_time(candidate, native_stats_start_at) doif Timex.after?(native_stats_start_at, candidate) donative_stats_start_at@@ -486,92 +286,6 @@ defmodule Plausible.Stats.Base do"^#{escaped}$"end- defp split_goals(clauses, map_fn \\ &Function.identity/1) do- groups =- Enum.group_by(clauses, fn {goal_type, _v} -> goal_type end, fn {_k, val} -> map_fn.(val) end)-- {- Map.get(groups, :event, []),- Map.get(groups, :page, [])- }- end-- defp filter_by_custom_prop({"event:props:" <> prop_name, {:is, "(none)"}}, q) do- from(- e in q,- where: not has_key(e, :meta, ^prop_name)- )- end-- defp filter_by_custom_prop({"event:props:" <> prop_name, {:is, value}}, q) do- from(- e in q,- where: has_key(e, :meta, ^prop_name) and get_by_key(e, :meta, ^prop_name) == ^value- )- end-- defp filter_by_custom_prop({"event:props:" <> prop_name, {:is_not, "(none)"}}, q) do- from(- e in q,- where: has_key(e, :meta, ^prop_name)- )- end-- defp filter_by_custom_prop({"event:props:" <> prop_name, {:is_not, value}}, q) do- from(- e in q,- where: not has_key(e, :meta, ^prop_name) or get_by_key(e, :meta, ^prop_name) != ^value- )- end-- defp filter_by_custom_prop({"event:props:" <> prop_name, {:matches, value}}, q) do- regex = page_regex(value)-- from(- e in q,- where:- has_key(e, :meta, ^prop_name) and- fragment("match(?, ?)", get_by_key(e, :meta, ^prop_name), ^regex)- )- end-- defp filter_by_custom_prop({"event:props:" <> prop_name, {:member, values}}, q) do- none_value_included = Enum.member?(values, "(none)")-- from(- e in q,- where:- (has_key(e, :meta, ^prop_name) and get_by_key(e, :meta, ^prop_name) in ^values) or- (^none_value_included and not has_key(e, :meta, ^prop_name))- )- end-- defp filter_by_custom_prop({"event:props:" <> prop_name, {:not_member, values}}, q) do- none_value_included = Enum.member?(values, "(none)")-- from(- e in q,- where:- (has_key(e, :meta, ^prop_name) and- get_by_key(e, :meta, ^prop_name) not in ^values) or- (^none_value_included and- has_key(e, :meta, ^prop_name) and- get_by_key(e, :meta, ^prop_name) not in ^values) or- (not (^none_value_included) and not has_key(e, :meta, ^prop_name))- )- end-- defp filter_by_custom_prop({"event:props:" <> prop_name, {:matches_member, clauses}}, q) do- regexes = Enum.map(clauses, &page_regex/1)-- from(- e in q,- where:- has_key(e, :meta, ^prop_name) and- fragment("arrayExists(k -> match(?, k), ?)", get_by_key(e, :meta, ^prop_name), ^regexes)- )- end-defp total_visitors(site, query) dobase_event_query(site, query)|> select([e], total_visitors: fragment(@uniq_users_expression, e.user_id))@@ -615,7 +329,7 @@ defmodule Plausible.Stats.Base do# filters.def maybe_add_conversion_rate(q, site, query, metrics) doif :conversion_rate in metrics do- total_query = query |> Query.remove_event_filters([:goal, :props])+ total_query = query |> Query.remove_filters(["event:goal", "event:props"])# :TRICKY: Subquery is used due to event:goal breakdown above doing an UNION ALLsubquery(q)diff --git a/lib/plausible/stats/breakdown.ex b/lib/plausible/stats/breakdown.exindex 566da8a071..305d80838b 100644--- a/lib/plausible/stats/breakdown.ex+++ b/lib/plausible/stats/breakdown.ex@@ -39,7 +39,7 @@ defmodule Plausible.Stats.Breakdown doevent_query = %Query{query- | filters: Map.put(query.filters, "event:name", {:member, events}),+ | filters: query.filters ++ [[:member, "event:name", events]],property: "event:name"}@@ -180,7 +180,7 @@ defmodule Plausible.Stats.Breakdown dopages ->query- |> Query.put_filter("visit:entry_page", {:member, Enum.map(pages, & &1[:page])})+ |> Query.put_filter([:member, "visit:entry_page", Enum.map(pages, & &1[:page])])|> struct!(property: "visit:entry_page")end@@ -256,12 +256,12 @@ defmodule Plausible.Stats.Breakdown doenddefp update_hostname(query, visit_prop) do- case query.filters["event:hostname"] do+ case Query.get_filter(query, "event:hostname") donil ->query- some ->- Plausible.Stats.Query.put_filter(query, visit_prop, some)+ [op, "event:hostname", value] ->+ Plausible.Stats.Query.put_filter(query, [op, visit_prop, value])endend@@ -353,7 +353,7 @@ defmodule Plausible.Stats.Breakdown doimport Ecto.Querywindowed_pages_q =- from e in base_event_query(site, Query.remove_event_filters(query, [:page, :props])),+ from e in base_event_query(site, Query.remove_filters(query, ["event:page", "event:props"])),select: %{next_timestamp: over(fragment("leadInFrame(?)", e.timestamp), :event_horizon),next_pathname: over(fragment("leadInFrame(?)", e.pathname), :event_horizon),@@ -719,7 +719,8 @@ defmodule Plausible.Stats.Breakdown dometrics) doif :conversion_rate in metrics do- breakdown_total_visitors_query = query |> Query.remove_event_filters([:goal, :props])+ breakdown_total_visitors_query =+ query |> Query.remove_filters(["event:goal", "event:props"])breakdown_total_visitors_q =breakdown_fn.(site, breakdown_total_visitors_query, [:visitors])diff --git a/lib/plausible/stats/custom_props.ex b/lib/plausible/stats/custom_props.exindex 8fe69b9d44..5c6df48eb4 100644--- a/lib/plausible/stats/custom_props.ex+++ b/lib/plausible/stats/custom_props.ex@@ -9,7 +9,7 @@ defmodule Plausible.Stats.CustomProps dodef fetch_prop_names(site, query) docase Query.get_filter_by_prefix(query, "event:props:") do- {"event:props:" <> key, _} ->+ [_op, "event:props:" <> key | _rest] ->[key]_ ->diff --git a/lib/plausible/stats/email_report.ex b/lib/plausible/stats/email_report.exindex e446e45326..63809e3af8 100644--- a/lib/plausible/stats/email_report.ex+++ b/lib/plausible/stats/email_report.ex@@ -48,7 +48,7 @@ defmodule Plausible.Stats.EmailReport dodefp put_top_5_sources(stats, site, query) doquery =query- |> Query.put_filter("visit:source", {:is_not, "Direct / None"})+ |> Query.put_filter([:is_not, "visit:source", "Direct / None"])|> struct!(property: "visit:source")sources = Stats.breakdown(site, query, [:visitors], {5, 1})diff --git a/lib/plausible/stats/filter_suggestions.ex b/lib/plausible/stats/filter_suggestions.exindex ee46392dae..6e38044f99 100644--- a/lib/plausible/stats/filter_suggestions.ex+++ b/lib/plausible/stats/filter_suggestions.ex@@ -152,10 +152,10 @@ defmodule Plausible.Stats.FilterSuggestions dodef filter_suggestions(site, query, "prop_value", filter_search) dofilter_query = if filter_search == nil, do: "%", else: "%#{filter_search}%"- {"event:props:" <> key, _filter} = Query.get_filter_by_prefix(query, "event:props")+ [_op, "event:props:" <> key | _rest] = Query.get_filter_by_prefix(query, "event:props")none_q =- from(e in base_event_query(site, Query.remove_event_filters(query, [:props])),+ from(e in base_event_query(site, Query.remove_filters(query, ["event:props"])),select: "(none)",where: not has_key(e, :meta, ^key),limit: 1diff --git a/lib/plausible/stats/filters/dashboard_filter_parser.ex b/lib/plausible/stats/filters/dashboard_filter_parser.exindex e1f80811c1..e57d1d49cb 100644--- a/lib/plausible/stats/filters/dashboard_filter_parser.ex+++ b/lib/plausible/stats/filters/dashboard_filter_parser.ex@@ -9,52 +9,76 @@ defmodule Plausible.Stats.Filters.DashboardFilterParser dothe internal React dashboard API"""def parse_and_prefix(filters_map) do- Enum.reduce(filters_map, %{}, fn {name, val}, new_filters ->+ Enum.flat_map(filters_map, fn {name, val} ->cond doname in Filters.visit_props() ->- Map.put(new_filters, "visit:" <> name, filter_value(name, val))+ [filter_value("visit:" <> name, val)]name in Filters.event_props() ->- Map.put(new_filters, "event:" <> name, filter_value(name, val))+ [filter_value("event:" <> name, val)]name == "props" ->- put_parsed_props(new_filters, name, val)+ parse_props(val)true ->- new_filters+ []endend)end- @spec filter_value(String.t(), String.t()) :: {atom(), String.t() | [String.t()]}def filter_value(key, val) do{is_negated, val} = parse_negated_prefix(val){is_contains, val} = parse_contains_prefix(val)is_list = list_expression?(val)is_wildcard = String.contains?(key, ["page", "goal", "hostname"]) && wildcard_expression?(val)val = if is_list, do: parse_member_list(val), else: remove_escape_chars(val)- val = if key == "goal", do: wrap_goal_value(val), else: val+ val = if key == "event:goal", do: wrap_goal_value(val), else: valcond do- is_negated && is_wildcard && is_list -> {:not_matches_member, val}- is_negated && is_contains && is_list -> {:not_matches_member, Enum.map(val, &"**#{&1}**")}- is_wildcard && is_list -> {:matches_member, val}- is_negated && is_wildcard -> {:does_not_match, val}- is_negated && is_list -> {:not_member, val}- is_negated && is_contains -> {:does_not_match, "**" <> val <> "**"}- is_contains && is_list -> {:matches_member, Enum.map(val, &"**#{&1}**")}- is_wildcard && is_list -> {:matches_member, val}- is_negated -> {:is_not, val}- is_list -> {:member, val}- is_contains -> {:matches, "**" <> val <> "**"}- is_wildcard -> {:matches, val}- true -> {:is, val}+ is_negated && is_wildcard && is_list ->+ [:not_matches_member, key, val]++ is_negated && is_contains && is_list ->+ [:not_matches_member, key, Enum.map(val, &"**#{&1}**")]++ is_wildcard && is_list ->+ [:matches_member, key, val]++ is_negated && is_wildcard ->+ [:does_not_match, key, val]++ is_negated && is_list ->+ [:not_member, key, val]++ is_negated && is_contains ->+ [:does_not_match, key, "**" <> val <> "**"]++ is_contains && is_list ->+ [:matches_member, key, Enum.map(val, &"**#{&1}**")]++ is_wildcard && is_list ->+ [:matches_member, key, val]++ is_negated ->+ [:is_not, key, val]++ is_list ->+ [:member, key, val]++ is_contains ->+ [:matches, key, "**" <> val <> "**"]++ is_wildcard ->+ [:matches, key, val]++ true ->+ [:is, key, val]endend- defp put_parsed_props(new_filters, name, val) do- Enum.reduce(val, new_filters, fn {prop_key, prop_val}, new_filters ->- Map.put(new_filters, "event:props:" <> prop_key, filter_value(name, prop_val))+ defp parse_props(val) do+ Enum.map(val, fn {prop_key, prop_val} ->+ filter_value("event:props:" <> prop_key, prop_val)end)enddiff --git a/lib/plausible/stats/filters/filters.ex b/lib/plausible/stats/filters/filters.exindex 7323696864..3775e73435 100644--- a/lib/plausible/stats/filters/filters.ex+++ b/lib/plausible/stats/filters/filters.ex@@ -56,24 +56,24 @@ defmodule Plausible.Stats.Filters do### Examples:iex> Filters.parse("{\\"page\\":\\"/blog/**\\"}")- %{"event:page" => {:matches, "/blog/**"}}+ [[:matches, "event:page", "/blog/**"]]iex> Filters.parse("visit:browser!=Chrome")- %{"visit:browser" => {:is_not, "Chrome"}}+ [[:is_not, "visit:browser", "Chrome"]]iex> Filters.parse(nil)- %{}+ []"""def parse(filters) when is_binary(filters) docase Jason.decode(filters) do{:ok, filters} when is_map(filters) -> DashboardFilterParser.parse_and_prefix(filters)- {:ok, _} -> %{}+ {:ok, _} -> []{:error, err} -> StatsAPIFilterParser.parse_filter_expression(err.data)endenddef parse(filters) when is_map(filters), do: DashboardFilterParser.parse_and_prefix(filters)- def parse(_), do: %{}+ def parse(_), do: []def without_prefix(property) dopropertydiff --git a/lib/plausible/stats/filters/stats_api_filter_parser.ex b/lib/plausible/stats/filters/stats_api_filter_parser.exindex b946e5950a..fc58168993 100644--- a/lib/plausible/stats/filters/stats_api_filter_parser.ex+++ b/lib/plausible/stats/filters/stats_api_filter_parser.ex@@ -12,13 +12,12 @@ defmodule Plausible.Stats.Filters.StatsAPIFilterParser doEnum.map(filters, &parse_single_filter/1)|> Enum.reject(fn parsed -> parsed == :error end)- |> Enum.into(%{})enddefp parse_single_filter(str) docase to_kv(str) do["event:goal" = key, raw_value] ->- {key, parse_goal_filter(raw_value)}+ parse_goal_filter(key, raw_value)[key, raw_value] ->is_negated? = String.contains?(str, "!=")@@ -28,11 +27,11 @@ defmodule Plausible.Stats.Filters.StatsAPIFilterParser dofinal_value = remove_escape_chars(raw_value)cond do- is_wildcard? && is_negated? -> {key, {:does_not_match, raw_value}}- is_wildcard? -> {key, {:matches, raw_value}}- is_list? -> {key, {:member, parse_member_list(raw_value)}}- is_negated? -> {key, {:is_not, final_value}}- true -> {key, {:is, final_value}}+ is_wildcard? && is_negated? -> [:does_not_match, key, raw_value]+ is_wildcard? -> [:matches, key, raw_value]+ is_list? -> [:member, key, parse_member_list(raw_value)]+ is_negated? -> [:is_not, key, final_value]+ true -> [:is, key, final_value]end|> reject_invalid_country_codes()@@ -41,7 +40,7 @@ defmodule Plausible.Stats.Filters.StatsAPIFilterParser doendend- defp reject_invalid_country_codes({"visit:country", {_, code_or_codes}} = filter) do+ defp reject_invalid_country_codes([_op, "visit:country", code_or_codes] = filter) docode_or_codes|> List.wrap()|> Enum.reduce_while(filter, fn@@ -59,7 +58,7 @@ defmodule Plausible.Stats.Filters.StatsAPIFilterParser do|> Enum.map(&String.trim/1)end- defp parse_goal_filter(value) do+ defp parse_goal_filter(key, value) dois_list? = list_expression?(value)is_wildcard? = wildcard_expression?(value)@@ -72,10 +71,10 @@ defmodule Plausible.Stats.Filters.StatsAPIFilterParser do|> wrap_goal_value()cond do- is_list? && is_wildcard? -> {:matches_member, value}- is_list? -> {:member, value}- is_wildcard? -> {:matches, value}- true -> {:is, value}+ is_list? && is_wildcard? -> [:matches_member, key, value]+ is_list? -> [:member, key, value]+ is_wildcard? -> [:matches, key, value]+ true -> [:is, key, value]endendenddiff --git a/lib/plausible/stats/filters/where_builder.ex b/lib/plausible/stats/filters/where_builder.exnew file mode 100644index 0000000000..04f66c8363--- /dev/null+++ b/lib/plausible/stats/filters/where_builder.ex@@ -0,0 +1,304 @@+defmodule Plausible.Stats.Filters.WhereBuilder do+ @moduledoc """+ A module for building am ecto where clause of a query out of a query.+ """++ import Ecto.Query+ import Plausible.Stats.Base, only: [page_regex: 1, utc_boundaries: 2]++ alias Plausible.Stats.Query++ use Plausible.Stats.Fragments++ @sessions_only_visit_fields [+ :entry_page,+ :exit_page,+ :entry_page_hostname,+ :exit_page_hostname+ ]++ @doc "Builds WHERE clause for a given Query against sessions or events table"+ def build(table, site, query) do+ base_condition = filter_site_time_range(table, site, query)++ query.filters+ |> Enum.map(&add_filter(table, query, &1))+ |> Enum.reduce(base_condition, fn condition, acc -> dynamic([], ^acc and ^condition) end)+ end++ @doc """+ Builds WHERE clause condition based off of a filter and a custom column name+ Used for special business logic cases++ Accepts nil as the `filter` parameter, in which case the condition is a no-op (WHERE TRUE).+ """+ def build_condition(db_field, filter) do+ if filter do+ filter_field(db_field, filter)+ else+ true+ end+ end++ defp filter_site_time_range(:events, site, query) do+ {first_datetime, last_datetime} = utc_boundaries(query, site)++ dynamic(+ [e],+ e.site_id == ^site.id and e.timestamp >= ^first_datetime and e.timestamp < ^last_datetime+ )+ end++ defp filter_site_time_range(:sessions, site, %Query{experimental_session_count?: true} = query) do+ {first_datetime, last_datetime} = utc_boundaries(query, site)++ # Counts each _active_ session in time range even if they started before+ dynamic(+ [s],+ s.site_id == ^site.id and s.timestamp >= ^first_datetime and s.start < ^last_datetime+ )+ end++ defp filter_site_time_range(:sessions, site, query) do+ {first_datetime, last_datetime} = utc_boundaries(query, site)++ dynamic(+ [s],+ s.site_id == ^site.id and s.start >= ^first_datetime and s.start < ^last_datetime+ )+ end++ defp add_filter(:events, _query, [:is, "event:name", name]) do+ dynamic([e], e.name == ^name)+ end++ defp add_filter(:events, _query, [:member, "event:name", list]) do+ dynamic([e], e.name in ^list)+ end++ defp add_filter(:events, _query, [:is, "event:goal", {:page, path}]) do+ dynamic([e], e.pathname == ^path and e.name == "pageview")+ end++ defp add_filter(:events, _query, [:matches, "event:goal", {:page, expr}]) do+ regex = page_regex(expr)++ dynamic([e], fragment("match(?, ?)", e.pathname, ^regex) and e.name == "pageview")+ end++ defp add_filter(:events, _query, [:is, "event:goal", {:event, event}]) do+ dynamic([e], e.name == ^event)+ end++ defp add_filter(:events, _query, [:member, "event:goal", clauses]) do+ {events, pages} = split_goals(clauses)++ dynamic([e], (e.pathname in ^pages and e.name == "pageview") or e.name in ^events)+ end++ defp add_filter(:events, _query, [:matches_member, "event:goal", clauses]) do+ {events, pages} = split_goals(clauses, &page_regex/1)++ event_clause =+ if Enum.any?(events) do+ dynamic([x], fragment("multiMatchAny(?, ?)", x.name, ^events))+ else+ dynamic([x], false)+ end++ page_clause =+ if Enum.any?(pages) do+ dynamic(+ [x],+ fragment("multiMatchAny(?, ?)", x.pathname, ^pages) and x.name == "pageview"+ )+ else+ dynamic([x], false)+ end++ where_clause = dynamic([], ^event_clause or ^page_clause)++ dynamic([e], ^where_clause)+ end++ defp add_filter(:events, _query, [_, "event:page" | _rest] = filter) do+ filter_field(:pathname, filter)+ end++ defp add_filter(:events, _query, [_, "event:hostname" | _rest] = filter) do+ filter_field(:hostname, filter)+ end++ defp add_filter(:events, _query, [_, "event:props:" <> prop_name | _rest] = filter) do+ filter_custom_prop(prop_name, :meta, filter)+ end++ defp add_filter(:events, _query, [_, "visit:entry_props:" <> _prop_name | _rest]) do+ true+ end++ defp add_filter(+ :events,+ %Query{experimental_reduced_joins?: true},+ [_, "visit:" <> key | _rest] = filter+ ) do+ # Filter events query if experimental_reduced_joins? is true+ field_name = String.to_existing_atom(key)++ if Enum.member?(@sessions_only_visit_fields, field_name) do+ true+ else+ filter_field(field_name, filter)+ end+ end++ defp add_filter(:events, _query, [_, "visit:" <> _key | _rest]) do+ true+ end++ defp add_filter(:sessions, _query, [_, "visit:entry_props:" <> prop_name | _rest] = filter) do+ filter_custom_prop(prop_name, :entry_meta, filter)+ end++ defp add_filter(:sessions, _query, [_, "visit:" <> key | _rest] = filter) do+ filter_field(String.to_existing_atom(key), filter)+ end++ defp add_filter(:sessions, _query, [_, "event:" <> _ | _rest]) do+ # Cannot apply sessions filters directly on session query where clause.+ true+ end++ defp filter_custom_prop(prop_name, column_name, [:is, _, "(none)"]) do+ dynamic([t], not has_key(t, column_name, ^prop_name))+ end++ defp filter_custom_prop(prop_name, column_name, [:is, _, value]) do+ dynamic(+ [t],+ has_key(t, column_name, ^prop_name) and get_by_key(t, column_name, ^prop_name) == ^value+ )+ end++ defp filter_custom_prop(prop_name, column_name, [:is_not, _, "(none)"]) do+ dynamic([t], has_key(t, column_name, ^prop_name))+ end++ defp filter_custom_prop(prop_name, column_name, [:is_not, _, value]) do+ dynamic(+ [t],+ not has_key(t, column_name, ^prop_name) or get_by_key(t, column_name, ^prop_name) != ^value+ )+ end++ defp filter_custom_prop(prop_name, column_name, [:matches, _, value]) do+ regex = page_regex(value)++ dynamic(+ [t],+ has_key(t, column_name, ^prop_name) and+ fragment("match(?, ?)", get_by_key(t, column_name, ^prop_name), ^regex)+ )+ end++ defp filter_custom_prop(prop_name, column_name, [:member, _, values]) do+ none_value_included = Enum.member?(values, "(none)")++ dynamic(+ [t],+ (has_key(t, column_name, ^prop_name) and get_by_key(t, column_name, ^prop_name) in ^values) or+ (^none_value_included and not has_key(t, column_name, ^prop_name))+ )+ end++ defp filter_custom_prop(prop_name, column_name, [:not_member, _, values]) do+ none_value_included = Enum.member?(values, "(none)")++ dynamic(+ [t],+ (has_key(t, column_name, ^prop_name) and+ get_by_key(t, column_name, ^prop_name) not in ^values) or+ (^none_value_included and+ has_key(t, column_name, ^prop_name) and+ get_by_key(t, column_name, ^prop_name) not in ^values) or+ (not (^none_value_included) and not has_key(t, column_name, ^prop_name))+ )+ end++ defp filter_custom_prop(prop_name, column_name, [:matches_member, _, clauses]) do+ regexes = Enum.map(clauses, &page_regex/1)++ dynamic(+ [t],+ has_key(t, column_name, ^prop_name) and+ fragment(+ "arrayExists(k -> match(?, k), ?)",+ get_by_key(t, column_name, ^prop_name),+ ^regexes+ )+ )+ end++ defp filter_field(db_field, [:is, _key, value]) do+ value = db_field_val(db_field, value)+ dynamic([x], field(x, ^db_field) == ^value)+ end++ defp filter_field(db_field, [:is_not, _key, value]) do+ value = db_field_val(db_field, value)+ dynamic([x], field(x, ^db_field) != ^value)+ end++ defp filter_field(db_field, [:matches_member, _key, glob_exprs]) do+ page_regexes = Enum.map(glob_exprs, &page_regex/1)+ dynamic([x], fragment("multiMatchAny(?, ?)", field(x, ^db_field), ^page_regexes))+ end++ defp filter_field(db_field, [:not_matches_member, _key, glob_exprs]) do+ page_regexes = Enum.map(glob_exprs, &page_regex/1)+ dynamic([x], fragment("not(multiMatchAny(?, ?))", field(x, ^db_field), ^page_regexes))+ end++ defp filter_field(db_field, [:matches, _key, glob_expr]) do+ regex = page_regex(glob_expr)+ dynamic([x], fragment("match(?, ?)", field(x, ^db_field), ^regex))+ end++ defp filter_field(db_field, [:does_not_match, _key, glob_expr]) do+ regex = page_regex(glob_expr)+ dynamic([x], fragment("not(match(?, ?))", field(x, ^db_field), ^regex))+ end++ defp filter_field(db_field, [:member, _key, list]) do+ list = Enum.map(list, &db_field_val(db_field, &1))+ dynamic([x], field(x, ^db_field) in ^list)+ end++ defp filter_field(db_field, [:not_member, _key, list]) do+ list = Enum.map(list, &db_field_val(db_field, &1))+ dynamic([x], field(x, ^db_field) not in ^list)+ end++ @no_ref "Direct / None"+ @not_set "(not set)"++ defp db_field_val(:source, @no_ref), do: ""+ defp db_field_val(:referrer, @no_ref), do: ""+ defp db_field_val(:utm_medium, @no_ref), do: ""+ defp db_field_val(:utm_source, @no_ref), do: ""+ defp db_field_val(:utm_campaign, @no_ref), do: ""+ defp db_field_val(:utm_content, @no_ref), do: ""+ defp db_field_val(:utm_term, @no_ref), do: ""+ defp db_field_val(_, @not_set), do: ""+ defp db_field_val(_, val), do: val++ defp split_goals(clauses, map_fn \\ &Function.identity/1) do+ groups =+ Enum.group_by(clauses, fn {goal_type, _v} -> goal_type end, fn {_k, val} -> map_fn.(val) end)++ {+ Map.get(groups, :event, []),+ Map.get(groups, :page, [])+ }+ end+enddiff --git a/lib/plausible/stats/fragments.ex b/lib/plausible/stats/fragments.exindex ae5ef4e44c..27f49839f5 100644--- a/lib/plausible/stats/fragments.ex+++ b/lib/plausible/stats/fragments.ex@@ -110,7 +110,7 @@ defmodule Plausible.Stats.Fragments doquote dofragment("has(?, ?)",- field(unquote(table), unquote(meta_key_column(meta_column))),+ field(unquote(table), ^meta_key_column(unquote(meta_column))),unquote(key))end@@ -130,18 +130,18 @@ defmodule Plausible.Stats.Fragments doquote dofragment("?[indexOf(?, ?)]",- field(unquote(table), unquote(meta_value_column(meta_column))),- field(unquote(table), unquote(meta_key_column(meta_column))),+ field(unquote(table), ^meta_value_column(unquote(meta_column))),+ field(unquote(table), ^meta_key_column(unquote(meta_column))),unquote(key))endend- defp meta_key_column(:meta), do: :"meta.key"- defp meta_key_column(:entry_meta), do: :"entry_meta.key"+ def meta_key_column(:meta), do: :"meta.key"+ def meta_key_column(:entry_meta), do: :"entry_meta.key"- defp meta_value_column(:meta), do: :"meta.value"- defp meta_value_column(:entry_meta), do: :"entry_meta.value"+ def meta_value_column(:meta), do: :"meta.value"+ def meta_value_column(:entry_meta), do: :"entry_meta.value"defmacro __using__(_) doquote dodiff --git a/lib/plausible/stats/imported.ex b/lib/plausible/stats/imported.exindex 9ab3dd703e..3c43c17f85 100644--- a/lib/plausible/stats/imported.ex+++ b/lib/plausible/stats/imported.ex@@ -33,27 +33,29 @@ defmodule Plausible.Stats.Imported do@imported_properties Map.keys(@property_to_table_mappings)+ @goals_with_url ["Outbound Link: Click", "Cloaked Link: Click", "File Download"]++ @doc """+ Returns a boolean indicating whether the combination of filters and+ breakdown property is possible to query from the imported tables.++ Usually, when no filters are used, the imported schema supports the+ query. There is one exception though - breakdown by a custom property.+ We are currently importing only two custom properties - `url` and `path.+ Both these properties can only be used with their special goal filter+ (see `@goals_with_url` and `@goals_with_path`).+ """def schema_supports_query?(query) do- filter_count = length(Map.keys(query.filters))+ filter_count = length(query.filters)case {filter_count, query.property} do{0, "event:props:" <> _} -> false{0, _} -> true- {1, _} -> supports_single_filter?(query)+ {1, "event:props:url"} -> has_special_goal_filter?(query, @goals_with_url){_, _} -> falseendend- defp supports_single_filter?(%Query{- filters: %{"event:goal" => {:is, {:event, event}}},- property: "event:props:url"- })- when event in ["Outbound Link: Click", "File Download"] do- true- end-- defp supports_single_filter?(_query), do: false-def merge_imported_timeseries(native_q, _, %Plausible.Stats.Query{include_imported: false}, _),do: native_q@@ -118,7 +120,7 @@ defmodule Plausible.Stats.Imported dowhere: i.visitors > 0,select: %{})- |> maybe_apply_filter(query.filters, property, dim)+ |> maybe_apply_filter(query, property, dim)|> group_imported_by(dim)|> select_imported_metrics(metrics)@@ -214,23 +216,35 @@ defmodule Plausible.Stats.Imported do)end- defp maybe_apply_filter(- q,- %{"event:goal" => {:is, {:event, event_name}}},- "event:props:url",- _dim- )- when event_name in ["Outbound Link: Click", "File Download"] do- where(q, [i], i.name == ^event_name)+ defp maybe_apply_filter(q, query, "event:props:url", _) do+ if name = find_special_goal_filter(query, @goals_with_url) do+ where(q, [i], i.name == ^name)+ else+ q+ endend- defp maybe_apply_filter(q, filters, property, dim) do- case filters[property] do- {:member, list} -> where(q, [i], field(i, ^dim) in ^list)+ defp maybe_apply_filter(q, query, property, dim) do+ case Query.get_filter(query, property) do+ [:member, _, list] -> where(q, [i], field(i, ^dim) in ^list)_ -> qendend+ defp has_special_goal_filter?(query, event_names) do+ not is_nil(find_special_goal_filter(query, event_names))+ end++ defp find_special_goal_filter(query, event_names) do+ case Query.get_filter(query, "event:goal") do+ [:is, "event:goal", {:event, name}] ->+ if name in event_names, do: name, else: nil++ _ ->+ nil+ end+ end+defp select_imported_metrics(q, []), do: qdefp select_imported_metrics(q, [:visitors | rest]) dodiff --git a/lib/plausible/stats/query.ex b/lib/plausible/stats/query.exindex 5af5ede7f4..4fa8f4a94f 100644--- a/lib/plausible/stats/query.ex+++ b/lib/plausible/stats/query.ex@@ -5,7 +5,7 @@ defmodule Plausible.Stats.Query dointerval: nil,period: nil,property: nil,- filters: %{},+ filters: [],sample_threshold: 20_000_000,imported_data_requested: false,include_imported: false,@@ -200,50 +200,36 @@ defmodule Plausible.Stats.Query dostruct!(query, filters: Filters.parse(params["filters"]))end- def put_filter(query, key, val) do- parsed_val =- if is_binary(val) do- Filters.DashboardFilterParser.filter_value(key, val)- else- val- end-+ def put_filter(query, filter) dostruct!(query,- filters: Map.put(query.filters, key, parsed_val)+ filters: query.filters ++ [filter])end- def remove_event_filters(query, opts) do+ def remove_filters(query, prefixes) donew_filters =- Enum.filter(query.filters, fn {filter_key, _} ->- cond do- :page in opts && filter_key == "event:page" -> false- :goal in opts && filter_key == "event:goal" -> false- :props in opts && filter_key && String.starts_with?(filter_key, "event:props:") -> false- true -> true- end+ Enum.reject(query.filters, fn [_, filter_key | _rest] ->+ Enum.any?(prefixes, &String.starts_with?(filter_key, &1))end)- |> Enum.into(%{})struct!(query, filters: new_filters)enddef has_event_filters?(query) do- Enum.any?(query.filters, fn- {"event:" <> _, _} -> true- _ -> false+ Enum.any?(query.filters, fn [_op, prop | _rest] ->+ String.starts_with?(prop, "event:")end)enddef get_filter_by_prefix(query, prefix) do- Enum.find(query.filters, fn {prop, _value} ->+ Enum.find(query.filters, fn [_op, prop | _rest] ->String.starts_with?(prop, prefix)end)end- def get_all_filters_by_prefix(query, prefix) do- Enum.filter(query.filters, fn {prop, _value} ->- String.starts_with?(prop, prefix)+ def get_filter(query, name) do+ Enum.find(query.filters, fn [_op, prop | _rest] ->+ prop == nameend)end@@ -281,7 +267,12 @@ defmodule Plausible.Stats.Query do@spec trace(%__MODULE__{}, [atom()]) :: %__MODULE__{}def trace(%__MODULE__{} = query, metrics) do- filter_keys = Map.keys(query.filters) |> Enum.sort() |> Enum.join(";")+ filter_keys =+ query.filters+ |> Enum.map(fn [_op, prop | _rest] -> prop end)+ |> Enum.sort()+ |> Enum.join(";")+metrics = metrics |> Enum.sort() |> Enum.join(";")Tracer.set_attributes([diff --git a/lib/plausible/stats/table_decider.ex b/lib/plausible/stats/table_decider.exindex ef976a0604..8b77ca5928 100644--- a/lib/plausible/stats/table_decider.ex+++ b/lib/plausible/stats/table_decider.ex@@ -23,15 +23,15 @@ defmodule Plausible.Stats.TableDecider dopartition(metrics, query, &metric_partitioner/2)# Treat breakdown property as yet another filter- filters =+ query =if breakdown_property do- Map.put(query.filters, breakdown_property, nil)+ Query.put_filter(query, [:is, breakdown_property, []])else- query.filters+ queryend%{event: event_only_filters, session: session_only_filters} =- partition(filters, query, &filters_partitioner/2)+ partition(query.filters, query, &filters_partitioner/2)cond do# Only one table needs to be queried@@ -83,16 +83,16 @@ defmodule Plausible.Stats.TableDecider dodefp metric_partitioner(_, _), do: :either- defp filters_partitioner(_, {"event:" <> _, _}), do: :event- defp filters_partitioner(_, {"visit:entry_page", _}), do: :session- defp filters_partitioner(_, {"visit:entry_page_hostname", _}), do: :session- defp filters_partitioner(_, {"visit:exit_page", _}), do: :session- defp filters_partitioner(_, {"visit:exit_page_hostname", _}), do: :session+ defp filters_partitioner(_, [_, "event:" <> _ | _rest]), do: :event+ defp filters_partitioner(_, [_, "visit:entry_page" | _rest]), do: :session+ defp filters_partitioner(_, [_, "visit:entry_page_hostname" | _rest]), do: :session+ defp filters_partitioner(_, [_, "visit:exit_page" | _rest]), do: :session+ defp filters_partitioner(_, [_, "visit:exit_page_hostname" | _rest]), do: :session- defp filters_partitioner(%Query{experimental_reduced_joins?: true}, {"visit:" <> _, _}),+ defp filters_partitioner(%Query{experimental_reduced_joins?: true}, [_, "visit:" <> _ | _rest]),do: :either- defp filters_partitioner(_, {"visit:" <> _, _}),+ defp filters_partitioner(_, [_, "visit:" <> _ | _rest]),do: :sessiondefp filters_partitioner(%Query{experimental_reduced_joins?: false}, {unknown, _}) dodiff --git a/lib/plausible/stats/timeseries.ex b/lib/plausible/stats/timeseries.exindex 0ca145330b..08dfaf18bf 100644--- a/lib/plausible/stats/timeseries.ex+++ b/lib/plausible/stats/timeseries.ex@@ -314,7 +314,7 @@ defmodule Plausible.Stats.Timeseries dodefp maybe_add_timeseries_conversion_rate(q, site, query, metrics) doif :conversion_rate in metrics do- totals_query = query |> Query.remove_event_filters([:goal, :props])+ totals_query = query |> Query.remove_filters(["event:goal", "event:props"])totals_timeseries_q =from(e in base_event_query(site, totals_query),diff --git a/lib/plausible_web/controllers/api/external_stats_controller.ex b/lib/plausible_web/controllers/api/external_stats_controller.exindex 0623873993..a14abf6cd7 100644--- a/lib/plausible_web/controllers/api/external_stats_controller.ex+++ b/lib/plausible_web/controllers/api/external_stats_controller.ex@@ -119,7 +119,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController do{_, _, :all} ->true- {{"event:props:" <> prop, _}, _property, allowed_props} ->+ {[_, "event:props:" <> prop | _], _property, allowed_props} ->prop in allowed_props{_filter, "event:props:" <> prop, allowed_props} ->@@ -156,10 +156,10 @@ defmodule PlausibleWeb.Api.ExternalStatsController dodefp validate_metric("time_on_page" = metric, query) docond do- query.filters["event:goal"] ->+ Query.get_filter(query, "event:goal") ->{:error, "Metric `#{metric}` cannot be queried when filtering by `event:goal`"}- query.filters["event:name"] ->+ Query.get_filter(query, "event:name") ->{:error, "Metric `#{metric}` cannot be queried when filtering by `event:name`"}query.property == "event:page" ->@@ -169,7 +169,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController do{:error,"Metric `#{metric}` is not supported in breakdown queries (except `event:page` breakdown)"}- query.filters["event:page"] ->+ Query.get_filter(query, "event:page") ->{:ok, metric}true ->@@ -183,7 +183,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController doquery.property == "event:goal" ->{:ok, metric}- query.filters["event:goal"] ->+ Query.get_filter(query, "event:goal") ->{:ok, metric}true ->@@ -198,7 +198,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController dodefp validate_metric("views_per_visit" = metric, query) docond do- query.filters["event:page"] ->+ Query.get_filter(query, "event:page") ->{:error, "Metric `#{metric}` cannot be queried with a filter on `event:page`."}query.property != nil ->@@ -235,7 +235,9 @@ defmodule PlausibleWeb.Api.ExternalStatsController doenddefp find_event_only_filter(query) do- Map.keys(query.filters) |> Enum.find(&event_only_property?/1)+ query.filters+ |> Enum.map(fn [_op, prop | _] -> prop end)+ |> Enum.find(&event_only_property?/1)enddefp event_only_property?("event:name"), do: true@@ -327,7 +329,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController doend)end- defp validate_filter(site, {"event:goal", {_type, goal_filter}}) do+ defp validate_filter(site, [_type, "event:goal", goal_filter]) doconfigured_goals =Plausible.Goals.for_site(site)|> Enum.map(fn@@ -350,7 +352,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController doendend- defp validate_filter(_site, {property, _}) do+ defp validate_filter(_site, [_, property | _]) doif Plausible.Stats.Props.valid_prop?(property) do:okelsediff --git a/lib/plausible_web/controllers/api/stats_controller.ex b/lib/plausible_web/controllers/api/stats_controller.exindex dc2a562cdd..bc9086ca84 100644--- a/lib/plausible_web/controllers/api/stats_controller.ex+++ b/lib/plausible_web/controllers/api/stats_controller.ex@@ -6,6 +6,7 @@ defmodule PlausibleWeb.Api.StatsController doalias Plausible.Statsalias Plausible.Stats.{Query, Comparisons}+ alias Plausible.Stats.Filters.DashboardFilterParseralias PlausibleWeb.Api.Helpers, as: Hrequire Logger@@ -276,11 +277,25 @@ defmodule PlausibleWeb.Api.StatsController doendend- defp fetch_top_stats(- site,- %Query{period: "realtime", filters: %{"event:goal" => _goal}} = query,- _comparison_query- ) do+ defp fetch_top_stats(site, query, comparison_query) do+ goal_filter = Query.get_filter(query, "event:goal")++ cond do+ query.period == "realtime" && goal_filter ->+ fetch_goal_realtime_top_stats(site, query, comparison_query)++ query.period == "realtime" ->+ fetch_realtime_top_stats(site, query, comparison_query)++ goal_filter ->+ fetch_goal_top_stats(site, query, comparison_query)++ true ->+ fetch_other_top_stats(site, query, comparison_query)+ end+ end++ defp fetch_goal_realtime_top_stats(site, query, _comparison_query) doquery_30m = %Query{query | period: "30m"}%{@@ -308,7 +323,7 @@ defmodule PlausibleWeb.Api.StatsController do{stats, 100}end- defp fetch_top_stats(site, %Query{period: "realtime"} = query, _comparison_query) do+ defp fetch_realtime_top_stats(site, query, _comparison_query) doquery_30m = %Query{query | period: "30m"}%{@@ -336,7 +351,7 @@ defmodule PlausibleWeb.Api.StatsController do{stats, 100}end- defp fetch_top_stats(site, %Query{filters: %{"event:goal" => _}} = query, comparison_query) do+ defp fetch_goal_top_stats(site, query, comparison_query) dometrics =[:total_visitors, :visitors, :events, :conversion_rate] ++ @revenue_metrics@@ -365,9 +380,9 @@ defmodule PlausibleWeb.Api.StatsController do|> then(&{&1, 100})end- defp fetch_top_stats(site, query, comparison_query) do+ defp fetch_other_top_stats(site, query, comparison_query) dometrics =- if query.filters["event:page"] do+ if Query.get_filter(query, "event:page") do[:visitors,:visits,@@ -469,7 +484,7 @@ defmodule PlausibleWeb.Api.StatsController do|> transform_keys(%{source: :name})if params["csv"] do- if Map.has_key?(query.filters, "event:goal") do+ if Query.get_filter(query, "event:goal") dores|> transform_keys(%{visitors: :conversions})|> to_csv([:name, :conversions, :conversion_rate])@@ -519,17 +534,17 @@ defmodule PlausibleWeb.Api.StatsController doenddefp validate_funnel_query(query) do- case query do- _ when is_map_key(query.filters, "event:goal") ->+ cond do+ Query.get_filter(query, "event:goal") ->{:error, {:invalid_funnel_query, "goals"}}- _ when is_map_key(query.filters, "event:page") ->+ Query.get_filter(query, "event:page") ->{:error, {:invalid_funnel_query, "pages"}}- _ when query.period == "realtime" ->+ query.period == "realtime" ->{:error, {:invalid_funnel_query, "realtime period"}}- _ ->+ true ->:okendend@@ -547,7 +562,7 @@ defmodule PlausibleWeb.Api.StatsController do|> transform_keys(%{utm_medium: :name})if params["csv"] do- if Map.has_key?(query.filters, "event:goal") do+ if Query.get_filter(query, "event:goal") dores|> transform_keys(%{visitors: :conversions})|> to_csv([:name, :conversions, :conversion_rate])@@ -571,7 +586,7 @@ defmodule PlausibleWeb.Api.StatsController do|> transform_keys(%{utm_campaign: :name})if params["csv"] do- if Map.has_key?(query.filters, "event:goal") do+ if Query.get_filter(query, "event:goal") dores|> transform_keys(%{visitors: :conversions})|> to_csv([:name, :conversions, :conversion_rate])@@ -595,7 +610,7 @@ defmodule PlausibleWeb.Api.StatsController do|> transform_keys(%{utm_content: :name})if params["csv"] do- if Map.has_key?(query.filters, "event:goal") do+ if Query.get_filter(query, "event:goal") dores|> transform_keys(%{visitors: :conversions})|> to_csv([:name, :conversions, :conversion_rate])@@ -619,7 +634,7 @@ defmodule PlausibleWeb.Api.StatsController do|> transform_keys(%{utm_term: :name})if params["csv"] do- if Map.has_key?(query.filters, "event:goal") do+ if Query.get_filter(query, "event:goal") dores|> transform_keys(%{visitors: :conversions})|> to_csv([:name, :conversions, :conversion_rate])@@ -643,7 +658,7 @@ defmodule PlausibleWeb.Api.StatsController do|> transform_keys(%{utm_source: :name})if params["csv"] do- if Map.has_key?(query.filters, "event:goal") do+ if Query.get_filter(query, "event:goal") dores|> transform_keys(%{visitors: :conversions})|> to_csv([:name, :conversions, :conversion_rate])@@ -667,7 +682,7 @@ defmodule PlausibleWeb.Api.StatsController do|> transform_keys(%{referrer: :name})if params["csv"] do- if Map.has_key?(query.filters, "event:goal") do+ if Query.get_filter(query, "event:goal") dores|> transform_keys(%{visitors: :conversions})|> to_csv([:name, :conversions, :conversion_rate])@@ -711,9 +726,11 @@ defmodule PlausibleWeb.Api.StatsController dosite = conn.assigns[:site]params = Map.put(params, "property", "visit:referrer")+ referrer_filter = DashboardFilterParser.filter_value("visit:source", referrer)+query =Query.from(site, params)- |> Query.put_filter("visit:source", referrer)+ |> Query.put_filter(referrer_filter)pagination = parse_pagination(params)@@ -747,7 +764,7 @@ defmodule PlausibleWeb.Api.StatsController do|> transform_keys(%{page: :name})if params["csv"] do- if Map.has_key?(query.filters, "event:goal") do+ if Query.get_filter(query, "event:goal") dopages|> transform_keys(%{visitors: :conversions})|> to_csv([:name, :conversions, :conversion_rate])@@ -771,7 +788,7 @@ defmodule PlausibleWeb.Api.StatsController do|> transform_keys(%{entry_page: :name})if params["csv"] do- if Map.has_key?(query.filters, "event:goal") do+ if Query.get_filter(query, "event:goal") doto_csv(entry_pages, [:name, :visitors, :conversion_rate], [:name,:conversions,@@ -803,7 +820,7 @@ defmodule PlausibleWeb.Api.StatsController do|> transform_keys(%{exit_page: :name})if params["csv"] do- if Map.has_key?(query.filters, "event:goal") do+ if Query.get_filter(query, "event:goal") doto_csv(exit_pages, [:name, :visitors, :conversion_rate], [:name,:conversions,@@ -829,8 +846,9 @@ defmodule PlausibleWeb.Api.StatsController dopages = Enum.map(breakdown_results, & &1[:exit_page])total_visits_query =- Query.put_filter(query, "event:page", {:member, pages})- |> Query.put_filter("event:name", {:is, "pageview"})+ query+ |> Query.put_filter([:member, "event:page", pages])+ |> Query.put_filter([:is, "event:name", "pageview"])|> struct!(property: "event:page")total_pageviews =@@ -870,7 +888,7 @@ defmodule PlausibleWeb.Api.StatsController doMap.put(country, :name, country_info.name)end)- if Map.has_key?(query.filters, "event:goal") do+ if Query.get_filter(query, "event:goal") docountries|> transform_keys(%{visitors: :conversions})|> to_csv([:name, :conversions, :conversion_rate])@@ -926,7 +944,7 @@ defmodule PlausibleWeb.Api.StatsController doend)if params["csv"] do- if Map.has_key?(query.filters, "event:goal") do+ if Query.get_filter(query, "event:goal") doregions|> transform_keys(%{visitors: :conversions})|> to_csv([:name, :conversions, :conversion_rate])@@ -966,7 +984,7 @@ defmodule PlausibleWeb.Api.StatsController doend)if params["csv"] do- if Map.has_key?(query.filters, "event:goal") do+ if Query.get_filter(query, "event:goal") docities|> transform_keys(%{visitors: :conversions})|> to_csv([:name, :conversions, :conversion_rate])@@ -990,7 +1008,7 @@ defmodule PlausibleWeb.Api.StatsController do|> transform_keys(%{browser: :name})if params["csv"] do- if Map.has_key?(query.filters, "event:goal") do+ if Query.get_filter(query, "event:goal") dobrowsers|> transform_keys(%{visitors: :conversions})|> to_csv([:name, :conversions, :conversion_rate])@@ -1014,7 +1032,7 @@ defmodule PlausibleWeb.Api.StatsController do|> transform_keys(%{browser_version: :name})if params["csv"] do- if Map.has_key?(query.filters, "event:goal") do+ if Query.get_filter(query, "event:goal") doversions|> transform_keys(%{name: :version,@@ -1044,7 +1062,7 @@ defmodule PlausibleWeb.Api.StatsController do|> transform_keys(%{os: :name})if params["csv"] do- if Map.has_key?(query.filters, "event:goal") do+ if Query.get_filter(query, "event:goal") dosystems|> transform_keys(%{visitors: :conversions})|> to_csv([:name, :conversions, :conversion_rate])@@ -1068,7 +1086,7 @@ defmodule PlausibleWeb.Api.StatsController do|> transform_keys(%{os_version: :name})if params["csv"] do- if Map.has_key?(query.filters, "event:goal") do+ if Query.get_filter(query, "event:goal") doversions|> transform_keys(%{name: :version, os: :name, visitors: :conversions})|> to_csv([:name, :version, :conversions, :conversion_rate])@@ -1094,7 +1112,7 @@ defmodule PlausibleWeb.Api.StatsController do|> transform_keys(%{device: :name})if params["csv"] do- if Map.has_key?(query.filters, "event:goal") do+ if Query.get_filter(query, "event:goal") dosizes|> transform_keys(%{visitors: :conversions})|> to_csv([:name, :conversions, :conversion_rate])@@ -1183,7 +1201,7 @@ defmodule PlausibleWeb.Api.StatsController do|> Enum.concat()percent_or_cr =- if query.filters["event:goal"],+ if Query.get_filter(query, "event:goal"),do: :conversion_rate,else: :percentage@@ -1200,7 +1218,7 @@ defmodule PlausibleWeb.Api.StatsController doquery = Query.from(site, params)metrics =- if query.filters["event:goal"] do+ if Query.get_filter(query, "event:goal") do[:visitors, :events, :conversion_rate] ++ @revenue_metricselse[:visitors, :events, :percentage] ++ @revenue_metrics@@ -1359,7 +1377,7 @@ defmodule PlausibleWeb.Api.StatsController dorequires_goal_filter? = metric in [:conversion_rate, :events]- if requires_goal_filter? and !query.filters["event:goal"] do+ if requires_goal_filter? and !Query.get_filter(query, "event:goal") do{:error, "Metric `#{metric}` can only be queried with a goal filter"}else{:ok, metric}@@ -1401,7 +1419,7 @@ defmodule PlausibleWeb.Api.StatsController doenddefp breakdown_metrics(query, extra_metrics \\ []) do- if query.filters["event:goal"] do+ if Query.get_filter(query, "event:goal") do[:visitors, :conversion_rate, :total_visitors]else[:visitors] ++ extra_metricsdiff --git a/lib/plausible_web/controllers/stats_controller.ex b/lib/plausible_web/controllers/stats_controller.exindex ca921c045f..d198647432 100644--- a/lib/plausible_web/controllers/stats_controller.ex+++ b/lib/plausible_web/controllers/stats_controller.ex@@ -180,16 +180,28 @@ defmodule PlausibleWeb.StatsController do|> Enum.join()end- defp csv_graph_metrics(%Query{filters: %{"event:goal" => _}}) do- metrics = [:visitors, :events, :conversion_rate]- column_headers = [:date, :unique_conversions, :total_conversions, :conversion_rate]-- {metrics, column_headers}- end-- defp csv_graph_metrics(_) do- metrics = [:visitors, :pageviews, :visits, :views_per_visit, :bounce_rate, :visit_duration]- column_headers = [:date | metrics]+ defp csv_graph_metrics(query) do+ {metrics, column_headers} =+ if Query.get_filter(query, "event:goal") do+ {+ [:visitors, :events, :conversion_rate],+ [:date, :unique_conversions, :total_conversions, :conversion_rate]+ }+ else+ metrics = [+ :visitors,+ :pageviews,+ :visits,+ :views_per_visit,+ :bounce_rate,+ :visit_duration+ ]++ {+ metrics,+ [:date | metrics]+ }+ end{metrics, column_headers}enddiff --git a/test/plausible/google/search_console/filters_test.exs b/test/plausible/google/search_console/filters_test.exsindex 7d73dacda8..c6efffa069 100644--- a/test/plausible/google/search_console/filters_test.exs+++ b/test/plausible/google/search_console/filters_test.exs@@ -3,9 +3,9 @@ defmodule Plausible.Google.SearchConsole.FiltersTest douse Plausible.DataCase, async: truetest "transforms simple page filter" do- filters = %{- "visit:entry_page" => {:is, "/page"}- }+ filters = [+ [:is, "visit:entry_page", "/page"]+ ]{:ok, transformed} = Filters.transform("sc-domain:plausible.io", filters)@@ -15,9 +15,9 @@ defmodule Plausible.Google.SearchConsole.FiltersTest doendtest "transforms matches page filter" do- filters = %{- "visit:entry_page" => {:matches, "*page*"}- }+ filters = [+ [:matches, "visit:entry_page", "*page*"]+ ]{:ok, transformed} = Filters.transform("sc-domain:plausible.io", filters)@@ -35,9 +35,9 @@ defmodule Plausible.Google.SearchConsole.FiltersTest doendtest "transforms member page filter" do- filters = %{- "visit:entry_page" => {:member, ["/pageA", "/pageB"]}- }+ filters = [+ [:member, "visit:entry_page", ["/pageA", "/pageB"]]+ ]{:ok, transformed} = Filters.transform("sc-domain:plausible.io", filters)@@ -55,9 +55,9 @@ defmodule Plausible.Google.SearchConsole.FiltersTest doendtest "transforms matches_member page filter" do- filters = %{- "visit:entry_page" => {:matches_member, ["/pageA*", "/pageB*"]}- }+ filters = [+ [:matches_member, "visit:entry_page", ["/pageA*", "/pageB*"]]+ ]{:ok, transformed} = Filters.transform("sc-domain:plausible.io", filters)@@ -75,9 +75,9 @@ defmodule Plausible.Google.SearchConsole.FiltersTest doendtest "transforms event:page exactly like visit:entry_page" do- filters = %{- "event:page" => {:matches_member, ["/pageA*", "/pageB*"]}- }+ filters = [+ [:matches_member, "event:page", ["/pageA*", "/pageB*"]]+ ]{:ok, transformed} = Filters.transform("sc-domain:plausible.io", filters)@@ -95,9 +95,9 @@ defmodule Plausible.Google.SearchConsole.FiltersTest doendtest "transforms simple visit:screen filter" do- filters = %{- "visit:screen" => {:is, "Desktop"}- }+ filters = [+ [:is, "visit:screen", "Desktop"]+ ]{:ok, transformed} = Filters.transform("sc-domain:plausible.io", filters)@@ -105,9 +105,9 @@ defmodule Plausible.Google.SearchConsole.FiltersTest doendtest "transforms member visit:screen filter" do- filters = %{- "visit:screen" => {:member, ["Mobile", "Tablet"]}- }+ filters = [+ [:member, "visit:screen", ["Mobile", "Tablet"]]+ ]{:ok, transformed} = Filters.transform("sc-domain:plausible.io", filters)@@ -121,9 +121,9 @@ defmodule Plausible.Google.SearchConsole.FiltersTest doendtest "transforms simple visit:country filter to alpha3" do- filters = %{- "visit:country" => {:is, "EE"}- }+ filters = [+ [:is, "visit:country", "EE"]+ ]{:ok, transformed} = Filters.transform("sc-domain:plausible.io", filters)@@ -131,9 +131,9 @@ defmodule Plausible.Google.SearchConsole.FiltersTest doendtest "transforms member visit:country filter" do- filters = %{- "visit:country" => {:member, ["EE", "PL"]}- }+ filters = [+ [:member, "visit:country", ["EE", "PL"]]+ ]{:ok, transformed} = Filters.transform("sc-domain:plausible.io", filters)@@ -147,11 +147,11 @@ defmodule Plausible.Google.SearchConsole.FiltersTest doendtest "filters can be combined" do- filters = %{- "visit:entry_page" => {:matches, "*web-analytics*"},- "visit:screen" => {:is, "Desktop"},- "visit:country" => {:member, ["EE", "PL"]}- }+ filters = [+ [:member, "visit:country", ["EE", "PL"]],+ [:matches, "visit:entry_page", "*web-analytics*"],+ [:is, "visit:screen", "Desktop"]+ ]{:ok, transformed} = Filters.transform("sc-domain:plausible.io", filters)@@ -171,12 +171,12 @@ defmodule Plausible.Google.SearchConsole.FiltersTest doendtest "when unsupported filter is included the whole set becomes invalid" do- filters = %{- "visit:entry_page" => {:matches, "*web-analytics*"},- "visit:screen" => {:is, "Desktop"},- "visit:country" => {:member, ["EE", "PL"]},- "visit:utm_medium" => {:is, "facebook"}- }+ filters = [+ [:matches, "visit:entry_page", "*web-analytics*"],+ [:is, "visit:screen", "Desktop"],+ [:member, "visit:country", ["EE", "PL"]],+ [:is, "visit:utm_medium", "facebook"]+ ]assert :unsupported_filters = Filters.transform("sc-domain:plausible.io", filters)enddiff --git a/test/plausible/stats/dashboard_filter_parser_test.exs b/test/plausible/stats/dashboard_filter_parser_test.exsindex e5469bdfda..1602380f37 100644--- a/test/plausible/stats/dashboard_filter_parser_test.exs+++ b/test/plausible/stats/dashboard_filter_parser_test.exs@@ -9,199 +9,199 @@ defmodule Plausible.Stats.DashboardFilterParserTest dodescribe "adding prefix" dotest "adds appropriate prefix to filter" do%{"page" => "/"}- |> assert_parsed(%{"event:page" => {:is, "/"}})+ |> assert_parsed([[:is, "event:page", "/"]])%{"goal" => "Signup"}- |> assert_parsed(%{"event:goal" => {:is, {:event, "Signup"}}})+ |> assert_parsed([[:is, "event:goal", {:event, "Signup"}]])%{"goal" => "Visit /blog"}- |> assert_parsed(%{"event:goal" => {:is, {:page, "/blog"}}})+ |> assert_parsed([[:is, "event:goal", {:page, "/blog"}]])%{"source" => "Google"}- |> assert_parsed(%{"visit:source" => {:is, "Google"}})+ |> assert_parsed([[:is, "visit:source", "Google"]])%{"referrer" => "cnn.com"}- |> assert_parsed(%{"visit:referrer" => {:is, "cnn.com"}})+ |> assert_parsed([[:is, "visit:referrer", "cnn.com"]])%{"utm_medium" => "search"}- |> assert_parsed(%{"visit:utm_medium" => {:is, "search"}})+ |> assert_parsed([[:is, "visit:utm_medium", "search"]])%{"utm_source" => "bing"}- |> assert_parsed(%{"visit:utm_source" => {:is, "bing"}})+ |> assert_parsed([[:is, "visit:utm_source", "bing"]])%{"utm_content" => "content"}- |> assert_parsed(%{"visit:utm_content" => {:is, "content"}})+ |> assert_parsed([[:is, "visit:utm_content", "content"]])%{"utm_term" => "term"}- |> assert_parsed(%{"visit:utm_term" => {:is, "term"}})+ |> assert_parsed([[:is, "visit:utm_term", "term"]])%{"screen" => "Desktop"}- |> assert_parsed(%{"visit:screen" => {:is, "Desktop"}})+ |> assert_parsed([[:is, "visit:screen", "Desktop"]])%{"browser" => "Opera"}- |> assert_parsed(%{"visit:browser" => {:is, "Opera"}})+ |> assert_parsed([[:is, "visit:browser", "Opera"]])%{"browser_version" => "10.1"}- |> assert_parsed(%{"visit:browser_version" => {:is, "10.1"}})+ |> assert_parsed([[:is, "visit:browser_version", "10.1"]])%{"os" => "Linux"}- |> assert_parsed(%{"visit:os" => {:is, "Linux"}})+ |> assert_parsed([[:is, "visit:os", "Linux"]])%{"os_version" => "13.0"}- |> assert_parsed(%{"visit:os_version" => {:is, "13.0"}})+ |> assert_parsed([[:is, "visit:os_version", "13.0"]])%{"country" => "EE"}- |> assert_parsed(%{"visit:country" => {:is, "EE"}})+ |> assert_parsed([[:is, "visit:country", "EE"]])%{"region" => "EE-12"}- |> assert_parsed(%{"visit:region" => {:is, "EE-12"}})+ |> assert_parsed([[:is, "visit:region", "EE-12"]])%{"city" => "123"}- |> assert_parsed(%{"visit:city" => {:is, "123"}})+ |> assert_parsed([[:is, "visit:city", "123"]])%{"entry_page" => "/blog"}- |> assert_parsed(%{"visit:entry_page" => {:is, "/blog"}})+ |> assert_parsed([[:is, "visit:entry_page", "/blog"]])%{"exit_page" => "/blog"}- |> assert_parsed(%{"visit:exit_page" => {:is, "/blog"}})+ |> assert_parsed([[:is, "visit:exit_page", "/blog"]])%{"props" => %{"cta" => "Top"}}- |> assert_parsed(%{"event:props:cta" => {:is, "Top"}})+ |> assert_parsed([[:is, "event:props:cta", "Top"]])%{"hostname" => "dummy.site"}- |> assert_parsed(%{"event:hostname" => {:is, "dummy.site"}})+ |> assert_parsed([[:is, "event:hostname", "dummy.site"]])endenddescribe "escaping pipe character" dotest "in simple is filter" do%{"goal" => ~S(Foo \| Bar)}- |> assert_parsed(%{"event:goal" => {:is, {:event, "Foo | Bar"}}})+ |> assert_parsed([[:is, "event:goal", {:event, "Foo | Bar"}]])endtest "in member filter" do%{"page" => ~S(/|\|)}- |> assert_parsed(%{"event:page" => {:member, ["/", "|"]}})+ |> assert_parsed([[:member, "event:page", ["/", "|"]]])endenddescribe "is not filter type" dotest "simple is not filter" do%{"page" => "!/"}- |> assert_parsed(%{"event:page" => {:is_not, "/"}})+ |> assert_parsed([[:is_not, "event:page", "/"]])%{"props" => %{"cta" => "!Top"}}- |> assert_parsed(%{"event:props:cta" => {:is_not, "Top"}})+ |> assert_parsed([[:is_not, "event:props:cta", "Top"]])endenddescribe "member filter type" dotest "simple member filter" do%{"page" => "/|/blog"}- |> assert_parsed(%{"event:page" => {:member, ["/", "/blog"]}})+ |> assert_parsed([[:member, "event:page", ["/", "/blog"]]])endtest "escaping pipe character" do%{"page" => "/|\\|"}- |> assert_parsed(%{"event:page" => {:member, ["/", "|"]}})+ |> assert_parsed([[:member, "event:page", ["/", "|"]]])endtest "mixed goals" do%{"goal" => "Signup|Visit /thank-you"}- |> assert_parsed(%{"event:goal" => {:member, [{:event, "Signup"}, {:page, "/thank-you"}]}})+ |> assert_parsed([[:member, "event:goal", [{:event, "Signup"}, {:page, "/thank-you"}]]])%{"goal" => "Visit /thank-you|Signup"}- |> assert_parsed(%{"event:goal" => {:member, [{:page, "/thank-you"}, {:event, "Signup"}]}})+ |> assert_parsed([[:member, "event:goal", [{:page, "/thank-you"}, {:event, "Signup"}]]])endenddescribe "matches_member filter type" dotest "parses matches_member filter type" do%{"page" => "/|/blog**"}- |> assert_parsed(%{"event:page" => {:matches_member, ["/", "/blog**"]}})+ |> assert_parsed([[:matches_member, "event:page", ["/", "/blog**"]]])endtest "parses not_matches_member filter type" do%{"page" => "!/|/blog**"}- |> assert_parsed(%{"event:page" => {:not_matches_member, ["/", "/blog**"]}})+ |> assert_parsed([[:not_matches_member, "event:page", ["/", "/blog**"]]])endenddescribe "contains filter type" dotest "single contains" do%{"page" => "~blog"}- |> assert_parsed(%{"event:page" => {:matches, "**blog**"}})+ |> assert_parsed([[:matches, "event:page", "**blog**"]])endtest "negated contains" do%{"page" => "!~articles"}- |> assert_parsed(%{"event:page" => {:does_not_match, "**articles**"}})+ |> assert_parsed([[:does_not_match, "event:page", "**articles**"]])endtest "contains member" do%{"page" => "~articles|blog"}- |> assert_parsed(%{"event:page" => {:matches_member, ["**articles**", "**blog**"]}})+ |> assert_parsed([[:matches_member, "event:page", ["**articles**", "**blog**"]]])endtest "not contains member" do%{"page" => "!~articles|blog"}- |> assert_parsed(%{"event:page" => {:not_matches_member, ["**articles**", "**blog**"]}})+ |> assert_parsed([[:not_matches_member, "event:page", ["**articles**", "**blog**"]]])endenddescribe "not_member filter type" dotest "simple not_member filter" do%{"page" => "!/|/blog"}- |> assert_parsed(%{"event:page" => {:not_member, ["/", "/blog"]}})+ |> assert_parsed([[:not_member, "event:page", ["/", "/blog"]]])endtest "mixed goals" do%{"goal" => "!Signup|Visit /thank-you"}- |> assert_parsed(%{- "event:goal" => {:not_member, [{:event, "Signup"}, {:page, "/thank-you"}]}- })+ |> assert_parsed([+ [:not_member, "event:goal", [{:event, "Signup"}, {:page, "/thank-you"}]]+ ])%{"goal" => "!Visit /thank-you|Signup"}- |> assert_parsed(%{- "event:goal" => {:not_member, [{:page, "/thank-you"}, {:event, "Signup"}]}- })+ |> assert_parsed([+ [:not_member, "event:goal", [{:page, "/thank-you"}, {:event, "Signup"}]]+ ])endenddescribe "matches filter type" dotest "can be used with `goal` or `page` filters" do%{"page" => "/blog/post-*"}- |> assert_parsed(%{"event:page" => {:matches, "/blog/post-*"}})+ |> assert_parsed([[:matches, "event:page", "/blog/post-*"]])%{"goal" => "Visit /blog/post-*"}- |> assert_parsed(%{"event:goal" => {:matches, {:page, "/blog/post-*"}}})+ |> assert_parsed([[:matches, "event:goal", {:page, "/blog/post-*"}]])endtest "other filters default to `is` even when wildcard is present" do%{"country" => "Germa**"}- |> assert_parsed(%{"visit:country" => {:is, "Germa**"}})+ |> assert_parsed([[:is, "visit:country", "Germa**"]])endenddescribe "does_not_match filter type" dotest "can be used with `page` filter" do%{"page" => "!/blog/post-*"}- |> assert_parsed(%{"event:page" => {:does_not_match, "/blog/post-*"}})+ |> assert_parsed([[:does_not_match, "event:page", "/blog/post-*"]])endtest "other filters default to is_not even when wildcard is present" do%{"country" => "!Germa**"}- |> assert_parsed(%{"visit:country" => {:is_not, "Germa**"}})+ |> assert_parsed([[:is_not, "visit:country", "Germa**"]])endenddescribe "contains prefix filter type" dotest "can be used with any filter" do%{"page" => "~/blog/post"}- |> assert_parsed(%{"event:page" => {:matches, "**/blog/post**"}})+ |> assert_parsed([[:matches, "event:page", "**/blog/post**"]])%{"source" => "~facebook"}- |> assert_parsed(%{"visit:source" => {:matches, "**facebook**"}})+ |> assert_parsed([[:matches, "visit:source", "**facebook**"]])endendenddiff --git a/test/plausible/stats/filters_test.exs b/test/plausible/stats/filters_test.exsindex 4f87bd9957..aa5b77412f 100644--- a/test/plausible/stats/filters_test.exs+++ b/test/plausible/stats/filters_test.exs@@ -12,90 +12,90 @@ defmodule Plausible.Stats.FiltersTest dodescribe "parses filter expression" dotest "simple positive" do"event:name==pageview"- |> assert_parsed(%{"event:name" => {:is, "pageview"}})+ |> assert_parsed([[:is, "event:name", "pageview"]])endtest "simple negative" do"event:name!=pageview"- |> assert_parsed(%{"event:name" => {:is_not, "pageview"}})+ |> assert_parsed([[:is_not, "event:name", "pageview"]])endtest "whitespace is trimmed" do" event:name == pageview "- |> assert_parsed(%{"event:name" => {:is, "pageview"}})+ |> assert_parsed([[:is, "event:name", "pageview"]])endtest "wildcard" do"event:page==/blog/post-*"- |> assert_parsed(%{"event:page" => {:matches, "/blog/post-*"}})+ |> assert_parsed([[:matches, "event:page", "/blog/post-*"]])endtest "negative wildcard" do"event:page!=/blog/post-*"- |> assert_parsed(%{"event:page" => {:does_not_match, "/blog/post-*"}})+ |> assert_parsed([[:does_not_match, "event:page", "/blog/post-*"]])endtest "custom event goal" do"event:goal==Signup"- |> assert_parsed(%{"event:goal" => {:is, {:event, "Signup"}}})+ |> assert_parsed([[:is, "event:goal", {:event, "Signup"}]])endtest "pageview goal" do"event:goal==Visit /blog"- |> assert_parsed(%{"event:goal" => {:is, {:page, "/blog"}}})+ |> assert_parsed([[:is, "event:goal", {:page, "/blog"}]])endtest "member" do"visit:country==FR|GB|DE"- |> assert_parsed(%{"visit:country" => {:member, ["FR", "GB", "DE"]}})+ |> assert_parsed([[:member, "visit:country", ["FR", "GB", "DE"]]])endtest "member + wildcard" do"event:page==/blog**|/newsletter|/*/"- |> assert_parsed(%{"event:page" => {:matches, "/blog**|/newsletter|/*/"}})+ |> assert_parsed([[:matches, "event:page", "/blog**|/newsletter|/*/"]])endtest "combined with \";\"" do"event:page==/blog**|/newsletter|/*/ ; visit:country==FR|GB|DE"- |> assert_parsed(%{- "event:page" => {:matches, "/blog**|/newsletter|/*/"},- "visit:country" => {:member, ["FR", "GB", "DE"]}- })+ |> assert_parsed([+ [:matches, "event:page", "/blog**|/newsletter|/*/"],+ [:member, "visit:country", ["FR", "GB", "DE"]]+ ])endtest "escaping pipe character" do"utm_campaign==campaign \\| 1"- |> assert_parsed(%{"utm_campaign" => {:is, "campaign | 1"}})+ |> assert_parsed([[:is, "utm_campaign", "campaign | 1"]])endtest "escaping pipe character in member filter" do"utm_campaign==campaign \\| 1|campaign \\| 2"- |> assert_parsed(%{"utm_campaign" => {:member, ["campaign | 1", "campaign | 2"]}})+ |> assert_parsed([[:member, "utm_campaign", ["campaign | 1", "campaign | 2"]]])endtest "keeps escape characters in member + wildcard filter" do"event:page==/**\\|page|/other/page"- |> assert_parsed(%{"event:page" => {:matches, "/**\\|page|/other/page"}})+ |> assert_parsed([[:matches, "event:page", "/**\\|page|/other/page"]])endtest "gracefully fails to parse garbage" do"bfg10309\uff1cs1\ufe65s2\u02bas3\u02b9hjl10309"- |> assert_parsed(%{})+ |> assert_parsed([])endtest "gracefully fails to parse garbage with double quotes" do"\";print(md5(31337));$a=\""- |> assert_parsed(%{})+ |> assert_parsed([])endtest "gracefully fails to parse garbage country code" do"visit:country==AKSJSDFKJSS"- |> assert_parsed(%{})+ |> assert_parsed([])endtest "gracefully fails to parse garbage country code (with pipes)" do"visit:country==ET'||DBMS_PIPE.RECEIVE_MESSAGE(CHR(98)||CHR(98)||CHR(98),15)||'"- |> assert_parsed(%{})+ |> assert_parsed([])endendenddiff --git a/test/plausible/stats/query_test.exs b/test/plausible/stats/query_test.exsindex f568589549..b509b6914c 100644--- a/test/plausible/stats/query_test.exs+++ b/test/plausible/stats/query_test.exs@@ -194,14 +194,14 @@ defmodule Plausible.Stats.QueryTest dofilters = Jason.encode!(%{"goal" => "Signup"})q = Query.from(site, %{"period" => "6mo", "filters" => filters})- assert q.filters["event:goal"] == {:is, {:event, "Signup"}}+ assert q.filters == [[:is, "event:goal", {:event, "Signup"}]]endtest "parses source filter", %{site: site} dofilters = Jason.encode!(%{"source" => "Twitter"})q = Query.from(site, %{"period" => "6mo", "filters" => filters})- assert q.filters["visit:source"] == {:is, "Twitter"}+ assert q.filters == [[:is, "visit:source", "Twitter"]]endend