Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.
TypeScript analyticsanalytics-dashboardclickhouseelixirgoogle-analyticsmarketingopen-source-analyticsphoenixplausible-analyticspostgresqlprivacyprivacy-friendlyself-hostedsimple-analyticsstatisticstailwindcssweb-analyticswebsite-analyticswebsite-statswebsite-tracking
APIv2: and/or/not support (#4480)
Karl-Aksel Puulmann8fa3a831291d
diff --git a/extra/lib/plausible/stats/goal/revenue.ex b/extra/lib/plausible/stats/goal/revenue.exindex 84d69d07e3..d08ab369c6 100644--- a/extra/lib/plausible/stats/goal/revenue.ex+++ b/extra/lib/plausible/stats/goal/revenue.ex@@ -4,7 +4,7 @@ defmodule Plausible.Stats.Goal.Revenue do"""import Ecto.Query- alias Plausible.Stats.Query+ alias Plausible.Stats.Filters@revenue_metrics [:average_revenue, :total_revenue]@@ -25,7 +25,7 @@ defmodule Plausible.Stats.Goal.Revenue do"""def get_revenue_tracking_currency(site, query, metrics) dogoal_filters =- case Query.get_filter(query, "event:goal") do+ case Filters.get_toplevel_filter(query, "event:goal") do[:is, "event:goal", list] -> list_ -> []enddiff --git a/lib/plausible/stats/aggregate.ex b/lib/plausible/stats/aggregate.exindex bcf1faffa8..8bce4103dd 100644--- a/lib/plausible/stats/aggregate.ex+++ b/lib/plausible/stats/aggregate.ex@@ -9,7 +9,7 @@ defmodule Plausible.Stats.Aggregate douse Plausibleimport Plausible.Stats.Baseimport Ecto.Query- alias Plausible.Stats.{Query, Util, SQL}+ alias Plausible.Stats.{Query, Util, SQL, Filters}def aggregate(site, query, metrics) do{currency, metrics} =@@ -49,7 +49,7 @@ defmodule Plausible.Stats.Aggregate dodefp aggregate_time_on_page(site, query) dowindowed_pages_q =- from e in base_event_query(site, Query.remove_filters(query, ["event:page"])),+ from e in base_event_query(site, Query.remove_top_level_filters(query, ["event:page"])),select: %{next_timestamp: over(fragment("leadInFrame(?)", e.timestamp), :event_horizon),next_pathname: over(fragment("leadInFrame(?)", e.pathname), :event_horizon),@@ -65,7 +65,7 @@ defmodule Plausible.Stats.Aggregate do]]- event_page_filter = Query.get_filter(query, "event:page")+ event_page_filter = Filters.get_toplevel_filter(query, "event:page")timed_page_transitions_q =from e in Ecto.Query.subquery(windowed_pages_q),diff --git a/lib/plausible/stats/breakdown.ex b/lib/plausible/stats/breakdown.exindex 356b4cabda..76f08c37eb 100644--- a/lib/plausible/stats/breakdown.ex+++ b/lib/plausible/stats/breakdown.ex@@ -82,21 +82,24 @@ defmodule Plausible.Stats.Breakdown doimport Ecto.Querywindowed_pages_q =- from e in base_event_query(site, Query.remove_filters(query, ["event:page", "event:props"])),- select: %{- next_timestamp: over(fragment("leadInFrame(?)", e.timestamp), :event_horizon),- next_pathname: over(fragment("leadInFrame(?)", e.pathname), :event_horizon),- timestamp: e.timestamp,- pathname: e.pathname,- session_id: e.session_id- },- windows: [- event_horizon: [- partition_by: e.session_id,- order_by: e.timestamp,- frame: fragment("ROWS BETWEEN CURRENT ROW AND 1 FOLLOWING")- ]- ]+ from e in base_event_query(+ site,+ Query.remove_top_level_filters(query, ["event:page", "event:props"])+ ),+ select: %{+ next_timestamp: over(fragment("leadInFrame(?)", e.timestamp), :event_horizon),+ next_pathname: over(fragment("leadInFrame(?)", e.pathname), :event_horizon),+ timestamp: e.timestamp,+ pathname: e.pathname,+ session_id: e.session_id+ },+ windows: [+ event_horizon: [+ partition_by: e.session_id,+ order_by: e.timestamp,+ frame: fragment("ROWS BETWEEN CURRENT ROW AND 1 FOLLOWING")+ ]+ ]timed_page_transitions_q =from e in subquery(windowed_pages_q),diff --git a/lib/plausible/stats/custom_props.ex b/lib/plausible/stats/custom_props.exindex 2b11d7ef92..932e5079f6 100644--- a/lib/plausible/stats/custom_props.ex+++ b/lib/plausible/stats/custom_props.ex@@ -3,12 +3,12 @@ defmodule Plausible.Stats.CustomProps doModule for querying user defined 'custom properties'."""- alias Plausible.Stats.Query+ alias Plausible.Stats.Filtersuse Plausible.ClickhouseRepoimport Plausible.Stats.Basedef fetch_prop_names(site, query) do- case Query.get_filter_by_prefix(query, "event:props:") do+ case Filters.get_toplevel_filter(query, "event:props:") do[_op, "event:props:" <> key | _rest] ->[key]diff --git a/lib/plausible/stats/filter_suggestions.ex b/lib/plausible/stats/filter_suggestions.exindex b52dfea339..b9d9c32541 100644--- a/lib/plausible/stats/filter_suggestions.ex+++ b/lib/plausible/stats/filter_suggestions.ex@@ -8,6 +8,7 @@ defmodule Plausible.Stats.FilterSuggestions doalias Plausible.Stats.Queryalias Plausible.Stats.Imported+ alias Plausible.Stats.Filtersdef filter_suggestions(site, query, "country", filter_search) domatches = Location.search_country(filter_search)@@ -187,10 +188,10 @@ defmodule Plausible.Stats.FilterSuggestions dodef filter_suggestions(site, query, "prop_value", filter_search) dofilter_query = if filter_search == nil, do: "%", else: "%#{filter_search}%"- [_op, "event:props:" <> key | _rest] = Query.get_filter_by_prefix(query, "event:props")+ [_op, "event:props:" <> key | _rest] = Filters.get_toplevel_filter(query, "event:props")none_q =- from(e in base_event_query(site, Query.remove_filters(query, ["event:props"])),+ from(e in base_event_query(site, Query.remove_top_level_filters(query, ["event:props"])),select: "(none)",where: not has_key(e, :meta, ^key),limit: 1diff --git a/lib/plausible/stats/filters/filters.ex b/lib/plausible/stats/filters/filters.exindex 5195dd3a4e..62b15cedab 100644--- a/lib/plausible/stats/filters/filters.ex+++ b/lib/plausible/stats/filters/filters.ex@@ -92,4 +92,89 @@ defmodule Plausible.Stats.Filters do|> List.last()|> String.to_existing_atom()end++ def dimensions_used_in_filters(filters, opts \\ []) do+ min_depth = Keyword.get(opts, :min_depth, 0)++ filters+ |> traverse()+ |> Enum.filter(fn {_filter, _root, depth} -> depth >= min_depth end)+ |> Enum.map(fn {[_operator, dimension | _rest], _root, _depth} -> dimension end)+ end++ def filtering_on_dimension?(query, dimension) do+ dimension in dimensions_used_in_filters(query.filters)+ end++ @doc """+ Gets the first top level filter with matching dimension (or nil).++ Only use in cases where it's known that filters are only set on the top level as it+ does not handle AND/OR/NOT!+ """+ def get_toplevel_filter(query, prefix) do+ Enum.find(query.filters, fn [_op, dimension | _rest] ->+ String.starts_with?(dimension, prefix)+ end)+ end++ def rename_dimensions_used_in_filter(filters, renames) do+ transform_filters(filters, fn+ [operation, dimension, clauses] ->+ [[operation, Map.get(renames, dimension, dimension), clauses]]++ _subtree ->+ nil+ end)+ end++ @doc """+ Updates filters via `transformer`.++ Transformer will receive each node (filter, and/or/not subtree) of+ query and must return a list of nodes to replace it with or nil+ to ignore and look deeper.+ """+ def transform_filters(filters, transformer) do+ filters+ |> Enum.flat_map(&transform_tree(&1, transformer))+ end++ defp transform_tree(filter, transformer) do+ case {transformer.(filter), filter} do+ # Transformer did not return that value - transform that subtree+ {nil, [:not, child_filter]} ->+ [[:not, transform_tree(child_filter, transformer)]]++ {nil, [operation, filters]} when operation in [:and, :or] ->+ [[operation, transform_filters(filters, transformer)]]++ # Reached a leaf node, return existing value+ {nil, filter} ->+ [[filter]]++ # Transformer returned a value - don't transform that subtree+ {transformed_filters, _filter} ->+ transformed_filters+ end+ end++ defp traverse(filters, root \\ nil, depth \\ -1) do+ filters+ |> Enum.flat_map(&traverse_tree(&1, root || &1, depth + 1))+ end++ defp traverse_tree(filter, root, depth) do+ case filter do+ [:not, child_filter] ->+ traverse_tree(child_filter, root, depth + 1)++ [operation, filters] when operation in [:and, :or] ->+ traverse(filters, root || filter, depth + 1)++ # Leaf node+ _ ->+ [{filter, root, depth}]+ end+ endenddiff --git a/lib/plausible/stats/filters/query_parser.ex b/lib/plausible/stats/filters/query_parser.exindex 4814f322ac..d79cbacc6e 100644--- a/lib/plausible/stats/filters/query_parser.ex+++ b/lib/plausible/stats/filters/query_parser.ex@@ -1,7 +1,7 @@defmodule Plausible.Stats.Filters.QueryParser do@moduledoc false- alias Plausible.Stats.{TableDecider, Filters, Query, Metrics, DateTimeRange, JSONSchema}+ alias Plausible.Stats.{TableDecider, Filters, Metrics, DateTimeRange, JSONSchema}@default_include %{imports: false,@@ -36,8 +36,10 @@ defmodule Plausible.Stats.Filters.QueryParser doinclude: include},:ok <- validate_order_by(query),- :ok <- validate_goal_filters(query),:ok <- validate_custom_props_access(site, query),+ :ok <- validate_toplevel_only_filter_dimension(query),+ :ok <- validate_special_metrics_filters(query),+ :ok <- validate_filtered_goals_exist(query),:ok <- validate_metrics(query),:ok <- validate_include(query) do{:ok, query}@@ -63,9 +65,9 @@ defmodule Plausible.Stats.Filters.QueryParser dodefp parse_filter(filter) dowith {:ok, operator} <- parse_operator(filter),- {:ok, filter_key} <- parse_filter_key(filter),+ {:ok, second} <- parse_filter_second(operator, filter),{:ok, rest} <- parse_filter_rest(operator, filter) do- {:ok, [operator, filter_key | rest]}+ {:ok, [operator, second | rest]}endend@@ -75,8 +77,18 @@ defmodule Plausible.Stats.Filters.QueryParser dodefp parse_operator(["does_not_match" | _rest]), do: {:ok, :does_not_match}defp parse_operator(["contains" | _rest]), do: {:ok, :contains}defp parse_operator(["does_not_contain" | _rest]), do: {:ok, :does_not_contain}+ defp parse_operator(["not" | _rest]), do: {:ok, :not}+ defp parse_operator(["and" | _rest]), do: {:ok, :and}+ defp parse_operator(["or" | _rest]), do: {:ok, :or}defp parse_operator(filter), do: {:error, "Unknown operator for filter '#{i(filter)}'."}+ def parse_filter_second(:not, [_, filter | _rest]), do: parse_filter(filter)++ def parse_filter_second(operator, [_, filters | _rest]) when operator in [:and, :or],+ do: parse_filters(filters)++ def parse_filter_second(_operator, filter), do: parse_filter_key(filter)+defp parse_filter_key([_operator, filter_key | _rest] = filter) doparse_filter_key_string(filter_key, "Invalid filter '#{i(filter)}")end@@ -87,6 +99,10 @@ defmodule Plausible.Stats.Filters.QueryParser dowhen operator in [:is, :is_not, :matches, :does_not_match, :contains, :does_not_contain],do: parse_clauses_list(filter)+ defp parse_filter_rest(operator, _filter)+ when operator in [:not, :and, :or],+ do: {:ok, []}+defp parse_clauses_list([operation, filter_key, list] = filter) when is_list(list) doall_strings? = Enum.all?(list, &is_binary/1)all_integers? = Enum.all?(list, &is_integer/1)@@ -359,7 +375,37 @@ defmodule Plausible.Stats.Filters.QueryParser doendend- defp validate_goal_filters(query) do+ @only_toplevel ["event:goal", "event:hostname"]+ defp validate_toplevel_only_filter_dimension(query) do+ not_toplevel = Filters.dimensions_used_in_filters(query.filters, min_depth: 1)++ if Enum.any?(not_toplevel, &(&1 in @only_toplevel)) do+ {:error,+ "Invalid filters. Dimension `#{List.first(not_toplevel)}` can only be filtered at the top level."}+ else+ :ok+ end+ end++ @special_metrics [:conversion_rate, :group_conversion_rate]+ defp validate_special_metrics_filters(query) do+ special_metric? = Enum.any?(@special_metrics, &(&1 in query.metrics))++ deep_custom_property? =+ query.filters+ |> Filters.dimensions_used_in_filters(min_depth: 1)+ |> Enum.any?(fn dimension -> String.starts_with?(dimension, "event:props:") end)++ if special_metric? and deep_custom_property? do+ {:error,+ "Invalid filters. When `conversion_rate` or `group_conversion_rate` metrics are used, custom property filters can only be used on top level."}+ else+ :ok+ end+ end++ defp validate_filtered_goals_exist(query) do+ # Note: Only works since event:goal is allowed as a top level filtergoal_filter_clauses =Enum.flat_map(query.filters, fn[:is, "event:goal", clauses] -> clauses@@ -396,7 +442,7 @@ defmodule Plausible.Stats.Filters.QueryParser dodefp validate_custom_props_access(_site, query, allowed_props) dovalid? =query.filters- |> Enum.map(fn [_operation, filter_key | _rest] -> filter_key end)+ |> Filters.dimensions_used_in_filters()|> Enum.concat(query.dimensions)|> Enum.all?(fn"event:props:" <> prop -> prop in allowed_props@@ -418,7 +464,7 @@ defmodule Plausible.Stats.Filters.QueryParser dodefp validate_metric(metric, query) when metric in [:conversion_rate, :group_conversion_rate] doif Enum.member?(query.dimensions, "event:goal") or- not is_nil(Query.get_filter(query, "event:goal")) do+ Filters.filtering_on_dimension?(query, "event:goal") do:okelse{:error, "Metric `#{metric}` can only be queried with event:goal filters or dimensions."}@@ -427,7 +473,7 @@ defmodule Plausible.Stats.Filters.QueryParser dodefp validate_metric(:views_per_visit = metric, query) docond do- not is_nil(Query.get_filter(query, "event:page")) ->+ Filters.filtering_on_dimension?(query, "event:page") ->{:error, "Metric `#{metric}` cannot be queried with a filter on `event:page`."}length(query.dimensions) > 0 ->diff --git a/lib/plausible/stats/imported/base.ex b/lib/plausible/stats/imported/base.exindex 23bfebda41..4a8744b5d5 100644--- a/lib/plausible/stats/imported/base.ex+++ b/lib/plausible/stats/imported/base.ex@@ -6,7 +6,9 @@ defmodule Plausible.Stats.Imported.Base doimport Ecto.Queryalias Plausible.Imported- alias Plausible.Stats.{Filters, Query, SQL, DateTimeRange}+ alias Plausible.Stats.{Query, DateTimeRange}++ import Plausible.Stats.Filters, only: [dimensions_used_in_filters: 1]@property_to_table_mappings %{"visit:source" => "imported_sources",@@ -47,20 +49,6 @@ defmodule Plausible.Stats.Imported.Base do@imported_custom_props Imported.imported_custom_props()- @db_field_mappings %{- referrer_source: :source,- screen_size: :device,- screen: :device,- os: :operating_system,- os_version: :operating_system_version,- country_code: :country,- subdivision1_code: :region,- city_geoname_id: :city,- entry_page_hostname: :hostname,- pathname: :page,- url: :link_url- }-def property_to_table_mappings(), do: @property_to_table_mappingsdef query_imported(site, query) do@@ -78,9 +66,9 @@ defmodule Plausible.Stats.Imported.Base dowhere: i.import_id in ^import_ids,where: i.date >= ^date_from,where: i.date <= ^date_to,+ where: ^Plausible.Stats.Imported.SQL.WhereBuilder.build(query),select: %{})- |> apply_filter(query)enddef decide_tables(query) do@@ -92,8 +80,7 @@ defmodule Plausible.Stats.Imported.Base doenddefp custom_prop_query?(query) do- query.filters- |> Enum.map(&Enum.at(&1, 1))+ dimensions_used_in_filters(query.filters)|> Enum.concat(query.dimensions)|> Enum.any?(&(&1 in @imported_custom_props))end@@ -108,8 +95,7 @@ defmodule Plausible.Stats.Imported.Base doif dimensions == [] or(length(dimensions) == 1 and hd(dimensions) in @queriable_custom_prop_dimensions) docustom_prop_filters =- query.filters- |> Enum.map(&Enum.at(&1, 1))+ dimensions_used_in_filters(query.filters)|> Enum.filter(&(&1 in @imported_custom_props))|> Enum.uniq()@@ -153,8 +139,8 @@ defmodule Plausible.Stats.Imported.Base do["imported_pages", "imported_custom_events"]end- defp do_decide_tables(%Query{filters: filters, dimensions: ["event:goal"]} = query) do- filter_props = Enum.map(filters, &Enum.at(&1, 1))+ defp do_decide_tables(%Query{dimensions: ["event:goal"]} = query) do+ filter_dimensions = dimensions_used_in_filters(query.filters)filter_goals = get_filter_goals(query)@@ -163,11 +149,11 @@ defmodule Plausible.Stats.Imported.Base doany_pageview_goals? =Enum.any?(filter_goals, fn goal -> Plausible.Goal.type(goal) == :page end)- any_event_name_filters? = "event:name" in filter_props or any_event_goals?- any_page_filters? = "event:page" in filter_props or any_pageview_goals?+ any_event_name_filters? = "event:name" in filter_dimensions or any_event_goals?+ any_page_filters? = "event:page" in filter_dimensions or any_pageview_goals?any_other_filters? =- Enum.any?(filter_props, &(&1 not in ["event:page", "event:name", "event:goal"]))+ Enum.any?(filter_dimensions, &(&1 not in ["event:page", "event:name", "event:goal"]))cond doany_other_filters? -> []@@ -177,11 +163,10 @@ defmodule Plausible.Stats.Imported.Base doendend- defp do_decide_tables(%Query{filters: filters, dimensions: dimensions} = query) do+ defp do_decide_tables(query) dotable_candidates =- filters- |> Enum.map(fn [_, filter_key | _] -> filter_key end)- |> Enum.concat(dimensions)+ dimensions_used_in_filters(query.filters)+ |> Enum.concat(query.dimensions)|> Enum.reject(&(&1 in @queriable_time_dimensions or &1 == "event:goal"))|> Enum.flat_map(fn"visit:screen" -> ["visit:device"]@@ -206,10 +191,10 @@ defmodule Plausible.Stats.Imported.Base doendend- defp get_filter_goals(%Query{filters: filters} = query) do- filters- |> Enum.filter(fn [_, key, _] -> key == "event:goal" end)- |> Enum.flat_map(fn [operation, _, clauses] ->+ defp get_filter_goals(query) do+ query.filters+ |> Enum.filter(fn [_, dimension | _rest] -> dimension == "event:goal" end)+ |> Enum.flat_map(fn [operation, _dimension, clauses] ->Enum.flat_map(clauses, fn clause ->query.preloaded_goals|> Plausible.Goals.Filters.filter_preloaded(operation, clause)@@ -217,22 +202,6 @@ defmodule Plausible.Stats.Imported.Base doend)end- defp apply_filter(q, %Query{filters: filters} = query) do- Enum.reduce(filters, q, fn [_, filter_key, _] = filter, q ->- db_field = Filters.without_prefix(filter_key)-- if db_field == :goal do- condition = Plausible.Goals.Filters.add_filter(query, filter, imported?: true)- where(q, ^condition)- else- mapped_db_field = Map.get(@db_field_mappings, db_field, db_field)- condition = SQL.WhereBuilder.build_condition(mapped_db_field, filter)-- where(q, ^condition)- end- end)- end-def special_goals_for("event:props:url"), do: Imported.goals_with_url()def special_goals_for("event:props:path"), do: Imported.goals_with_path()enddiff --git a/lib/plausible/stats/imported/imported.ex b/lib/plausible/stats/imported/imported.exindex 36e13605a9..e2330f08bd 100644--- a/lib/plausible/stats/imported/imported.ex+++ b/lib/plausible/stats/imported/imported.ex@@ -3,7 +3,7 @@ defmodule Plausible.Stats.Imported douse Plausible.Stats.SQL.Fragmentsimport Ecto.Query- import Plausible.Stats.Imported.SQL.Builder+ import Plausible.Stats.Imported.SQL.Expressionalias Plausible.Stats.Filtersalias Plausible.Stats.Importeddiff --git a/lib/plausible/stats/imported/sql/builder.ex b/lib/plausible/stats/imported/sql/expression.exsimilarity index 99%rename from lib/plausible/stats/imported/sql/builder.exrename to lib/plausible/stats/imported/sql/expression.exindex 76a8b011d0..776796950e 100644--- a/lib/plausible/stats/imported/sql/builder.ex+++ b/lib/plausible/stats/imported/sql/expression.ex@@ -1,4 +1,4 @@-defmodule Plausible.Stats.Imported.SQL.Builder do+defmodule Plausible.Stats.Imported.SQL.Expression do@moduledoc """This module is responsible for generating SQL/Ecto expressionsfor dimensions, filters and metrics used in import table queriesdiff --git a/lib/plausible/stats/imported/sql/where_builder.ex b/lib/plausible/stats/imported/sql/where_builder.exnew file mode 100644index 0000000000..de3d8cb1e9--- /dev/null+++ b/lib/plausible/stats/imported/sql/where_builder.ex@@ -0,0 +1,59 @@+defmodule Plausible.Stats.Imported.SQL.WhereBuilder do+ @moduledoc """+ A module for building an ecto where clause of a query out of a query for Imported tables.+ """++ import Ecto.Query++ alias Plausible.Stats.Query++ @db_field_mappings %{+ referrer_source: :source,+ screen_size: :device,+ screen: :device,+ os: :operating_system,+ os_version: :operating_system_version,+ country_code: :country,+ subdivision1_code: :region,+ city_geoname_id: :city,+ entry_page_hostname: :hostname,+ pathname: :page,+ url: :link_url+ }++ def build(%Query{filters: []}), do: true++ def build(query) do+ query.filters+ |> Enum.map(&add_filter(query, &1))+ |> Enum.reduce(fn condition, acc -> dynamic([], ^acc and ^condition) end)+ end++ defp add_filter(query, [:not, filter]) do+ dynamic([i], not (^add_filter(query, filter)))+ end++ defp add_filter(query, [:and, filters]) do+ filters+ |> Enum.map(&add_filter(query, &1))+ |> Enum.reduce(fn condition, acc -> dynamic([], ^acc and ^condition) end)+ end++ defp add_filter(query, [:or, filters]) do+ filters+ |> Enum.map(&add_filter(query, &1))+ |> Enum.reduce(fn condition, acc -> dynamic([], ^acc or ^condition) end)+ end++ defp add_filter(query, [_operation, dimension, _clauses] = filter) do+ db_field = Plausible.Stats.Filters.without_prefix(dimension)++ if db_field == :goal do+ Plausible.Goals.Filters.add_filter(query, filter, imported?: true)+ else+ mapped_db_field = Map.get(@db_field_mappings, db_field, db_field)++ Plausible.Stats.SQL.WhereBuilder.build_condition(mapped_db_field, filter)+ end+ end+enddiff --git a/lib/plausible/stats/query.ex b/lib/plausible/stats/query.exindex 44af1764d6..ee2d6556f4 100644--- a/lib/plausible/stats/query.ex+++ b/lib/plausible/stats/query.ex@@ -76,7 +76,13 @@ defmodule Plausible.Stats.Query do|> refresh_imported_opts()end- def remove_filters(query, prefixes) do+ @doc """+ Removes top level filters matching any of passed prefix from the query.++ Note that this doesn't handle cases with AND/OR/NOT and as such is discouraged+ from use.+ """+ def remove_top_level_filters(query, prefixes) donew_filters =Enum.reject(query.filters, fn [_, filter_key | _rest] ->Enum.any?(prefixes, &String.starts_with?(filter_key, &1))@@ -91,24 +97,6 @@ defmodule Plausible.Stats.Query doput_imported_opts(query, nil, %{})end- def has_event_filters?(query) do- Enum.any?(query.filters, fn [_op, prop | _rest] ->- String.starts_with?(prop, "event:")- end)- end-- def get_filter_by_prefix(query, prefix) do- Enum.find(query.filters, fn [_op, prop | _rest] ->- String.starts_with?(prop, prefix)- end)- end-- def get_filter(query, name) do- Enum.find(query.filters, fn [_op, prop | _rest] ->- prop == name- end)- end-def put_imported_opts(query, site, params) dorequested? = params["with_imported"] == "true" || query.include.imports@@ -141,6 +129,9 @@ defmodule Plausible.Stats.Query do:ok | {:error, :no_imported_data | :out_of_range | :unsupported_query | :not_requested}def ensure_include_imported(query, requested?) docond do+ not requested? ->+ {:error, :not_requested}+is_nil(query.latest_import_end_date) ->{:error, :no_imported_data}@@ -156,9 +147,6 @@ defmodule Plausible.Stats.Query donot Imported.schema_supports_query?(query) ->{:error, :unsupported_query}- not requested? ->- {:error, :not_requested}-true ->:okenddiff --git a/lib/plausible/stats/query_optimizer.ex b/lib/plausible/stats/query_optimizer.exindex f5e9b272b2..c93958503e 100644--- a/lib/plausible/stats/query_optimizer.ex+++ b/lib/plausible/stats/query_optimizer.ex@@ -4,7 +4,7 @@ defmodule Plausible.Stats.QueryOptimizer do"""use Plausible- alias Plausible.Stats.{Query, TableDecider, Util, DateTimeRange}+ alias Plausible.Stats.{DateTimeRange, Filters, Query, TableDecider, Util}@doc """This module manipulates an existing query, updating it according to business logic.@@ -112,6 +112,7 @@ defmodule Plausible.Stats.QueryOptimizer do# filter is present for breakdowns, add entry/exit page hostname# filtersdefp extend_hostname_filters_to_visit(query) do+ # Note: Only works since event:hostname is only allowed as a top level filterhostname_filters =query.filters|> Enum.filter(fn [_operation, filter_key | _rest] -> filter_key == "event:hostname" end)@@ -152,11 +153,9 @@ defmodule Plausible.Stats.QueryOptimizer dofilters =if "event:page" in query.dimensions do- query.filters- |> Enum.map(fn- [op, "event:page" | rest] -> [op, "visit:entry_page" | rest]- filter -> filter- end)+ Filters.rename_dimensions_used_in_filter(query.filters, %{+ "event:page" => "visit:entry_page"+ })elsequery.filtersenddiff --git a/lib/plausible/stats/sql/expression.ex b/lib/plausible/stats/sql/expression.exindex cdec499d15..34a0c2dec9 100644--- a/lib/plausible/stats/sql/expression.ex+++ b/lib/plausible/stats/sql/expression.ex@@ -12,7 +12,7 @@ defmodule Plausible.Stats.SQL.Expression doimport Ecto.Query- alias Plausible.Stats.{Query, SQL}+ alias Plausible.Stats.{Filters, SQL}@no_ref "Direct / None"@not_set "(not set)"@@ -246,7 +246,7 @@ defmodule Plausible.Stats.SQL.Expression dodef session_metric(:bounce_rate, query) do# :TRICKY: If page is passed to query, we only count bounce rate where users _entered_ at page.- event_page_filter = Query.get_filter(query, "event:page")+ event_page_filter = Filters.get_toplevel_filter(query, "event:page")condition = SQL.WhereBuilder.build_condition(:entry_page, event_page_filter)wrap_alias([], %{diff --git a/lib/plausible/stats/sql/query_builder.ex b/lib/plausible/stats/sql/query_builder.exindex d05389d385..dcc37f02d5 100644--- a/lib/plausible/stats/sql/query_builder.ex+++ b/lib/plausible/stats/sql/query_builder.ex@@ -89,7 +89,7 @@ defmodule Plausible.Stats.SQL.QueryBuilder doenddef join_events_if_needed(q, site, query) do- if Query.has_event_filters?(query) do+ if TableDecider.sessions_join_events?(query) doevents_q =from(e in "events_v2",where: ^SQL.WhereBuilder.build(:events, site, query),diff --git a/lib/plausible/stats/sql/special_metrics.ex b/lib/plausible/stats/sql/special_metrics.exindex c46a5f1d7b..5a1e7b39f7 100644--- a/lib/plausible/stats/sql/special_metrics.ex+++ b/lib/plausible/stats/sql/special_metrics.ex@@ -49,7 +49,7 @@ defmodule Plausible.Stats.SQL.SpecialMetrics doif :conversion_rate in query.metrics dototal_query =query- |> Query.remove_filters(["event:goal", "event:props"])+ |> Query.remove_top_level_filters(["event:goal", "event:props"])|> Query.set(dimensions: [],include_imported: query.include_imported@@ -89,7 +89,7 @@ defmodule Plausible.Stats.SQL.SpecialMetrics doif :group_conversion_rate in query.metrics dogroup_totals_query =query- |> Query.remove_filters(["event:goal", "event:props"])+ |> Query.remove_top_level_filters(["event:goal", "event:props"])|> Query.set(metrics: [:visitors],order_by: [],diff --git a/lib/plausible/stats/sql/where_builder.ex b/lib/plausible/stats/sql/where_builder.exindex 51a7b70904..fbf7e33f63 100644--- a/lib/plausible/stats/sql/where_builder.ex+++ b/lib/plausible/stats/sql/where_builder.ex@@ -75,6 +75,22 @@ defmodule Plausible.Stats.SQL.WhereBuilder do)end+ defp add_filter(table, query, [:not, filter]) do+ dynamic([e], not (^add_filter(table, query, filter)))+ end++ defp add_filter(table, query, [:and, filters]) do+ filters+ |> Enum.map(&add_filter(table, query, &1))+ |> Enum.reduce(fn condition, acc -> dynamic([], ^acc and ^condition) end)+ end++ defp add_filter(table, query, [:or, filters]) do+ filters+ |> Enum.map(&add_filter(table, query, &1))+ |> Enum.reduce(fn condition, acc -> dynamic([], ^acc or ^condition) end)+ end+defp add_filter(:events, _query, [:is, "event:name", list]) dodynamic([e], e.name in ^list)enddiff --git a/lib/plausible/stats/table_decider.ex b/lib/plausible/stats/table_decider.exindex 7f3efcf25e..c798220044 100644--- a/lib/plausible/stats/table_decider.ex+++ b/lib/plausible/stats/table_decider.ex@@ -5,15 +5,22 @@ defmodule Plausible.Stats.TableDecider do"""import Enum, only: [empty?: 1]+ import Plausible.Stats.Filters, only: [dimensions_used_in_filters: 1]alias Plausible.Stats.Querydef events_join_sessions?(query) do- query- |> filter_keys()+ query.filters+ |> dimensions_used_in_filters()|> Enum.any?(&(filters_partitioner(query, &1) == :session))end+ def sessions_join_events?(query) do+ query.filters+ |> dimensions_used_in_filters()+ |> Enum.any?(&(filters_partitioner(query, &1) == :event))+ end+def partition_metrics(metrics, query) do%{event: event_only_metrics,@@ -25,8 +32,8 @@ defmodule Plausible.Stats.TableDecider dopartition(metrics, query, &metric_partitioner/2)%{event: event_only_filters, session: session_only_filters} =- query- |> filter_keys()+ query.filters+ |> dimensions_used_in_filters()|> partition(query, &filters_partitioner/2)%{event: event_only_dimensions, session: session_only_dimensions} =@@ -55,11 +62,6 @@ defmodule Plausible.Stats.TableDecider doendend- defp filter_keys(query) do- query.filters- |> Enum.map(fn [_, filter_key | _rest] -> filter_key end)- end-defp metric_partitioner(%Query{v2: true}, :conversion_rate), do: :eitherdefp metric_partitioner(%Query{v2: true}, :group_conversion_rate), do: :eitherdefp metric_partitioner(%Query{v2: true}, :visitors), do: :eitherdiff --git a/lib/plausible_web/controllers/api/external_stats_controller.ex b/lib/plausible_web/controllers/api/external_stats_controller.exindex 490bd07ca9..f932b80d4c 100644--- a/lib/plausible_web/controllers/api/external_stats_controller.ex+++ b/lib/plausible_web/controllers/api/external_stats_controller.ex@@ -2,7 +2,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController douse PlausibleWeb, :controlleruse Plausible.Repouse PlausibleWeb.Plugs.ErrorHandler- alias Plausible.Stats.{Query, Compare, Comparisons, Metrics}+ alias Plausible.Stats.{Query, Compare, Comparisons, Metrics, Filters}def realtime_visitors(conn, _params) dosite = conn.assigns.site@@ -121,7 +121,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController do:ok | {:error, {402, String.t()}}defp ensure_custom_props_access(site, query) doallowed_props = Plausible.Props.allowed_for(site, bypass_setup?: true)- prop_filter = Query.get_filter_by_prefix(query, "event:props:")+ prop_filter = Filters.get_toplevel_filter(query, "event:props:")query_allowed? =case {prop_filter, query.dimensions, allowed_props} do@@ -165,10 +165,10 @@ defmodule PlausibleWeb.Api.ExternalStatsController dodefp validate_metric("time_on_page" = metric, query) docond do- Query.get_filter(query, "event:goal") ->+ Filters.filtering_on_dimension?(query, "event:goal") ->{:error, "Metric `#{metric}` cannot be queried when filtering by `event:goal`"}- Query.get_filter(query, "event:name") ->+ Filters.filtering_on_dimension?(query, "event:name") ->{:error, "Metric `#{metric}` cannot be queried when filtering by `event:name`"}query.dimensions == ["event:page"] ->@@ -178,7 +178,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController do{:error,"Metric `#{metric}` is not supported in breakdown queries (except `event:page` breakdown)"}- Query.get_filter(query, "event:page") ->+ Filters.filtering_on_dimension?(query, "event:page") ->{:ok, metric}true ->@@ -192,7 +192,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController doquery.dimensions == ["event:goal"] ->{:ok, metric}- Query.get_filter(query, "event:goal") ->+ Filters.filtering_on_dimension?(query, "event:goal") ->{:ok, metric}true ->@@ -207,7 +207,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController dodefp validate_metric("views_per_visit" = metric, query) docond do- Query.get_filter(query, "event:page") ->+ Filters.filtering_on_dimension?(query, "event:page") ->{:error, "Metric `#{metric}` cannot be queried with a filter on `event:page`."}not Enum.empty?(query.dimensions) ->diff --git a/lib/plausible_web/controllers/api/stats_controller.ex b/lib/plausible_web/controllers/api/stats_controller.exindex 8bcb2aec9d..375606d94c 100644--- a/lib/plausible_web/controllers/api/stats_controller.ex+++ b/lib/plausible_web/controllers/api/stats_controller.ex@@ -5,7 +5,7 @@ defmodule PlausibleWeb.Api.StatsController douse PlausibleWeb.Plugs.ErrorHandleralias Plausible.Stats- alias Plausible.Stats.{Query, Comparisons, Time, DateTimeRange}+ alias Plausible.Stats.{Query, Comparisons, Filters, Time, TableDecider, DateTimeRange}alias Plausible.Stats.Filters.LegacyDashboardFilterParseralias PlausibleWeb.Api.Helpers, as: H@@ -304,16 +304,16 @@ defmodule PlausibleWeb.Api.StatsController doenddefp fetch_top_stats(site, query, comparison_query) do- goal_filter = Query.get_filter(query, "event:goal")+ goal_filter? = Filters.filtering_on_dimension?(query, "event:goal")cond do- query.period == "30m" && goal_filter ->+ query.period == "30m" && goal_filter? ->fetch_goal_realtime_top_stats(site, query, comparison_query)query.period == "30m" ->fetch_realtime_top_stats(site, query, comparison_query)- goal_filter ->+ goal_filter? ->fetch_goal_top_stats(site, query, comparison_query)true ->@@ -403,7 +403,7 @@ defmodule PlausibleWeb.Api.StatsController doenddefp fetch_other_top_stats(site, query, comparison_query) do- page_filter? = Query.get_filter(query, "event:page")+ page_filter? = Filters.filtering_on_dimension?(query, "event:page")metrics = [:visitors, :visits, :pageviews, :sample_percent]@@ -495,7 +495,7 @@ defmodule PlausibleWeb.Api.StatsController do|> transform_keys(%{source: :name})if params["csv"] do- if Query.get_filter(query, "event:goal") do+ if Filters.filtering_on_dimension?(query, "event:goal") dores|> transform_keys(%{visitors: :conversions})|> to_csv([:name, :conversions, :conversion_rate])@@ -549,10 +549,10 @@ defmodule PlausibleWeb.Api.StatsController dodefp validate_funnel_query(query) docond do- Query.get_filter(query, "event:goal") ->+ Filters.filtering_on_dimension?(query, "event:goal") ->{:error, {:invalid_funnel_query, "goals"}}- Query.get_filter(query, "event:page") ->+ Filters.filtering_on_dimension?(query, "event:page") ->{:error, {:invalid_funnel_query, "pages"}}query.period == "realtime" ->@@ -576,7 +576,7 @@ defmodule PlausibleWeb.Api.StatsController do|> transform_keys(%{utm_medium: :name})if params["csv"] do- if Query.get_filter(query, "event:goal") do+ if Filters.filtering_on_dimension?(query, "event:goal") dores|> transform_keys(%{visitors: :conversions})|> to_csv([:name, :conversions, :conversion_rate])@@ -603,7 +603,7 @@ defmodule PlausibleWeb.Api.StatsController do|> transform_keys(%{utm_campaign: :name})if params["csv"] do- if Query.get_filter(query, "event:goal") do+ if Filters.filtering_on_dimension?(query, "event:goal") dores|> transform_keys(%{visitors: :conversions})|> to_csv([:name, :conversions, :conversion_rate])@@ -630,7 +630,7 @@ defmodule PlausibleWeb.Api.StatsController do|> transform_keys(%{utm_content: :name})if params["csv"] do- if Query.get_filter(query, "event:goal") do+ if Filters.filtering_on_dimension?(query, "event:goal") dores|> transform_keys(%{visitors: :conversions})|> to_csv([:name, :conversions, :conversion_rate])@@ -657,7 +657,7 @@ defmodule PlausibleWeb.Api.StatsController do|> transform_keys(%{utm_term: :name})if params["csv"] do- if Query.get_filter(query, "event:goal") do+ if Filters.filtering_on_dimension?(query, "event:goal") dores|> transform_keys(%{visitors: :conversions})|> to_csv([:name, :conversions, :conversion_rate])@@ -684,7 +684,7 @@ defmodule PlausibleWeb.Api.StatsController do|> transform_keys(%{utm_source: :name})if params["csv"] do- if Query.get_filter(query, "event:goal") do+ if Filters.filtering_on_dimension?(query, "event:goal") dores|> transform_keys(%{visitors: :conversions})|> to_csv([:name, :conversions, :conversion_rate])@@ -711,7 +711,7 @@ defmodule PlausibleWeb.Api.StatsController do|> transform_keys(%{referrer: :name})if params["csv"] do- if Query.get_filter(query, "event:goal") do+ if Filters.filtering_on_dimension?(query, "event:goal") dores|> transform_keys(%{visitors: :conversions})|> to_csv([:name, :conversions, :conversion_rate])@@ -826,7 +826,7 @@ defmodule PlausibleWeb.Api.StatsController do|> transform_keys(%{page: :name})if params["csv"] do- if Query.get_filter(query, "event:goal") do+ if Filters.filtering_on_dimension?(query, "event:goal") dopages|> transform_keys(%{visitors: :conversions})|> to_csv([:name, :conversions, :conversion_rate])@@ -853,7 +853,7 @@ defmodule PlausibleWeb.Api.StatsController do|> transform_keys(%{entry_page: :name})if params["csv"] do- if Query.get_filter(query, "event:goal") do+ if Filters.filtering_on_dimension?(query, "event:goal") doto_csv(entry_pages, [:name, :visitors, :conversion_rate], [:name,:conversions,@@ -888,7 +888,7 @@ defmodule PlausibleWeb.Api.StatsController do|> transform_keys(%{exit_page: :name})if params["csv"] do- if Query.get_filter(query, "event:goal") do+ if Filters.filtering_on_dimension?(query, "event:goal") doto_csv(exit_pages, [:name, :visitors, :conversion_rate], [:name,:conversions,@@ -911,14 +911,14 @@ defmodule PlausibleWeb.Api.StatsController doenddefp add_exit_rate(breakdown_results, site, query, limit) do- if Query.has_event_filters?(query) do+ if TableDecider.sessions_join_events?(query) dobreakdown_resultselsepages = Enum.map(breakdown_results, & &1[:exit_page])total_pageviews_query =query- |> Query.remove_filters(["visit:exit_page"])+ |> Query.remove_top_level_filters(["visit:exit_page"])|> Query.add_filter([:is, "event:page", pages])|> Query.set(dimensions: ["event:page"])@@ -959,7 +959,7 @@ defmodule PlausibleWeb.Api.StatsController doMap.put(country, :name, country_info.name)end)- if Query.get_filter(query, "event:goal") do+ if Filters.filtering_on_dimension?(query, "event:goal") docountries|> transform_keys(%{visitors: :conversions})|> to_csv([:name, :conversions, :conversion_rate])@@ -1018,7 +1018,7 @@ defmodule PlausibleWeb.Api.StatsController doend)if params["csv"] do- if Query.get_filter(query, "event:goal") do+ if Filters.filtering_on_dimension?(query, "event:goal") doregions|> transform_keys(%{visitors: :conversions})|> to_csv([:name, :conversions, :conversion_rate])@@ -1061,7 +1061,7 @@ defmodule PlausibleWeb.Api.StatsController doend)if params["csv"] do- if Query.get_filter(query, "event:goal") do+ if Filters.filtering_on_dimension?(query, "event:goal") docities|> transform_keys(%{visitors: :conversions})|> to_csv([:name, :conversions, :conversion_rate])@@ -1092,7 +1092,7 @@ defmodule PlausibleWeb.Api.StatsController do|> transform_keys(%{browser: :name})if params["csv"] do- if Query.get_filter(query, "event:goal") do+ if Filters.filtering_on_dimension?(query, "event:goal") dobrowsers|> transform_keys(%{visitors: :conversions})|> to_csv([:name, :conversions, :conversion_rate])@@ -1123,7 +1123,7 @@ defmodule PlausibleWeb.Api.StatsController do|> transform_keys(%{browser_version: :version})if params["csv"] do- if Query.get_filter(query, "event:goal") do+ if Filters.filtering_on_dimension?(query, "event:goal") doresults|> transform_keys(%{browser: :name, visitors: :conversions})|> to_csv([:name, :version, :conversions, :conversion_rate])@@ -1163,7 +1163,7 @@ defmodule PlausibleWeb.Api.StatsController do|> transform_keys(%{os: :name})if params["csv"] do- if Query.get_filter(query, "event:goal") do+ if Filters.filtering_on_dimension?(query, "event:goal") dosystems|> transform_keys(%{visitors: :conversions})|> to_csv([:name, :conversions, :conversion_rate])@@ -1194,7 +1194,7 @@ defmodule PlausibleWeb.Api.StatsController do|> transform_keys(%{os_version: :version})if params["csv"] do- if Query.get_filter(query, "event:goal") do+ if Filters.filtering_on_dimension?(query, "event:goal") doresults|> transform_keys(%{os: :name, visitors: :conversions})|> to_csv([:name, :version, :conversions, :conversion_rate])@@ -1234,7 +1234,7 @@ defmodule PlausibleWeb.Api.StatsController do|> transform_keys(%{device: :name})if params["csv"] do- if Query.get_filter(query, "event:goal") do+ if Filters.filtering_on_dimension?(query, "event:goal") dosizes|> transform_keys(%{visitors: :conversions})|> to_csv([:name, :conversions, :conversion_rate])@@ -1327,7 +1327,7 @@ defmodule PlausibleWeb.Api.StatsController do|> Enum.concat()percent_or_cr =- if Query.get_filter(query, "event:goal"),+ if Filters.filtering_on_dimension?(query, "event:goal"),do: :conversion_rate,else: :percentage@@ -1344,7 +1344,7 @@ defmodule PlausibleWeb.Api.StatsController doquery = Query.from(site, params, debug_metadata(conn))metrics =- if Query.get_filter(query, "event:goal") do+ if Filters.filtering_on_dimension?(query, "event:goal") do[:visitors, :events, :conversion_rate] ++ @revenue_metricselse[:visitors, :events, :percentage] ++ @revenue_metrics@@ -1505,7 +1505,7 @@ defmodule PlausibleWeb.Api.StatsController dorequires_goal_filter? = metric in [:conversion_rate, :events]- if requires_goal_filter? and !Query.get_filter(query, "event:goal") do+ if requires_goal_filter? and !Filters.filtering_on_dimension?(query, "event:goal") do{:error, "Metric `#{metric}` can only be queried with a goal filter"}else{:ok, metric}@@ -1547,7 +1547,7 @@ defmodule PlausibleWeb.Api.StatsController doenddefp breakdown_metrics(query, extra_metrics \\ []) do- if Query.get_filter(query, "event:goal") do+ if Filters.filtering_on_dimension?(query, "event:goal") do[:visitors, :conversion_rate, :total_visitors]else[:visitors] ++ extra_metricsdiff --git a/lib/plausible_web/controllers/stats_controller.ex b/lib/plausible_web/controllers/stats_controller.exindex 36a894dbea..e5e568cc27 100644--- a/lib/plausible_web/controllers/stats_controller.ex+++ b/lib/plausible_web/controllers/stats_controller.ex@@ -45,7 +45,7 @@ defmodule PlausibleWeb.StatsController douse Plausible.Repoalias Plausible.Sites- alias Plausible.Stats.Query+ alias Plausible.Stats.{Filters, Query}alias PlausibleWeb.Apiplug(PlausibleWeb.AuthorizeSiteAccess when action in [:stats, :csv_export])@@ -181,7 +181,7 @@ defmodule PlausibleWeb.StatsController dodefp csv_graph_metrics(query) do{metrics, column_headers} =- if Query.get_filter(query, "event:goal") do+ if Filters.filtering_on_dimension?(query, "event:goal") do{[:visitors, :events, :conversion_rate],[:date, :unique_conversions, :total_conversions, :conversion_rate]diff --git a/priv/json-schemas/query-api-schema.json b/priv/json-schemas/query-api-schema.jsonindex 8e62b288e3..e12f299710 100644--- a/priv/json-schemas/query-api-schema.json+++ b/priv/json-schemas/query-api-schema.json@@ -263,9 +263,7 @@{"$ref": "#/definitions/clauses"}- ],- "minItems": 3,- "maxItems": 3+ ]},{"type": "array",@@ -294,9 +292,43 @@{"$ref": "#/definitions/clauses"}- ],- "minItems": 3,- "maxItems": 3+ ]+ },+ {+ "$ref": "#/definitions/filter_and_or"+ },+ {+ "$ref": "#/definitions/filter_not"+ }+ ]+ },+ "filter_not": {+ "type": "array",+ "items": [+ {+ "const": "not"+ },+ {+ "$ref": "#/definitions/filter_entry"+ }+ ]+ },+ "filter_and_or": {+ "type": "array",+ "items": [+ {+ "type": "string",+ "enum": [+ "and",+ "or"+ ]+ },+ {+ "type": "array",+ "items": {+ "$ref": "#/definitions/filter_entry"+ },+ "minItems": 1}]},diff --git a/test/plausible/stats/query_parser_test.exs b/test/plausible/stats/query_parser_test.exsindex b1072db845..5aaf631461 100644--- a/test/plausible/stats/query_parser_test.exs+++ b/test/plausible/stats/query_parser_test.exs@@ -412,6 +412,113 @@ defmodule Plausible.Stats.Filters.QueryParserTest do"Invalid visit:country filter, visit:country needs to be a valid 2-letter country code.")end++ test "valid nested `not`, `and` and `or`", %{site: site} do+ %{+ "site_id" => site.domain,+ "metrics" => ["visitors"],+ "date_range" => "all",+ "filters" => [+ [+ "or",+ [+ [+ "and",+ [+ ["is", "visit:city_name", ["Tallinn"]],+ ["is", "visit:country_name", ["Estonia"]]+ ]+ ],+ ["not", ["is", "visit:country_name", ["Estonia"]]]+ ]+ ]+ ]+ }+ |> check_success(site, %{+ metrics: [:visitors],+ date_range: @date_range_day,+ filters: [+ [+ :or,+ [+ [+ :and,+ [+ [:is, "visit:city_name", ["Tallinn"]],+ [:is, "visit:country_name", ["Estonia"]]+ ]+ ],+ [:not, [:is, "visit:country_name", ["Estonia"]]]+ ]+ ]+ ],+ dimensions: [],+ order_by: nil,+ timezone: site.timezone,+ include: %{imports: false, time_labels: false},+ preloaded_goals: []+ })+ end++ test "invalid `not` clause", %{site: site} do+ %{+ "site_id" => site.domain,+ "metrics" => ["visitors"],+ "date_range" => "all",+ "filters" => [["not", []]]+ }+ |> check_error(+ site,+ "#/filters/0: Invalid filter [\"not\", []]"+ )+ end++ test "invalid `or` clause", %{site: site} do+ %{+ "site_id" => site.domain,+ "metrics" => ["visitors"],+ "date_range" => "all",+ "filters" => [["or", []]]+ }+ |> check_error(+ site,+ "#/filters/0: Invalid filter [\"or\", []]"+ )+ end++ test "event:hostname filter", %{site: site} do+ %{+ "site_id" => site.domain,+ "metrics" => ["visitors"],+ "date_range" => "all",+ "filters" => [["is", "event:hostname", ["a.plausible.io"]]]+ }+ |> check_success(site, %{+ metrics: [:visitors],+ date_range: @date_range_day,+ filters: [+ [:is, "event:hostname", ["a.plausible.io"]]+ ],+ dimensions: [],+ order_by: nil,+ timezone: site.timezone,+ include: %{imports: false, time_labels: false},+ preloaded_goals: []+ })+ end++ test "event:hostname filter not at top level is invalid", %{site: site} do+ %{+ "site_id" => site.domain,+ "metrics" => ["visitors"],+ "date_range" => "all",+ "filters" => [["not", ["is", "event:hostname", ["a.plausible.io"]]]]+ }+ |> check_error(+ site,+ "Invalid filters. Dimension `event:hostname` can only be filtered at the top level."+ )+ endenddescribe "include validation" do@@ -519,6 +626,42 @@ defmodule Plausible.Stats.Filters.QueryParserTest do"The goal `Visit /thank-you` is not configured for this site. Find out how to configure goals here: https://plausible.io/docs/stats-api#filtering-by-goals")end++ test "unsupported filter", %{site: site} do+ %{+ "site_id" => site.domain,+ "metrics" => ["visitors"],+ "date_range" => "all",+ "filters" => [+ ["is_not", "event:goal", ["Signup"]]+ ]+ }+ |> check_error(+ site,+ "#/filters/0: Invalid filter [\"is_not\", \"event:goal\", [\"Signup\"]]"+ )+ end++ test "not top-level filter", %{site: site} do+ %{+ "site_id" => site.domain,+ "metrics" => ["visitors"],+ "date_range" => "all",+ "filters" => [+ [+ "or",+ [+ ["is", "event:goal", ["Signup"]],+ ["is", "event:name", ["pageview"]]+ ]+ ]+ ]+ }+ |> check_error(+ site,+ "Invalid filters. Dimension `event:goal` can only be filtered at the top level."+ )+ endenddescribe "date range validation" do@@ -955,7 +1098,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest doenddescribe "custom props access" do- test "error if invalid filter", %{site: site, user: user} do+ test "filters - no access", %{site: site, user: user} doep =insert(:enterprise_plan, features: [Plausible.Billing.Feature.StatsAPI], user_id: user.id)@@ -965,7 +1108,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest do"site_id" => site.domain,"metrics" => ["visitors"],"date_range" => "all",- "filters" => [["is", "event:props:foobar", ["foo"]]]+ "filters" => [["not", ["is", "event:props:foobar", ["foo"]]]]}|> check_error(site,@@ -973,7 +1116,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest do)end- test "error if invalid dimension", %{site: site, user: user} do+ test "dimensions - no access", %{site: site, user: user} doep =insert(:enterprise_plan, features: [Plausible.Billing.Feature.StatsAPI], user_id: user.id)@@ -1044,6 +1187,42 @@ defmodule Plausible.Stats.Filters.QueryParserTest do# preloaded_goals: [goal]# })# end++ test "custom properties filter with special metric", %{site: site} do+ %{+ "site_id" => site.domain,+ "metrics" => ["conversion_rate", "group_conversion_rate"],+ "date_range" => "all",+ "filters" => [["is", "event:props:foo", ["bar"]]],+ "dimensions" => ["event:goal"]+ }+ |> check_success(site, %{+ metrics: [:conversion_rate, :group_conversion_rate],+ date_range: @date_range_day,+ filters: [+ [:is, "event:props:foo", ["bar"]]+ ],+ dimensions: ["event:goal"],+ order_by: nil,+ timezone: site.timezone,+ include: %{imports: false, time_labels: false},+ preloaded_goals: []+ })+ end++ test "not top level custom properties filter with special metric is invalid", %{site: site} do+ %{+ "site_id" => site.domain,+ "metrics" => ["conversion_rate", "group_conversion_rate"],+ "date_range" => "all",+ "filters" => [["not", ["is", "event:props:foo", ["bar"]]]],+ "dimensions" => ["event:goal"]+ }+ |> check_error(+ site,+ "Invalid filters. When `conversion_rate` or `group_conversion_rate` metrics are used, custom property filters can only be used on top level."+ )+ endenddescribe "views_per_visit metric" dodiff --git a/test/plausible/stats/table_decider_test.exs b/test/plausible/stats/table_decider_test.exsindex b216d02c7d..96d34023ff 100644--- a/test/plausible/stats/table_decider_test.exs+++ b/test/plausible/stats/table_decider_test.exs@@ -4,20 +4,27 @@ defmodule Plausible.Stats.TableDeciderTest doimport Plausible.Stats.TableDecider- test "events_join_sessions? with experimental_reduced_joins disabled" do- assert not events_join_sessions?(make_query(false, []))- assert not events_join_sessions?(make_query(false, ["event:name"]))- assert events_join_sessions?(make_query(false, ["visit:source"]))- assert events_join_sessions?(make_query(false, ["visit:entry_page"]))- assert events_join_sessions?(make_query(false, ["visit:exit_page"]))- end+ describe "events_join_sessions?" do+ test "with experimental_reduced_joins disabled" do+ assert not events_join_sessions?(make_query(false, []))+ assert not events_join_sessions?(make_query(false, ["event:name"]))+ assert events_join_sessions?(make_query(false, ["visit:source"]))+ assert events_join_sessions?(make_query(false, ["visit:entry_page"]))+ assert events_join_sessions?(make_query(false, ["visit:exit_page"]))+ end- test "events_join_sessions? with experimental_reduced_joins enabled" do- assert not events_join_sessions?(make_query(true, []))- assert not events_join_sessions?(make_query(true, ["event:name"]))- assert not events_join_sessions?(make_query(true, ["visit:source"]))- assert events_join_sessions?(make_query(true, ["visit:entry_page"]))- assert events_join_sessions?(make_query(true, ["visit:exit_page"]))+ test "with experimental_reduced_joins enabled" do+ assert not events_join_sessions?(make_query(true, []))+ assert not events_join_sessions?(make_query(true, ["event:name"]))+ assert not events_join_sessions?(make_query(true, ["visit:source"]))+ assert events_join_sessions?(make_query(true, ["visit:entry_page"]))+ assert events_join_sessions?(make_query(true, ["visit:exit_page"]))+ end++ test "with nested filter" do+ assert not events_join_sessions?(make_query([["not", ["is", "event:name", []]]]))+ assert events_join_sessions?(make_query([["not", ["is", "visit:exit_page", []]]]))+ endenddescribe "partition_metrics" do@@ -165,4 +172,11 @@ defmodule Plausible.Stats.TableDeciderTest do"dimensions" => dimensions})end++ defp make_query(filters) do+ Query.from(build(:site), %{+ "dimensions" => [],+ "filters" => filters+ })+ endenddiff --git a/test/plausible_web/controllers/api/external_stats_controller/query_imported_test.exs b/test/plausible_web/controllers/api/external_stats_controller/query_imported_test.exsindex 4063b50ee5..4f2becf889 100644--- a/test/plausible_web/controllers/api/external_stats_controller/query_imported_test.exs+++ b/test/plausible_web/controllers/api/external_stats_controller/query_imported_test.exs@@ -860,5 +860,73 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryImportedTest do%{"dimensions" => ["London", "Canada"], "metrics" => [1]}]end++ test "imported country and city names with complex conditions", %{+ site: site,+ conn: conn+ } do+ site_import = insert(:site_import, site: site)++ populate_stats(site, site_import.id, [+ build(:pageview,+ country_code: "GB",+ # London+ city_geoname_id: 2_643_743+ ),+ build(:pageview,+ country_code: "CA",+ # Different London+ city_geoname_id: 6_058_560+ ),+ build(:imported_locations,+ country: "EE",+ # Tallinn+ city: 588_409,+ visitors: 3+ ),+ build(:imported_locations,+ country: "EE",+ # Kärdla+ city: 591_632,+ visitors: 2+ ),+ build(:imported_locations,+ country: "GB",+ # London+ city: 2_643_743,+ visitors: 33+ )+ ])++ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "metrics" => ["visitors"],+ "date_range" => "all",+ "dimensions" => ["visit:city_name", "visit:country_name"],+ "filters" => [+ [+ "or",+ [+ [+ "and",+ [+ ["is", "visit:city_name", ["London"]],+ ["not", ["is", "visit:country_name", ["Canada"]]]+ ]+ ],+ ["is", "visit:country_name", ["Estonia"]]+ ]+ ]+ ],+ "include" => %{"imports" => true}+ })++ assert json_response(conn, 200)["results"] == [+ %{"dimensions" => ["London", "United Kingdom"], "metrics" => [34]},+ %{"dimensions" => ["Tallinn", "Estonia"], "metrics" => [3]},+ %{"dimensions" => ["Kärdla", "Estonia"], "metrics" => [2]}+ ]+ endendenddiff --git a/test/plausible_web/controllers/api/external_stats_controller/query_test.exs b/test/plausible_web/controllers/api/external_stats_controller/query_test.exsindex 6a74ca02af..cb820d9c7c 100644--- a/test/plausible_web/controllers/api/external_stats_controller/query_test.exs+++ b/test/plausible_web/controllers/api/external_stats_controller/query_test.exs@@ -752,7 +752,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest doassert json_response(conn, 200)["results"] == [%{"metrics" => [2], "dimensions" => []}]end- test "negated contains page filter", %{conn: conn, site: site} do+ test "does_not_contain page filter", %{conn: conn, site: site} dopopulate_stats(site, [build(:pageview, pathname: "/en/page1"),build(:pageview, pathname: "/en/page2"),@@ -772,6 +772,40 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest doassert json_response(conn, 200)["results"] == [%{"metrics" => [1], "dimensions" => []}]end+ test "contains with and/or/not filters", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:pageview, pathname: "/en/page1"),+ build(:pageview, pathname: "/en/page2"),+ build(:pageview, pathname: "/eng/page1"),+ build(:pageview, pathname: "/pl/page1"),+ build(:pageview, pathname: "/gb/page1")+ ])++ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "date_range" => "all",+ "metrics" => ["visitors"],+ "filters" => [+ [+ "or",+ [+ [+ "and",+ [+ ["contains", "event:page", ["/en"]],+ ["not", ["contains", "event:page", ["/eng"]]]+ ]+ ],+ ["contains", "event:page", ["/gb"]]+ ]+ ]+ ]+ })++ assert json_response(conn, 200)["results"] == [%{"metrics" => [3], "dimensions" => []}]+ end+test "contains and member filter combined", %{conn: conn, site: site} dopopulate_stats(site, [build(:pageview, pathname: "/en/page1"),