Skip to content

Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.

APIv2: and/or/not support (#4480)

Karl-Aksel Puulmann8fa3a831291d
diff --git a/extra/lib/plausible/stats/goal/revenue.ex b/extra/lib/plausible/stats/goal/revenue.ex
index 84d69d07e3..d08ab369c6 100644
--- a/extra/lib/plausible/stats/goal/revenue.ex
+++ b/extra/lib/plausible/stats/goal/revenue.ex
@@ -4,7 +4,7 @@ defmodule Plausible.Stats.Goal.Revenue do
"""
import Ecto.Query
- alias Plausible.Stats.Query
+ alias Plausible.Stats.Filters
@revenue_metrics [:average_revenue, :total_revenue]
@@ -25,7 +25,7 @@ defmodule Plausible.Stats.Goal.Revenue do
"""
def get_revenue_tracking_currency(site, query, metrics) do
goal_filters =
- case Query.get_filter(query, "event:goal") do
+ case Filters.get_toplevel_filter(query, "event:goal") do
[:is, "event:goal", list] -> list
_ -> []
end
diff --git a/lib/plausible/stats/aggregate.ex b/lib/plausible/stats/aggregate.ex
index bcf1faffa8..8bce4103dd 100644
--- a/lib/plausible/stats/aggregate.ex
+++ b/lib/plausible/stats/aggregate.ex
@@ -9,7 +9,7 @@ defmodule Plausible.Stats.Aggregate do
use Plausible
import Plausible.Stats.Base
import Ecto.Query
- alias Plausible.Stats.{Query, Util, SQL}
+ alias Plausible.Stats.{Query, Util, SQL, Filters}
def aggregate(site, query, metrics) do
{currency, metrics} =
@@ -49,7 +49,7 @@ defmodule Plausible.Stats.Aggregate do
defp aggregate_time_on_page(site, query) do
windowed_pages_q =
- from e in base_event_query(site, Query.remove_filters(query, ["event:page"])),
+ from e in base_event_query(site, Query.remove_top_level_filters(query, ["event:page"])),
select: %{
next_timestamp: over(fragment("leadInFrame(?)", e.timestamp), :event_horizon),
next_pathname: over(fragment("leadInFrame(?)", e.pathname), :event_horizon),
@@ -65,7 +65,7 @@ defmodule Plausible.Stats.Aggregate do
]
]
- event_page_filter = Query.get_filter(query, "event:page")
+ event_page_filter = Filters.get_toplevel_filter(query, "event:page")
timed_page_transitions_q =
from e in Ecto.Query.subquery(windowed_pages_q),
diff --git a/lib/plausible/stats/breakdown.ex b/lib/plausible/stats/breakdown.ex
index 356b4cabda..76f08c37eb 100644
--- a/lib/plausible/stats/breakdown.ex
+++ b/lib/plausible/stats/breakdown.ex
@@ -82,21 +82,24 @@ defmodule Plausible.Stats.Breakdown do
import Ecto.Query
windowed_pages_q =
- from e in base_event_query(site, Query.remove_filters(query, ["event:page", "event:props"])),
- select: %{
- next_timestamp: over(fragment("leadInFrame(?)", e.timestamp), :event_horizon),
- next_pathname: over(fragment("leadInFrame(?)", e.pathname), :event_horizon),
- timestamp: e.timestamp,
- pathname: e.pathname,
- session_id: e.session_id
- },
- windows: [
- event_horizon: [
- partition_by: e.session_id,
- order_by: e.timestamp,
- frame: fragment("ROWS BETWEEN CURRENT ROW AND 1 FOLLOWING")
- ]
- ]
+ from e in base_event_query(
+ site,
+ Query.remove_top_level_filters(query, ["event:page", "event:props"])
+ ),
+ select: %{
+ next_timestamp: over(fragment("leadInFrame(?)", e.timestamp), :event_horizon),
+ next_pathname: over(fragment("leadInFrame(?)", e.pathname), :event_horizon),
+ timestamp: e.timestamp,
+ pathname: e.pathname,
+ session_id: e.session_id
+ },
+ windows: [
+ event_horizon: [
+ partition_by: e.session_id,
+ order_by: e.timestamp,
+ frame: fragment("ROWS BETWEEN CURRENT ROW AND 1 FOLLOWING")
+ ]
+ ]
timed_page_transitions_q =
from e in subquery(windowed_pages_q),
diff --git a/lib/plausible/stats/custom_props.ex b/lib/plausible/stats/custom_props.ex
index 2b11d7ef92..932e5079f6 100644
--- a/lib/plausible/stats/custom_props.ex
+++ b/lib/plausible/stats/custom_props.ex
@@ -3,12 +3,12 @@ defmodule Plausible.Stats.CustomProps do
Module for querying user defined 'custom properties'.
"""
- alias Plausible.Stats.Query
+ alias Plausible.Stats.Filters
use Plausible.ClickhouseRepo
import Plausible.Stats.Base
def fetch_prop_names(site, query) do
- case Query.get_filter_by_prefix(query, "event:props:") do
+ case Filters.get_toplevel_filter(query, "event:props:") do
[_op, "event:props:" <> key | _rest] ->
[key]
diff --git a/lib/plausible/stats/filter_suggestions.ex b/lib/plausible/stats/filter_suggestions.ex
index b52dfea339..b9d9c32541 100644
--- a/lib/plausible/stats/filter_suggestions.ex
+++ b/lib/plausible/stats/filter_suggestions.ex
@@ -8,6 +8,7 @@ defmodule Plausible.Stats.FilterSuggestions do
alias Plausible.Stats.Query
alias Plausible.Stats.Imported
+ alias Plausible.Stats.Filters
def filter_suggestions(site, query, "country", filter_search) do
matches = Location.search_country(filter_search)
@@ -187,10 +188,10 @@ defmodule Plausible.Stats.FilterSuggestions do
def filter_suggestions(site, query, "prop_value", filter_search) do
filter_query = if filter_search == nil, do: "%", else: "%#{filter_search}%"
- [_op, "event:props:" <> key | _rest] = Query.get_filter_by_prefix(query, "event:props")
+ [_op, "event:props:" <> key | _rest] = Filters.get_toplevel_filter(query, "event:props")
none_q =
- from(e in base_event_query(site, Query.remove_filters(query, ["event:props"])),
+ from(e in base_event_query(site, Query.remove_top_level_filters(query, ["event:props"])),
select: "(none)",
where: not has_key(e, :meta, ^key),
limit: 1
diff --git a/lib/plausible/stats/filters/filters.ex b/lib/plausible/stats/filters/filters.ex
index 5195dd3a4e..62b15cedab 100644
--- a/lib/plausible/stats/filters/filters.ex
+++ b/lib/plausible/stats/filters/filters.ex
@@ -92,4 +92,89 @@ defmodule Plausible.Stats.Filters do
|> List.last()
|> String.to_existing_atom()
end
+
+ def dimensions_used_in_filters(filters, opts \\ []) do
+ min_depth = Keyword.get(opts, :min_depth, 0)
+
+ filters
+ |> traverse()
+ |> Enum.filter(fn {_filter, _root, depth} -> depth >= min_depth end)
+ |> Enum.map(fn {[_operator, dimension | _rest], _root, _depth} -> dimension end)
+ end
+
+ def filtering_on_dimension?(query, dimension) do
+ dimension in dimensions_used_in_filters(query.filters)
+ end
+
+ @doc """
+ Gets the first top level filter with matching dimension (or nil).
+
+ Only use in cases where it's known that filters are only set on the top level as it
+ does not handle AND/OR/NOT!
+ """
+ def get_toplevel_filter(query, prefix) do
+ Enum.find(query.filters, fn [_op, dimension | _rest] ->
+ String.starts_with?(dimension, prefix)
+ end)
+ end
+
+ def rename_dimensions_used_in_filter(filters, renames) do
+ transform_filters(filters, fn
+ [operation, dimension, clauses] ->
+ [[operation, Map.get(renames, dimension, dimension), clauses]]
+
+ _subtree ->
+ nil
+ end)
+ end
+
+ @doc """
+ Updates filters via `transformer`.
+
+ Transformer will receive each node (filter, and/or/not subtree) of
+ query and must return a list of nodes to replace it with or nil
+ to ignore and look deeper.
+ """
+ def transform_filters(filters, transformer) do
+ filters
+ |> Enum.flat_map(&transform_tree(&1, transformer))
+ end
+
+ defp transform_tree(filter, transformer) do
+ case {transformer.(filter), filter} do
+ # Transformer did not return that value - transform that subtree
+ {nil, [:not, child_filter]} ->
+ [[:not, transform_tree(child_filter, transformer)]]
+
+ {nil, [operation, filters]} when operation in [:and, :or] ->
+ [[operation, transform_filters(filters, transformer)]]
+
+ # Reached a leaf node, return existing value
+ {nil, filter} ->
+ [[filter]]
+
+ # Transformer returned a value - don't transform that subtree
+ {transformed_filters, _filter} ->
+ transformed_filters
+ end
+ end
+
+ defp traverse(filters, root \\ nil, depth \\ -1) do
+ filters
+ |> Enum.flat_map(&traverse_tree(&1, root || &1, depth + 1))
+ end
+
+ defp traverse_tree(filter, root, depth) do
+ case filter do
+ [:not, child_filter] ->
+ traverse_tree(child_filter, root, depth + 1)
+
+ [operation, filters] when operation in [:and, :or] ->
+ traverse(filters, root || filter, depth + 1)
+
+ # Leaf node
+ _ ->
+ [{filter, root, depth}]
+ end
+ end
end
diff --git a/lib/plausible/stats/filters/query_parser.ex b/lib/plausible/stats/filters/query_parser.ex
index 4814f322ac..d79cbacc6e 100644
--- a/lib/plausible/stats/filters/query_parser.ex
+++ b/lib/plausible/stats/filters/query_parser.ex
@@ -1,7 +1,7 @@
defmodule Plausible.Stats.Filters.QueryParser do
@moduledoc false
- alias Plausible.Stats.{TableDecider, Filters, Query, Metrics, DateTimeRange, JSONSchema}
+ alias Plausible.Stats.{TableDecider, Filters, Metrics, DateTimeRange, JSONSchema}
@default_include %{
imports: false,
@@ -36,8 +36,10 @@ defmodule Plausible.Stats.Filters.QueryParser do
include: include
},
:ok <- validate_order_by(query),
- :ok <- validate_goal_filters(query),
:ok <- validate_custom_props_access(site, query),
+ :ok <- validate_toplevel_only_filter_dimension(query),
+ :ok <- validate_special_metrics_filters(query),
+ :ok <- validate_filtered_goals_exist(query),
:ok <- validate_metrics(query),
:ok <- validate_include(query) do
{:ok, query}
@@ -63,9 +65,9 @@ defmodule Plausible.Stats.Filters.QueryParser do
defp parse_filter(filter) do
with {:ok, operator} <- parse_operator(filter),
- {:ok, filter_key} <- parse_filter_key(filter),
+ {:ok, second} <- parse_filter_second(operator, filter),
{:ok, rest} <- parse_filter_rest(operator, filter) do
- {:ok, [operator, filter_key | rest]}
+ {:ok, [operator, second | rest]}
end
end
@@ -75,8 +77,18 @@ defmodule Plausible.Stats.Filters.QueryParser do
defp parse_operator(["does_not_match" | _rest]), do: {:ok, :does_not_match}
defp parse_operator(["contains" | _rest]), do: {:ok, :contains}
defp parse_operator(["does_not_contain" | _rest]), do: {:ok, :does_not_contain}
+ defp parse_operator(["not" | _rest]), do: {:ok, :not}
+ defp parse_operator(["and" | _rest]), do: {:ok, :and}
+ defp parse_operator(["or" | _rest]), do: {:ok, :or}
defp parse_operator(filter), do: {:error, "Unknown operator for filter '#{i(filter)}'."}
+ def parse_filter_second(:not, [_, filter | _rest]), do: parse_filter(filter)
+
+ def parse_filter_second(operator, [_, filters | _rest]) when operator in [:and, :or],
+ do: parse_filters(filters)
+
+ def parse_filter_second(_operator, filter), do: parse_filter_key(filter)
+
defp parse_filter_key([_operator, filter_key | _rest] = filter) do
parse_filter_key_string(filter_key, "Invalid filter '#{i(filter)}")
end
@@ -87,6 +99,10 @@ defmodule Plausible.Stats.Filters.QueryParser do
when operator in [:is, :is_not, :matches, :does_not_match, :contains, :does_not_contain],
do: parse_clauses_list(filter)
+ defp parse_filter_rest(operator, _filter)
+ when operator in [:not, :and, :or],
+ do: {:ok, []}
+
defp parse_clauses_list([operation, filter_key, list] = filter) when is_list(list) do
all_strings? = Enum.all?(list, &is_binary/1)
all_integers? = Enum.all?(list, &is_integer/1)
@@ -359,7 +375,37 @@ defmodule Plausible.Stats.Filters.QueryParser do
end
end
- defp validate_goal_filters(query) do
+ @only_toplevel ["event:goal", "event:hostname"]
+ defp validate_toplevel_only_filter_dimension(query) do
+ not_toplevel = Filters.dimensions_used_in_filters(query.filters, min_depth: 1)
+
+ if Enum.any?(not_toplevel, &(&1 in @only_toplevel)) do
+ {:error,
+ "Invalid filters. Dimension `#{List.first(not_toplevel)}` can only be filtered at the top level."}
+ else
+ :ok
+ end
+ end
+
+ @special_metrics [:conversion_rate, :group_conversion_rate]
+ defp validate_special_metrics_filters(query) do
+ special_metric? = Enum.any?(@special_metrics, &(&1 in query.metrics))
+
+ deep_custom_property? =
+ query.filters
+ |> Filters.dimensions_used_in_filters(min_depth: 1)
+ |> Enum.any?(fn dimension -> String.starts_with?(dimension, "event:props:") end)
+
+ if special_metric? and deep_custom_property? do
+ {:error,
+ "Invalid filters. When `conversion_rate` or `group_conversion_rate` metrics are used, custom property filters can only be used on top level."}
+ else
+ :ok
+ end
+ end
+
+ defp validate_filtered_goals_exist(query) do
+ # Note: Only works since event:goal is allowed as a top level filter
goal_filter_clauses =
Enum.flat_map(query.filters, fn
[:is, "event:goal", clauses] -> clauses
@@ -396,7 +442,7 @@ defmodule Plausible.Stats.Filters.QueryParser do
defp validate_custom_props_access(_site, query, allowed_props) do
valid? =
query.filters
- |> Enum.map(fn [_operation, filter_key | _rest] -> filter_key end)
+ |> Filters.dimensions_used_in_filters()
|> Enum.concat(query.dimensions)
|> Enum.all?(fn
"event:props:" <> prop -> prop in allowed_props
@@ -418,7 +464,7 @@ defmodule Plausible.Stats.Filters.QueryParser do
defp validate_metric(metric, query) when metric in [:conversion_rate, :group_conversion_rate] do
if Enum.member?(query.dimensions, "event:goal") or
- not is_nil(Query.get_filter(query, "event:goal")) do
+ Filters.filtering_on_dimension?(query, "event:goal") do
:ok
else
{:error, "Metric `#{metric}` can only be queried with event:goal filters or dimensions."}
@@ -427,7 +473,7 @@ defmodule Plausible.Stats.Filters.QueryParser do
defp validate_metric(:views_per_visit = metric, query) do
cond do
- not is_nil(Query.get_filter(query, "event:page")) ->
+ Filters.filtering_on_dimension?(query, "event:page") ->
{:error, "Metric `#{metric}` cannot be queried with a filter on `event:page`."}
length(query.dimensions) > 0 ->
diff --git a/lib/plausible/stats/imported/base.ex b/lib/plausible/stats/imported/base.ex
index 23bfebda41..4a8744b5d5 100644
--- a/lib/plausible/stats/imported/base.ex
+++ b/lib/plausible/stats/imported/base.ex
@@ -6,7 +6,9 @@ defmodule Plausible.Stats.Imported.Base do
import Ecto.Query
alias Plausible.Imported
- alias Plausible.Stats.{Filters, Query, SQL, DateTimeRange}
+ alias Plausible.Stats.{Query, DateTimeRange}
+
+ import Plausible.Stats.Filters, only: [dimensions_used_in_filters: 1]
@property_to_table_mappings %{
"visit:source" => "imported_sources",
@@ -47,20 +49,6 @@ defmodule Plausible.Stats.Imported.Base do
@imported_custom_props Imported.imported_custom_props()
- @db_field_mappings %{
- referrer_source: :source,
- screen_size: :device,
- screen: :device,
- os: :operating_system,
- os_version: :operating_system_version,
- country_code: :country,
- subdivision1_code: :region,
- city_geoname_id: :city,
- entry_page_hostname: :hostname,
- pathname: :page,
- url: :link_url
- }
-
def property_to_table_mappings(), do: @property_to_table_mappings
def query_imported(site, query) do
@@ -78,9 +66,9 @@ defmodule Plausible.Stats.Imported.Base do
where: i.import_id in ^import_ids,
where: i.date >= ^date_from,
where: i.date <= ^date_to,
+ where: ^Plausible.Stats.Imported.SQL.WhereBuilder.build(query),
select: %{}
)
- |> apply_filter(query)
end
def decide_tables(query) do
@@ -92,8 +80,7 @@ defmodule Plausible.Stats.Imported.Base do
end
defp custom_prop_query?(query) do
- query.filters
- |> Enum.map(&Enum.at(&1, 1))
+ dimensions_used_in_filters(query.filters)
|> Enum.concat(query.dimensions)
|> Enum.any?(&(&1 in @imported_custom_props))
end
@@ -108,8 +95,7 @@ defmodule Plausible.Stats.Imported.Base do
if dimensions == [] or
(length(dimensions) == 1 and hd(dimensions) in @queriable_custom_prop_dimensions) do
custom_prop_filters =
- query.filters
- |> Enum.map(&Enum.at(&1, 1))
+ dimensions_used_in_filters(query.filters)
|> Enum.filter(&(&1 in @imported_custom_props))
|> Enum.uniq()
@@ -153,8 +139,8 @@ defmodule Plausible.Stats.Imported.Base do
["imported_pages", "imported_custom_events"]
end
- defp do_decide_tables(%Query{filters: filters, dimensions: ["event:goal"]} = query) do
- filter_props = Enum.map(filters, &Enum.at(&1, 1))
+ defp do_decide_tables(%Query{dimensions: ["event:goal"]} = query) do
+ filter_dimensions = dimensions_used_in_filters(query.filters)
filter_goals = get_filter_goals(query)
@@ -163,11 +149,11 @@ defmodule Plausible.Stats.Imported.Base do
any_pageview_goals? =
Enum.any?(filter_goals, fn goal -> Plausible.Goal.type(goal) == :page end)
- any_event_name_filters? = "event:name" in filter_props or any_event_goals?
- any_page_filters? = "event:page" in filter_props or any_pageview_goals?
+ any_event_name_filters? = "event:name" in filter_dimensions or any_event_goals?
+ any_page_filters? = "event:page" in filter_dimensions or any_pageview_goals?
any_other_filters? =
- Enum.any?(filter_props, &(&1 not in ["event:page", "event:name", "event:goal"]))
+ Enum.any?(filter_dimensions, &(&1 not in ["event:page", "event:name", "event:goal"]))
cond do
any_other_filters? -> []
@@ -177,11 +163,10 @@ defmodule Plausible.Stats.Imported.Base do
end
end
- defp do_decide_tables(%Query{filters: filters, dimensions: dimensions} = query) do
+ defp do_decide_tables(query) do
table_candidates =
- filters
- |> Enum.map(fn [_, filter_key | _] -> filter_key end)
- |> Enum.concat(dimensions)
+ dimensions_used_in_filters(query.filters)
+ |> Enum.concat(query.dimensions)
|> Enum.reject(&(&1 in @queriable_time_dimensions or &1 == "event:goal"))
|> Enum.flat_map(fn
"visit:screen" -> ["visit:device"]
@@ -206,10 +191,10 @@ defmodule Plausible.Stats.Imported.Base do
end
end
- defp get_filter_goals(%Query{filters: filters} = query) do
- filters
- |> Enum.filter(fn [_, key, _] -> key == "event:goal" end)
- |> Enum.flat_map(fn [operation, _, clauses] ->
+ defp get_filter_goals(query) do
+ query.filters
+ |> Enum.filter(fn [_, dimension | _rest] -> dimension == "event:goal" end)
+ |> Enum.flat_map(fn [operation, _dimension, clauses] ->
Enum.flat_map(clauses, fn clause ->
query.preloaded_goals
|> Plausible.Goals.Filters.filter_preloaded(operation, clause)
@@ -217,22 +202,6 @@ defmodule Plausible.Stats.Imported.Base do
end)
end
- defp apply_filter(q, %Query{filters: filters} = query) do
- Enum.reduce(filters, q, fn [_, filter_key, _] = filter, q ->
- db_field = Filters.without_prefix(filter_key)
-
- if db_field == :goal do
- condition = Plausible.Goals.Filters.add_filter(query, filter, imported?: true)
- where(q, ^condition)
- else
- mapped_db_field = Map.get(@db_field_mappings, db_field, db_field)
- condition = SQL.WhereBuilder.build_condition(mapped_db_field, filter)
-
- where(q, ^condition)
- end
- end)
- end
-
def special_goals_for("event:props:url"), do: Imported.goals_with_url()
def special_goals_for("event:props:path"), do: Imported.goals_with_path()
end
diff --git a/lib/plausible/stats/imported/imported.ex b/lib/plausible/stats/imported/imported.ex
index 36e13605a9..e2330f08bd 100644
--- a/lib/plausible/stats/imported/imported.ex
+++ b/lib/plausible/stats/imported/imported.ex
@@ -3,7 +3,7 @@ defmodule Plausible.Stats.Imported do
use Plausible.Stats.SQL.Fragments
import Ecto.Query
- import Plausible.Stats.Imported.SQL.Builder
+ import Plausible.Stats.Imported.SQL.Expression
alias Plausible.Stats.Filters
alias Plausible.Stats.Imported
diff --git a/lib/plausible/stats/imported/sql/builder.ex b/lib/plausible/stats/imported/sql/expression.ex
similarity index 99%
rename from lib/plausible/stats/imported/sql/builder.ex
rename to lib/plausible/stats/imported/sql/expression.ex
index 76a8b011d0..776796950e 100644
--- a/lib/plausible/stats/imported/sql/builder.ex
+++ b/lib/plausible/stats/imported/sql/expression.ex
@@ -1,4 +1,4 @@
-defmodule Plausible.Stats.Imported.SQL.Builder do
+defmodule Plausible.Stats.Imported.SQL.Expression do
@moduledoc """
This module is responsible for generating SQL/Ecto expressions
for dimensions, filters and metrics used in import table queries
diff --git a/lib/plausible/stats/imported/sql/where_builder.ex b/lib/plausible/stats/imported/sql/where_builder.ex
new file mode 100644
index 0000000000..de3d8cb1e9
--- /dev/null
+++ b/lib/plausible/stats/imported/sql/where_builder.ex
@@ -0,0 +1,59 @@
+defmodule Plausible.Stats.Imported.SQL.WhereBuilder do
+ @moduledoc """
+ A module for building an ecto where clause of a query out of a query for Imported tables.
+ """
+
+ import Ecto.Query
+
+ alias Plausible.Stats.Query
+
+ @db_field_mappings %{
+ referrer_source: :source,
+ screen_size: :device,
+ screen: :device,
+ os: :operating_system,
+ os_version: :operating_system_version,
+ country_code: :country,
+ subdivision1_code: :region,
+ city_geoname_id: :city,
+ entry_page_hostname: :hostname,
+ pathname: :page,
+ url: :link_url
+ }
+
+ def build(%Query{filters: []}), do: true
+
+ def build(query) do
+ query.filters
+ |> Enum.map(&add_filter(query, &1))
+ |> Enum.reduce(fn condition, acc -> dynamic([], ^acc and ^condition) end)
+ end
+
+ defp add_filter(query, [:not, filter]) do
+ dynamic([i], not (^add_filter(query, filter)))
+ end
+
+ defp add_filter(query, [:and, filters]) do
+ filters
+ |> Enum.map(&add_filter(query, &1))
+ |> Enum.reduce(fn condition, acc -> dynamic([], ^acc and ^condition) end)
+ end
+
+ defp add_filter(query, [:or, filters]) do
+ filters
+ |> Enum.map(&add_filter(query, &1))
+ |> Enum.reduce(fn condition, acc -> dynamic([], ^acc or ^condition) end)
+ end
+
+ defp add_filter(query, [_operation, dimension, _clauses] = filter) do
+ db_field = Plausible.Stats.Filters.without_prefix(dimension)
+
+ if db_field == :goal do
+ Plausible.Goals.Filters.add_filter(query, filter, imported?: true)
+ else
+ mapped_db_field = Map.get(@db_field_mappings, db_field, db_field)
+
+ Plausible.Stats.SQL.WhereBuilder.build_condition(mapped_db_field, filter)
+ end
+ end
+end
diff --git a/lib/plausible/stats/query.ex b/lib/plausible/stats/query.ex
index 44af1764d6..ee2d6556f4 100644
--- a/lib/plausible/stats/query.ex
+++ b/lib/plausible/stats/query.ex
@@ -76,7 +76,13 @@ defmodule Plausible.Stats.Query do
|> refresh_imported_opts()
end
- def remove_filters(query, prefixes) do
+ @doc """
+ Removes top level filters matching any of passed prefix from the query.
+
+ Note that this doesn't handle cases with AND/OR/NOT and as such is discouraged
+ from use.
+ """
+ def remove_top_level_filters(query, prefixes) do
new_filters =
Enum.reject(query.filters, fn [_, filter_key | _rest] ->
Enum.any?(prefixes, &String.starts_with?(filter_key, &1))
@@ -91,24 +97,6 @@ defmodule Plausible.Stats.Query do
put_imported_opts(query, nil, %{})
end
- def has_event_filters?(query) do
- Enum.any?(query.filters, fn [_op, prop | _rest] ->
- String.starts_with?(prop, "event:")
- end)
- end
-
- def get_filter_by_prefix(query, prefix) do
- Enum.find(query.filters, fn [_op, prop | _rest] ->
- String.starts_with?(prop, prefix)
- end)
- end
-
- def get_filter(query, name) do
- Enum.find(query.filters, fn [_op, prop | _rest] ->
- prop == name
- end)
- end
-
def put_imported_opts(query, site, params) do
requested? = params["with_imported"] == "true" || query.include.imports
@@ -141,6 +129,9 @@ defmodule Plausible.Stats.Query do
:ok | {:error, :no_imported_data | :out_of_range | :unsupported_query | :not_requested}
def ensure_include_imported(query, requested?) do
cond do
+ not requested? ->
+ {:error, :not_requested}
+
is_nil(query.latest_import_end_date) ->
{:error, :no_imported_data}
@@ -156,9 +147,6 @@ defmodule Plausible.Stats.Query do
not Imported.schema_supports_query?(query) ->
{:error, :unsupported_query}
- not requested? ->
- {:error, :not_requested}
-
true ->
:ok
end
diff --git a/lib/plausible/stats/query_optimizer.ex b/lib/plausible/stats/query_optimizer.ex
index f5e9b272b2..c93958503e 100644
--- a/lib/plausible/stats/query_optimizer.ex
+++ b/lib/plausible/stats/query_optimizer.ex
@@ -4,7 +4,7 @@ defmodule Plausible.Stats.QueryOptimizer do
"""
use Plausible
- alias Plausible.Stats.{Query, TableDecider, Util, DateTimeRange}
+ alias Plausible.Stats.{DateTimeRange, Filters, Query, TableDecider, Util}
@doc """
This module manipulates an existing query, updating it according to business logic.
@@ -112,6 +112,7 @@ defmodule Plausible.Stats.QueryOptimizer do
# filter is present for breakdowns, add entry/exit page hostname
# filters
defp extend_hostname_filters_to_visit(query) do
+ # Note: Only works since event:hostname is only allowed as a top level filter
hostname_filters =
query.filters
|> Enum.filter(fn [_operation, filter_key | _rest] -> filter_key == "event:hostname" end)
@@ -152,11 +153,9 @@ defmodule Plausible.Stats.QueryOptimizer do
filters =
if "event:page" in query.dimensions do
- query.filters
- |> Enum.map(fn
- [op, "event:page" | rest] -> [op, "visit:entry_page" | rest]
- filter -> filter
- end)
+ Filters.rename_dimensions_used_in_filter(query.filters, %{
+ "event:page" => "visit:entry_page"
+ })
else
query.filters
end
diff --git a/lib/plausible/stats/sql/expression.ex b/lib/plausible/stats/sql/expression.ex
index cdec499d15..34a0c2dec9 100644
--- a/lib/plausible/stats/sql/expression.ex
+++ b/lib/plausible/stats/sql/expression.ex
@@ -12,7 +12,7 @@ defmodule Plausible.Stats.SQL.Expression do
import Ecto.Query
- alias Plausible.Stats.{Query, SQL}
+ alias Plausible.Stats.{Filters, SQL}
@no_ref "Direct / None"
@not_set "(not set)"
@@ -246,7 +246,7 @@ defmodule Plausible.Stats.SQL.Expression do
def session_metric(:bounce_rate, query) do
# :TRICKY: If page is passed to query, we only count bounce rate where users _entered_ at page.
- event_page_filter = Query.get_filter(query, "event:page")
+ event_page_filter = Filters.get_toplevel_filter(query, "event:page")
condition = SQL.WhereBuilder.build_condition(:entry_page, event_page_filter)
wrap_alias([], %{
diff --git a/lib/plausible/stats/sql/query_builder.ex b/lib/plausible/stats/sql/query_builder.ex
index d05389d385..dcc37f02d5 100644
--- a/lib/plausible/stats/sql/query_builder.ex
+++ b/lib/plausible/stats/sql/query_builder.ex
@@ -89,7 +89,7 @@ defmodule Plausible.Stats.SQL.QueryBuilder do
end
def join_events_if_needed(q, site, query) do
- if Query.has_event_filters?(query) do
+ if TableDecider.sessions_join_events?(query) do
events_q =
from(e in "events_v2",
where: ^SQL.WhereBuilder.build(:events, site, query),
diff --git a/lib/plausible/stats/sql/special_metrics.ex b/lib/plausible/stats/sql/special_metrics.ex
index c46a5f1d7b..5a1e7b39f7 100644
--- a/lib/plausible/stats/sql/special_metrics.ex
+++ b/lib/plausible/stats/sql/special_metrics.ex
@@ -49,7 +49,7 @@ defmodule Plausible.Stats.SQL.SpecialMetrics do
if :conversion_rate in query.metrics do
total_query =
query
- |> Query.remove_filters(["event:goal", "event:props"])
+ |> Query.remove_top_level_filters(["event:goal", "event:props"])
|> Query.set(
dimensions: [],
include_imported: query.include_imported
@@ -89,7 +89,7 @@ defmodule Plausible.Stats.SQL.SpecialMetrics do
if :group_conversion_rate in query.metrics do
group_totals_query =
query
- |> Query.remove_filters(["event:goal", "event:props"])
+ |> Query.remove_top_level_filters(["event:goal", "event:props"])
|> Query.set(
metrics: [:visitors],
order_by: [],
diff --git a/lib/plausible/stats/sql/where_builder.ex b/lib/plausible/stats/sql/where_builder.ex
index 51a7b70904..fbf7e33f63 100644
--- a/lib/plausible/stats/sql/where_builder.ex
+++ b/lib/plausible/stats/sql/where_builder.ex
@@ -75,6 +75,22 @@ defmodule Plausible.Stats.SQL.WhereBuilder do
)
end
+ defp add_filter(table, query, [:not, filter]) do
+ dynamic([e], not (^add_filter(table, query, filter)))
+ end
+
+ defp add_filter(table, query, [:and, filters]) do
+ filters
+ |> Enum.map(&add_filter(table, query, &1))
+ |> Enum.reduce(fn condition, acc -> dynamic([], ^acc and ^condition) end)
+ end
+
+ defp add_filter(table, query, [:or, filters]) do
+ filters
+ |> Enum.map(&add_filter(table, query, &1))
+ |> Enum.reduce(fn condition, acc -> dynamic([], ^acc or ^condition) end)
+ end
+
defp add_filter(:events, _query, [:is, "event:name", list]) do
dynamic([e], e.name in ^list)
end
diff --git a/lib/plausible/stats/table_decider.ex b/lib/plausible/stats/table_decider.ex
index 7f3efcf25e..c798220044 100644
--- a/lib/plausible/stats/table_decider.ex
+++ b/lib/plausible/stats/table_decider.ex
@@ -5,15 +5,22 @@ defmodule Plausible.Stats.TableDecider do
"""
import Enum, only: [empty?: 1]
+ import Plausible.Stats.Filters, only: [dimensions_used_in_filters: 1]
alias Plausible.Stats.Query
def events_join_sessions?(query) do
- query
- |> filter_keys()
+ query.filters
+ |> dimensions_used_in_filters()
|> Enum.any?(&(filters_partitioner(query, &1) == :session))
end
+ def sessions_join_events?(query) do
+ query.filters
+ |> dimensions_used_in_filters()
+ |> Enum.any?(&(filters_partitioner(query, &1) == :event))
+ end
+
def partition_metrics(metrics, query) do
%{
event: event_only_metrics,
@@ -25,8 +32,8 @@ defmodule Plausible.Stats.TableDecider do
partition(metrics, query, &metric_partitioner/2)
%{event: event_only_filters, session: session_only_filters} =
- query
- |> filter_keys()
+ query.filters
+ |> dimensions_used_in_filters()
|> partition(query, &filters_partitioner/2)
%{event: event_only_dimensions, session: session_only_dimensions} =
@@ -55,11 +62,6 @@ defmodule Plausible.Stats.TableDecider do
end
end
- defp filter_keys(query) do
- query.filters
- |> Enum.map(fn [_, filter_key | _rest] -> filter_key end)
- end
-
defp metric_partitioner(%Query{v2: true}, :conversion_rate), do: :either
defp metric_partitioner(%Query{v2: true}, :group_conversion_rate), do: :either
defp metric_partitioner(%Query{v2: true}, :visitors), do: :either
diff --git a/lib/plausible_web/controllers/api/external_stats_controller.ex b/lib/plausible_web/controllers/api/external_stats_controller.ex
index 490bd07ca9..f932b80d4c 100644
--- a/lib/plausible_web/controllers/api/external_stats_controller.ex
+++ b/lib/plausible_web/controllers/api/external_stats_controller.ex
@@ -2,7 +2,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController do
use PlausibleWeb, :controller
use Plausible.Repo
use PlausibleWeb.Plugs.ErrorHandler
- alias Plausible.Stats.{Query, Compare, Comparisons, Metrics}
+ alias Plausible.Stats.{Query, Compare, Comparisons, Metrics, Filters}
def realtime_visitors(conn, _params) do
site = conn.assigns.site
@@ -121,7 +121,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController do
:ok | {:error, {402, String.t()}}
defp ensure_custom_props_access(site, query) do
allowed_props = Plausible.Props.allowed_for(site, bypass_setup?: true)
- prop_filter = Query.get_filter_by_prefix(query, "event:props:")
+ prop_filter = Filters.get_toplevel_filter(query, "event:props:")
query_allowed? =
case {prop_filter, query.dimensions, allowed_props} do
@@ -165,10 +165,10 @@ defmodule PlausibleWeb.Api.ExternalStatsController do
defp validate_metric("time_on_page" = metric, query) do
cond do
- Query.get_filter(query, "event:goal") ->
+ Filters.filtering_on_dimension?(query, "event:goal") ->
{:error, "Metric `#{metric}` cannot be queried when filtering by `event:goal`"}
- Query.get_filter(query, "event:name") ->
+ Filters.filtering_on_dimension?(query, "event:name") ->
{:error, "Metric `#{metric}` cannot be queried when filtering by `event:name`"}
query.dimensions == ["event:page"] ->
@@ -178,7 +178,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController do
{:error,
"Metric `#{metric}` is not supported in breakdown queries (except `event:page` breakdown)"}
- Query.get_filter(query, "event:page") ->
+ Filters.filtering_on_dimension?(query, "event:page") ->
{:ok, metric}
true ->
@@ -192,7 +192,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController do
query.dimensions == ["event:goal"] ->
{:ok, metric}
- Query.get_filter(query, "event:goal") ->
+ Filters.filtering_on_dimension?(query, "event:goal") ->
{:ok, metric}
true ->
@@ -207,7 +207,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController do
defp validate_metric("views_per_visit" = metric, query) do
cond do
- Query.get_filter(query, "event:page") ->
+ Filters.filtering_on_dimension?(query, "event:page") ->
{:error, "Metric `#{metric}` cannot be queried with a filter on `event:page`."}
not Enum.empty?(query.dimensions) ->
diff --git a/lib/plausible_web/controllers/api/stats_controller.ex b/lib/plausible_web/controllers/api/stats_controller.ex
index 8bcb2aec9d..375606d94c 100644
--- a/lib/plausible_web/controllers/api/stats_controller.ex
+++ b/lib/plausible_web/controllers/api/stats_controller.ex
@@ -5,7 +5,7 @@ defmodule PlausibleWeb.Api.StatsController do
use PlausibleWeb.Plugs.ErrorHandler
alias Plausible.Stats
- alias Plausible.Stats.{Query, Comparisons, Time, DateTimeRange}
+ alias Plausible.Stats.{Query, Comparisons, Filters, Time, TableDecider, DateTimeRange}
alias Plausible.Stats.Filters.LegacyDashboardFilterParser
alias PlausibleWeb.Api.Helpers, as: H
@@ -304,16 +304,16 @@ defmodule PlausibleWeb.Api.StatsController do
end
defp fetch_top_stats(site, query, comparison_query) do
- goal_filter = Query.get_filter(query, "event:goal")
+ goal_filter? = Filters.filtering_on_dimension?(query, "event:goal")
cond do
- query.period == "30m" && goal_filter ->
+ query.period == "30m" && goal_filter? ->
fetch_goal_realtime_top_stats(site, query, comparison_query)
query.period == "30m" ->
fetch_realtime_top_stats(site, query, comparison_query)
- goal_filter ->
+ goal_filter? ->
fetch_goal_top_stats(site, query, comparison_query)
true ->
@@ -403,7 +403,7 @@ defmodule PlausibleWeb.Api.StatsController do
end
defp fetch_other_top_stats(site, query, comparison_query) do
- page_filter? = Query.get_filter(query, "event:page")
+ page_filter? = Filters.filtering_on_dimension?(query, "event:page")
metrics = [:visitors, :visits, :pageviews, :sample_percent]
@@ -495,7 +495,7 @@ defmodule PlausibleWeb.Api.StatsController do
|> transform_keys(%{source: :name})
if params["csv"] do
- if Query.get_filter(query, "event:goal") do
+ if Filters.filtering_on_dimension?(query, "event:goal") do
res
|> transform_keys(%{visitors: :conversions})
|> to_csv([:name, :conversions, :conversion_rate])
@@ -549,10 +549,10 @@ defmodule PlausibleWeb.Api.StatsController do
defp validate_funnel_query(query) do
cond do
- Query.get_filter(query, "event:goal") ->
+ Filters.filtering_on_dimension?(query, "event:goal") ->
{:error, {:invalid_funnel_query, "goals"}}
- Query.get_filter(query, "event:page") ->
+ Filters.filtering_on_dimension?(query, "event:page") ->
{:error, {:invalid_funnel_query, "pages"}}
query.period == "realtime" ->
@@ -576,7 +576,7 @@ defmodule PlausibleWeb.Api.StatsController do
|> transform_keys(%{utm_medium: :name})
if params["csv"] do
- if Query.get_filter(query, "event:goal") do
+ if Filters.filtering_on_dimension?(query, "event:goal") do
res
|> transform_keys(%{visitors: :conversions})
|> to_csv([:name, :conversions, :conversion_rate])
@@ -603,7 +603,7 @@ defmodule PlausibleWeb.Api.StatsController do
|> transform_keys(%{utm_campaign: :name})
if params["csv"] do
- if Query.get_filter(query, "event:goal") do
+ if Filters.filtering_on_dimension?(query, "event:goal") do
res
|> transform_keys(%{visitors: :conversions})
|> to_csv([:name, :conversions, :conversion_rate])
@@ -630,7 +630,7 @@ defmodule PlausibleWeb.Api.StatsController do
|> transform_keys(%{utm_content: :name})
if params["csv"] do
- if Query.get_filter(query, "event:goal") do
+ if Filters.filtering_on_dimension?(query, "event:goal") do
res
|> transform_keys(%{visitors: :conversions})
|> to_csv([:name, :conversions, :conversion_rate])
@@ -657,7 +657,7 @@ defmodule PlausibleWeb.Api.StatsController do
|> transform_keys(%{utm_term: :name})
if params["csv"] do
- if Query.get_filter(query, "event:goal") do
+ if Filters.filtering_on_dimension?(query, "event:goal") do
res
|> transform_keys(%{visitors: :conversions})
|> to_csv([:name, :conversions, :conversion_rate])
@@ -684,7 +684,7 @@ defmodule PlausibleWeb.Api.StatsController do
|> transform_keys(%{utm_source: :name})
if params["csv"] do
- if Query.get_filter(query, "event:goal") do
+ if Filters.filtering_on_dimension?(query, "event:goal") do
res
|> transform_keys(%{visitors: :conversions})
|> to_csv([:name, :conversions, :conversion_rate])
@@ -711,7 +711,7 @@ defmodule PlausibleWeb.Api.StatsController do
|> transform_keys(%{referrer: :name})
if params["csv"] do
- if Query.get_filter(query, "event:goal") do
+ if Filters.filtering_on_dimension?(query, "event:goal") do
res
|> transform_keys(%{visitors: :conversions})
|> to_csv([:name, :conversions, :conversion_rate])
@@ -826,7 +826,7 @@ defmodule PlausibleWeb.Api.StatsController do
|> transform_keys(%{page: :name})
if params["csv"] do
- if Query.get_filter(query, "event:goal") do
+ if Filters.filtering_on_dimension?(query, "event:goal") do
pages
|> transform_keys(%{visitors: :conversions})
|> to_csv([:name, :conversions, :conversion_rate])
@@ -853,7 +853,7 @@ defmodule PlausibleWeb.Api.StatsController do
|> transform_keys(%{entry_page: :name})
if params["csv"] do
- if Query.get_filter(query, "event:goal") do
+ if Filters.filtering_on_dimension?(query, "event:goal") do
to_csv(entry_pages, [:name, :visitors, :conversion_rate], [
:name,
:conversions,
@@ -888,7 +888,7 @@ defmodule PlausibleWeb.Api.StatsController do
|> transform_keys(%{exit_page: :name})
if params["csv"] do
- if Query.get_filter(query, "event:goal") do
+ if Filters.filtering_on_dimension?(query, "event:goal") do
to_csv(exit_pages, [:name, :visitors, :conversion_rate], [
:name,
:conversions,
@@ -911,14 +911,14 @@ defmodule PlausibleWeb.Api.StatsController do
end
defp add_exit_rate(breakdown_results, site, query, limit) do
- if Query.has_event_filters?(query) do
+ if TableDecider.sessions_join_events?(query) do
breakdown_results
else
pages = Enum.map(breakdown_results, & &1[:exit_page])
total_pageviews_query =
query
- |> Query.remove_filters(["visit:exit_page"])
+ |> Query.remove_top_level_filters(["visit:exit_page"])
|> Query.add_filter([:is, "event:page", pages])
|> Query.set(dimensions: ["event:page"])
@@ -959,7 +959,7 @@ defmodule PlausibleWeb.Api.StatsController do
Map.put(country, :name, country_info.name)
end)
- if Query.get_filter(query, "event:goal") do
+ if Filters.filtering_on_dimension?(query, "event:goal") do
countries
|> transform_keys(%{visitors: :conversions})
|> to_csv([:name, :conversions, :conversion_rate])
@@ -1018,7 +1018,7 @@ defmodule PlausibleWeb.Api.StatsController do
end)
if params["csv"] do
- if Query.get_filter(query, "event:goal") do
+ if Filters.filtering_on_dimension?(query, "event:goal") do
regions
|> transform_keys(%{visitors: :conversions})
|> to_csv([:name, :conversions, :conversion_rate])
@@ -1061,7 +1061,7 @@ defmodule PlausibleWeb.Api.StatsController do
end)
if params["csv"] do
- if Query.get_filter(query, "event:goal") do
+ if Filters.filtering_on_dimension?(query, "event:goal") do
cities
|> transform_keys(%{visitors: :conversions})
|> to_csv([:name, :conversions, :conversion_rate])
@@ -1092,7 +1092,7 @@ defmodule PlausibleWeb.Api.StatsController do
|> transform_keys(%{browser: :name})
if params["csv"] do
- if Query.get_filter(query, "event:goal") do
+ if Filters.filtering_on_dimension?(query, "event:goal") do
browsers
|> transform_keys(%{visitors: :conversions})
|> to_csv([:name, :conversions, :conversion_rate])
@@ -1123,7 +1123,7 @@ defmodule PlausibleWeb.Api.StatsController do
|> transform_keys(%{browser_version: :version})
if params["csv"] do
- if Query.get_filter(query, "event:goal") do
+ if Filters.filtering_on_dimension?(query, "event:goal") do
results
|> transform_keys(%{browser: :name, visitors: :conversions})
|> to_csv([:name, :version, :conversions, :conversion_rate])
@@ -1163,7 +1163,7 @@ defmodule PlausibleWeb.Api.StatsController do
|> transform_keys(%{os: :name})
if params["csv"] do
- if Query.get_filter(query, "event:goal") do
+ if Filters.filtering_on_dimension?(query, "event:goal") do
systems
|> transform_keys(%{visitors: :conversions})
|> to_csv([:name, :conversions, :conversion_rate])
@@ -1194,7 +1194,7 @@ defmodule PlausibleWeb.Api.StatsController do
|> transform_keys(%{os_version: :version})
if params["csv"] do
- if Query.get_filter(query, "event:goal") do
+ if Filters.filtering_on_dimension?(query, "event:goal") do
results
|> transform_keys(%{os: :name, visitors: :conversions})
|> to_csv([:name, :version, :conversions, :conversion_rate])
@@ -1234,7 +1234,7 @@ defmodule PlausibleWeb.Api.StatsController do
|> transform_keys(%{device: :name})
if params["csv"] do
- if Query.get_filter(query, "event:goal") do
+ if Filters.filtering_on_dimension?(query, "event:goal") do
sizes
|> transform_keys(%{visitors: :conversions})
|> to_csv([:name, :conversions, :conversion_rate])
@@ -1327,7 +1327,7 @@ defmodule PlausibleWeb.Api.StatsController do
|> Enum.concat()
percent_or_cr =
- if Query.get_filter(query, "event:goal"),
+ if Filters.filtering_on_dimension?(query, "event:goal"),
do: :conversion_rate,
else: :percentage
@@ -1344,7 +1344,7 @@ defmodule PlausibleWeb.Api.StatsController do
query = Query.from(site, params, debug_metadata(conn))
metrics =
- if Query.get_filter(query, "event:goal") do
+ if Filters.filtering_on_dimension?(query, "event:goal") do
[:visitors, :events, :conversion_rate] ++ @revenue_metrics
else
[:visitors, :events, :percentage] ++ @revenue_metrics
@@ -1505,7 +1505,7 @@ defmodule PlausibleWeb.Api.StatsController do
requires_goal_filter? = metric in [:conversion_rate, :events]
- if requires_goal_filter? and !Query.get_filter(query, "event:goal") do
+ if requires_goal_filter? and !Filters.filtering_on_dimension?(query, "event:goal") do
{:error, "Metric `#{metric}` can only be queried with a goal filter"}
else
{:ok, metric}
@@ -1547,7 +1547,7 @@ defmodule PlausibleWeb.Api.StatsController do
end
defp breakdown_metrics(query, extra_metrics \\ []) do
- if Query.get_filter(query, "event:goal") do
+ if Filters.filtering_on_dimension?(query, "event:goal") do
[:visitors, :conversion_rate, :total_visitors]
else
[:visitors] ++ extra_metrics
diff --git a/lib/plausible_web/controllers/stats_controller.ex b/lib/plausible_web/controllers/stats_controller.ex
index 36a894dbea..e5e568cc27 100644
--- a/lib/plausible_web/controllers/stats_controller.ex
+++ b/lib/plausible_web/controllers/stats_controller.ex
@@ -45,7 +45,7 @@ defmodule PlausibleWeb.StatsController do
use Plausible.Repo
alias Plausible.Sites
- alias Plausible.Stats.Query
+ alias Plausible.Stats.{Filters, Query}
alias PlausibleWeb.Api
plug(PlausibleWeb.AuthorizeSiteAccess when action in [:stats, :csv_export])
@@ -181,7 +181,7 @@ defmodule PlausibleWeb.StatsController do
defp csv_graph_metrics(query) do
{metrics, column_headers} =
- if Query.get_filter(query, "event:goal") do
+ if Filters.filtering_on_dimension?(query, "event:goal") do
{
[:visitors, :events, :conversion_rate],
[:date, :unique_conversions, :total_conversions, :conversion_rate]
diff --git a/priv/json-schemas/query-api-schema.json b/priv/json-schemas/query-api-schema.json
index 8e62b288e3..e12f299710 100644
--- a/priv/json-schemas/query-api-schema.json
+++ b/priv/json-schemas/query-api-schema.json
@@ -263,9 +263,7 @@
{
"$ref": "#/definitions/clauses"
}
- ],
- "minItems": 3,
- "maxItems": 3
+ ]
},
{
"type": "array",
@@ -294,9 +292,43 @@
{
"$ref": "#/definitions/clauses"
}
- ],
- "minItems": 3,
- "maxItems": 3
+ ]
+ },
+ {
+ "$ref": "#/definitions/filter_and_or"
+ },
+ {
+ "$ref": "#/definitions/filter_not"
+ }
+ ]
+ },
+ "filter_not": {
+ "type": "array",
+ "items": [
+ {
+ "const": "not"
+ },
+ {
+ "$ref": "#/definitions/filter_entry"
+ }
+ ]
+ },
+ "filter_and_or": {
+ "type": "array",
+ "items": [
+ {
+ "type": "string",
+ "enum": [
+ "and",
+ "or"
+ ]
+ },
+ {
+ "type": "array",
+ "items": {
+ "$ref": "#/definitions/filter_entry"
+ },
+ "minItems": 1
}
]
},
diff --git a/test/plausible/stats/query_parser_test.exs b/test/plausible/stats/query_parser_test.exs
index b1072db845..5aaf631461 100644
--- a/test/plausible/stats/query_parser_test.exs
+++ b/test/plausible/stats/query_parser_test.exs
@@ -412,6 +412,113 @@ defmodule Plausible.Stats.Filters.QueryParserTest do
"Invalid visit:country filter, visit:country needs to be a valid 2-letter country code."
)
end
+
+ test "valid nested `not`, `and` and `or`", %{site: site} do
+ %{
+ "site_id" => site.domain,
+ "metrics" => ["visitors"],
+ "date_range" => "all",
+ "filters" => [
+ [
+ "or",
+ [
+ [
+ "and",
+ [
+ ["is", "visit:city_name", ["Tallinn"]],
+ ["is", "visit:country_name", ["Estonia"]]
+ ]
+ ],
+ ["not", ["is", "visit:country_name", ["Estonia"]]]
+ ]
+ ]
+ ]
+ }
+ |> check_success(site, %{
+ metrics: [:visitors],
+ date_range: @date_range_day,
+ filters: [
+ [
+ :or,
+ [
+ [
+ :and,
+ [
+ [:is, "visit:city_name", ["Tallinn"]],
+ [:is, "visit:country_name", ["Estonia"]]
+ ]
+ ],
+ [:not, [:is, "visit:country_name", ["Estonia"]]]
+ ]
+ ]
+ ],
+ dimensions: [],
+ order_by: nil,
+ timezone: site.timezone,
+ include: %{imports: false, time_labels: false},
+ preloaded_goals: []
+ })
+ end
+
+ test "invalid `not` clause", %{site: site} do
+ %{
+ "site_id" => site.domain,
+ "metrics" => ["visitors"],
+ "date_range" => "all",
+ "filters" => [["not", []]]
+ }
+ |> check_error(
+ site,
+ "#/filters/0: Invalid filter [\"not\", []]"
+ )
+ end
+
+ test "invalid `or` clause", %{site: site} do
+ %{
+ "site_id" => site.domain,
+ "metrics" => ["visitors"],
+ "date_range" => "all",
+ "filters" => [["or", []]]
+ }
+ |> check_error(
+ site,
+ "#/filters/0: Invalid filter [\"or\", []]"
+ )
+ end
+
+ test "event:hostname filter", %{site: site} do
+ %{
+ "site_id" => site.domain,
+ "metrics" => ["visitors"],
+ "date_range" => "all",
+ "filters" => [["is", "event:hostname", ["a.plausible.io"]]]
+ }
+ |> check_success(site, %{
+ metrics: [:visitors],
+ date_range: @date_range_day,
+ filters: [
+ [:is, "event:hostname", ["a.plausible.io"]]
+ ],
+ dimensions: [],
+ order_by: nil,
+ timezone: site.timezone,
+ include: %{imports: false, time_labels: false},
+ preloaded_goals: []
+ })
+ end
+
+ test "event:hostname filter not at top level is invalid", %{site: site} do
+ %{
+ "site_id" => site.domain,
+ "metrics" => ["visitors"],
+ "date_range" => "all",
+ "filters" => [["not", ["is", "event:hostname", ["a.plausible.io"]]]]
+ }
+ |> check_error(
+ site,
+ "Invalid filters. Dimension `event:hostname` can only be filtered at the top level."
+ )
+ end
end
describe "include validation" do
@@ -519,6 +626,42 @@ defmodule Plausible.Stats.Filters.QueryParserTest do
"The goal `Visit /thank-you` is not configured for this site. Find out how to configure goals here: https://plausible.io/docs/stats-api#filtering-by-goals"
)
end
+
+ test "unsupported filter", %{site: site} do
+ %{
+ "site_id" => site.domain,
+ "metrics" => ["visitors"],
+ "date_range" => "all",
+ "filters" => [
+ ["is_not", "event:goal", ["Signup"]]
+ ]
+ }
+ |> check_error(
+ site,
+ "#/filters/0: Invalid filter [\"is_not\", \"event:goal\", [\"Signup\"]]"
+ )
+ end
+
+ test "not top-level filter", %{site: site} do
+ %{
+ "site_id" => site.domain,
+ "metrics" => ["visitors"],
+ "date_range" => "all",
+ "filters" => [
+ [
+ "or",
+ [
+ ["is", "event:goal", ["Signup"]],
+ ["is", "event:name", ["pageview"]]
+ ]
+ ]
+ ]
+ }
+ |> check_error(
+ site,
+ "Invalid filters. Dimension `event:goal` can only be filtered at the top level."
+ )
+ end
end
describe "date range validation" do
@@ -955,7 +1098,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest do
end
describe "custom props access" do
- test "error if invalid filter", %{site: site, user: user} do
+ test "filters - no access", %{site: site, user: user} do
ep =
insert(:enterprise_plan, features: [Plausible.Billing.Feature.StatsAPI], user_id: user.id)
@@ -965,7 +1108,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest do
"site_id" => site.domain,
"metrics" => ["visitors"],
"date_range" => "all",
- "filters" => [["is", "event:props:foobar", ["foo"]]]
+ "filters" => [["not", ["is", "event:props:foobar", ["foo"]]]]
}
|> check_error(
site,
@@ -973,7 +1116,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest do
)
end
- test "error if invalid dimension", %{site: site, user: user} do
+ test "dimensions - no access", %{site: site, user: user} do
ep =
insert(:enterprise_plan, features: [Plausible.Billing.Feature.StatsAPI], user_id: user.id)
@@ -1044,6 +1187,42 @@ defmodule Plausible.Stats.Filters.QueryParserTest do
# preloaded_goals: [goal]
# })
# end
+
+ test "custom properties filter with special metric", %{site: site} do
+ %{
+ "site_id" => site.domain,
+ "metrics" => ["conversion_rate", "group_conversion_rate"],
+ "date_range" => "all",
+ "filters" => [["is", "event:props:foo", ["bar"]]],
+ "dimensions" => ["event:goal"]
+ }
+ |> check_success(site, %{
+ metrics: [:conversion_rate, :group_conversion_rate],
+ date_range: @date_range_day,
+ filters: [
+ [:is, "event:props:foo", ["bar"]]
+ ],
+ dimensions: ["event:goal"],
+ order_by: nil,
+ timezone: site.timezone,
+ include: %{imports: false, time_labels: false},
+ preloaded_goals: []
+ })
+ end
+
+ test "not top level custom properties filter with special metric is invalid", %{site: site} do
+ %{
+ "site_id" => site.domain,
+ "metrics" => ["conversion_rate", "group_conversion_rate"],
+ "date_range" => "all",
+ "filters" => [["not", ["is", "event:props:foo", ["bar"]]]],
+ "dimensions" => ["event:goal"]
+ }
+ |> check_error(
+ site,
+ "Invalid filters. When `conversion_rate` or `group_conversion_rate` metrics are used, custom property filters can only be used on top level."
+ )
+ end
end
describe "views_per_visit metric" do
diff --git a/test/plausible/stats/table_decider_test.exs b/test/plausible/stats/table_decider_test.exs
index b216d02c7d..96d34023ff 100644
--- a/test/plausible/stats/table_decider_test.exs
+++ b/test/plausible/stats/table_decider_test.exs
@@ -4,20 +4,27 @@ defmodule Plausible.Stats.TableDeciderTest do
import Plausible.Stats.TableDecider
- test "events_join_sessions? with experimental_reduced_joins disabled" do
- assert not events_join_sessions?(make_query(false, []))
- assert not events_join_sessions?(make_query(false, ["event:name"]))
- assert events_join_sessions?(make_query(false, ["visit:source"]))
- assert events_join_sessions?(make_query(false, ["visit:entry_page"]))
- assert events_join_sessions?(make_query(false, ["visit:exit_page"]))
- end
+ describe "events_join_sessions?" do
+ test "with experimental_reduced_joins disabled" do
+ assert not events_join_sessions?(make_query(false, []))
+ assert not events_join_sessions?(make_query(false, ["event:name"]))
+ assert events_join_sessions?(make_query(false, ["visit:source"]))
+ assert events_join_sessions?(make_query(false, ["visit:entry_page"]))
+ assert events_join_sessions?(make_query(false, ["visit:exit_page"]))
+ end
- test "events_join_sessions? with experimental_reduced_joins enabled" do
- assert not events_join_sessions?(make_query(true, []))
- assert not events_join_sessions?(make_query(true, ["event:name"]))
- assert not events_join_sessions?(make_query(true, ["visit:source"]))
- assert events_join_sessions?(make_query(true, ["visit:entry_page"]))
- assert events_join_sessions?(make_query(true, ["visit:exit_page"]))
+ test "with experimental_reduced_joins enabled" do
+ assert not events_join_sessions?(make_query(true, []))
+ assert not events_join_sessions?(make_query(true, ["event:name"]))
+ assert not events_join_sessions?(make_query(true, ["visit:source"]))
+ assert events_join_sessions?(make_query(true, ["visit:entry_page"]))
+ assert events_join_sessions?(make_query(true, ["visit:exit_page"]))
+ end
+
+ test "with nested filter" do
+ assert not events_join_sessions?(make_query([["not", ["is", "event:name", []]]]))
+ assert events_join_sessions?(make_query([["not", ["is", "visit:exit_page", []]]]))
+ end
end
describe "partition_metrics" do
@@ -165,4 +172,11 @@ defmodule Plausible.Stats.TableDeciderTest do
"dimensions" => dimensions
})
end
+
+ defp make_query(filters) do
+ Query.from(build(:site), %{
+ "dimensions" => [],
+ "filters" => filters
+ })
+ end
end
diff --git a/test/plausible_web/controllers/api/external_stats_controller/query_imported_test.exs b/test/plausible_web/controllers/api/external_stats_controller/query_imported_test.exs
index 4063b50ee5..4f2becf889 100644
--- a/test/plausible_web/controllers/api/external_stats_controller/query_imported_test.exs
+++ b/test/plausible_web/controllers/api/external_stats_controller/query_imported_test.exs
@@ -860,5 +860,73 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryImportedTest do
%{"dimensions" => ["London", "Canada"], "metrics" => [1]}
]
end
+
+ test "imported country and city names with complex conditions", %{
+ site: site,
+ conn: conn
+ } do
+ site_import = insert(:site_import, site: site)
+
+ populate_stats(site, site_import.id, [
+ build(:pageview,
+ country_code: "GB",
+ # London
+ city_geoname_id: 2_643_743
+ ),
+ build(:pageview,
+ country_code: "CA",
+ # Different London
+ city_geoname_id: 6_058_560
+ ),
+ build(:imported_locations,
+ country: "EE",
+ # Tallinn
+ city: 588_409,
+ visitors: 3
+ ),
+ build(:imported_locations,
+ country: "EE",
+ # Kärdla
+ city: 591_632,
+ visitors: 2
+ ),
+ build(:imported_locations,
+ country: "GB",
+ # London
+ city: 2_643_743,
+ visitors: 33
+ )
+ ])
+
+ conn =
+ post(conn, "/api/v2/query", %{
+ "site_id" => site.domain,
+ "metrics" => ["visitors"],
+ "date_range" => "all",
+ "dimensions" => ["visit:city_name", "visit:country_name"],
+ "filters" => [
+ [
+ "or",
+ [
+ [
+ "and",
+ [
+ ["is", "visit:city_name", ["London"]],
+ ["not", ["is", "visit:country_name", ["Canada"]]]
+ ]
+ ],
+ ["is", "visit:country_name", ["Estonia"]]
+ ]
+ ]
+ ],
+ "include" => %{"imports" => true}
+ })
+
+ assert json_response(conn, 200)["results"] == [
+ %{"dimensions" => ["London", "United Kingdom"], "metrics" => [34]},
+ %{"dimensions" => ["Tallinn", "Estonia"], "metrics" => [3]},
+ %{"dimensions" => ["Kärdla", "Estonia"], "metrics" => [2]}
+ ]
+ end
end
end
diff --git a/test/plausible_web/controllers/api/external_stats_controller/query_test.exs b/test/plausible_web/controllers/api/external_stats_controller/query_test.exs
index 6a74ca02af..cb820d9c7c 100644
--- a/test/plausible_web/controllers/api/external_stats_controller/query_test.exs
+++ b/test/plausible_web/controllers/api/external_stats_controller/query_test.exs
@@ -752,7 +752,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest do
assert json_response(conn, 200)["results"] == [%{"metrics" => [2], "dimensions" => []}]
end
- test "negated contains page filter", %{conn: conn, site: site} do
+ test "does_not_contain page filter", %{conn: conn, site: site} do
populate_stats(site, [
build(:pageview, pathname: "/en/page1"),
build(:pageview, pathname: "/en/page2"),
@@ -772,6 +772,40 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest do
assert json_response(conn, 200)["results"] == [%{"metrics" => [1], "dimensions" => []}]
end
+ test "contains with and/or/not filters", %{conn: conn, site: site} do
+ populate_stats(site, [
+ build(:pageview, pathname: "/en/page1"),
+ build(:pageview, pathname: "/en/page2"),
+ build(:pageview, pathname: "/eng/page1"),
+ build(:pageview, pathname: "/pl/page1"),
+ build(:pageview, pathname: "/gb/page1")
+ ])
+
+ conn =
+ post(conn, "/api/v2/query", %{
+ "site_id" => site.domain,
+ "date_range" => "all",
+ "metrics" => ["visitors"],
+ "filters" => [
+ [
+ "or",
+ [
+ [
+ "and",
+ [
+ ["contains", "event:page", ["/en"]],
+ ["not", ["contains", "event:page", ["/eng"]]]
+ ]
+ ],
+ ["contains", "event:page", ["/gb"]]
+ ]
+ ]
+ ]
+ })
+
+ assert json_response(conn, 200)["results"] == [%{"metrics" => [3], "dimensions" => []}]
+ end
+
test "contains and member filter combined", %{conn: conn, site: site} do
populate_stats(site, [
build(:pageview, pathname: "/en/page1"),