Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.
TypeScript analyticsanalytics-dashboardclickhouseelixirgoogle-analyticsmarketingopen-source-analyticsphoenixplausible-analyticspostgresqlprivacyprivacy-friendlyself-hostedsimple-analyticsstatisticstailwindcssweb-analyticswebsite-analyticswebsite-statswebsite-tracking
APIv2: Behavioral filtering (#4980)
Karl-Aksel Puulmann9a4969e1054d
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex a4838739b3..6b5620f617 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -14,7 +14,7 @@ All notable changes to this project will be documented in this file.### Changed-- Filters appear in the search bar as ?f=is,page,/docs,/blog&f=... instead of ?filters=((is,page,(/docs,/blog)),...) for Plausible links sent on various platforms to work reliably.+- Filters appear in the search bar as ?f=is,page,/docs,/blog&f=... instead of ?filters=((is,page,(/docs,/blog)),...) for Plausible links sent on various platforms to work reliably.- Details modal search inputs are now case-insensitive.- Improved report performance in cases where site has a lot of unique pathnamesdiff --git a/assets/js/types/query-api.d.ts b/assets/js/types/query-api.d.tsindex 51b52f5050..a52c60da4a 100644--- a/assets/js/types/query-api.d.ts+++ b/assets/js/types/query-api.d.ts@@ -63,7 +63,7 @@ export type SimpleFilterDimensions =export type CustomPropertyFilterDimensions = string;export type GoalDimension = "event:goal";export type TimeDimensions = "time" | "time:month" | "time:week" | "time:day" | "time:hour";-export type FilterTree = FilterEntry | FilterAndOr | FilterNot;+export type FilterTree = FilterEntry | FilterAndOr | FilterNot | FilterHasDone;export type FilterEntry = FilterWithoutGoals | FilterWithGoals | FilterWithPattern | FilterForSegment;/*** @minItems 3@@ -130,6 +130,11 @@ export type FilterAndOr = ["and" | "or", [FilterTree, ...FilterTree[]]];* @maxItems 2*/export type FilterNot = ["not", FilterTree];+/**+ * @minItems 2+ * @maxItems 2+ */+export type FilterHasDone = ["has_done" | "has_not_done", FilterTree];/*** @minItems 2* @maxItems 2diff --git a/extra/lib/plausible/stats/goal/revenue.ex b/extra/lib/plausible/stats/goal/revenue.exindex dba19909ca..1be42759d6 100644--- a/extra/lib/plausible/stats/goal/revenue.ex+++ b/extra/lib/plausible/stats/goal/revenue.ex@@ -24,11 +24,11 @@ defmodule Plausible.Stats.Goal.Revenue doThe resulting data structure is attached to a `Query` and used below in `format_revenue_metric/3`."""- def preload(site, goals, metrics, dimensions) do+ def preload(site, preloaded_goals, metrics, dimensions) docond donot requested?(metrics) -> {nil, %{}}not available?(site) -> {:revenue_goals_unavailable, %{}}- true -> preload(goals, dimensions)+ true -> preload(preloaded_goals.matching_toplevel_filters, dimensions)endenddiff --git a/lib/plausible/goals/filters.ex b/lib/plausible/goals/filters.exindex b9d7d5a749..0caf5e4e51 100644--- a/lib/plausible/goals/filters.ex+++ b/lib/plausible/goals/filters.ex@@ -4,6 +4,31 @@ defmodule Plausible.Goals.Filters doimport Ecto.Queryimport Plausible.Stats.Filters.Utils, only: [page_regex: 1]+ alias Plausible.Stats.Filters++ @doc """+ Preloads goals data if needed for query-building and related work.+ """+ def preload_needed_goals(site, dimensions, filters) do+ if Enum.member?(dimensions, "event:goal") or+ Filters.filtering_on_dimension?(filters, "event:goal") do+ goals = Plausible.Goals.for_site(site)++ %{+ # When grouping by event:goal, later pipeline needs to know which goals match filters exactly.+ # This can affect both calculations whether all goals have the same revenue currency and+ # whether we should skip imports.+ matching_toplevel_filters: goals_matching_toplevel_filters(goals, filters),+ all: goals+ }+ else+ %{+ all: [],+ matching_toplevel_filters: []+ }+ end+ end+@doc """Translates an event:goal filter into SQL. Similarly to other `add_filter` clauses in`Plausible.Stats.SQL.WhereBuilder`, returns an `Ecto.Query.dynamic` expression.@@ -26,7 +51,7 @@ defmodule Plausible.Goals.Filters doEnum.reduce(clauses, false, fn clause, dynamic_statement ->condition =- query.preloaded_goals+ query.preloaded_goals.all|> filter_preloaded(filter, clause)|> build_condition(imported?)@@ -34,9 +59,11 @@ defmodule Plausible.Goals.Filters doend)end- def preload_needed_goals(site, filters) do- goals = Plausible.Goals.for_site(site)+ defp filter_preloaded(goals, filter, clause) do+ Enum.filter(goals, fn goal -> matches?(goal, filter, clause) end)+ end+ defp goals_matching_toplevel_filters(goals, filters) doEnum.reduce(filters, goals, fn[_, "event:goal" | _] = filter, goals ->goals_matching_any_clause(goals, filter)@@ -46,10 +73,6 @@ defmodule Plausible.Goals.Filters doend)end- defp filter_preloaded(preloaded_goals, filter, clause) do- Enum.filter(preloaded_goals, fn goal -> matches?(goal, filter, clause) end)- end-defp goals_matching_any_clause(goals, [_, _, clauses | _] = filter) dogoals|> Enum.filter(fn goal ->diff --git a/lib/plausible/segments/filters.ex b/lib/plausible/segments/filters.exindex a18cf33e78..132ce2bd55 100644--- a/lib/plausible/segments/filters.ex+++ b/lib/plausible/segments/filters.ex@@ -22,7 +22,7 @@ defmodule Plausible.Segments.Filters dofilters|> Filters.traverse()|> Enum.flat_map(fn- {[_operation, "segment", clauses], _depth} -> clauses+ {[_operation, "segment", clauses], _} -> clauses_ -> []end)diff --git a/lib/plausible/stats/base.ex b/lib/plausible/stats/base.exindex c5304eaacd..e9c2a48b63 100644--- a/lib/plausible/stats/base.ex+++ b/lib/plausible/stats/base.ex@@ -26,8 +26,8 @@ defmodule Plausible.Stats.Base doendend- defp query_events(site, query) do- q = from(e in "events_v2", where: ^SQL.WhereBuilder.build(:events, site, query))+ defp query_events(_site, query) do+ q = from(e in "events_v2", where: ^SQL.WhereBuilder.build(:events, query))on_ee doq = Plausible.Stats.Sampling.add_query_hint(q, query)@@ -36,8 +36,8 @@ defmodule Plausible.Stats.Base doqend- def query_sessions(site, query) do- q = from(s in "sessions_v2", where: ^SQL.WhereBuilder.build(:sessions, site, query))+ def query_sessions(_site, query) do+ q = from(s in "sessions_v2", where: ^SQL.WhereBuilder.build(:sessions, query))on_ee doq = Plausible.Stats.Sampling.add_query_hint(q, query)diff --git a/lib/plausible/stats/clickhouse.ex b/lib/plausible/stats/clickhouse.exindex a201d0c94a..232d7d0e7f 100644--- a/lib/plausible/stats/clickhouse.ex+++ b/lib/plausible/stats/clickhouse.ex@@ -78,7 +78,7 @@ defmodule Plausible.Stats.Clickhouse dodef top_sources_for_spike(site, query, limit, page) dooffset = (page - 1) * limit- {first_datetime, last_datetime} = Plausible.Stats.Time.utc_boundaries(query, site)+ {first_datetime, last_datetime} = Plausible.Stats.Time.utc_boundaries(query)referrers =from(s in "sessions_v2",diff --git a/lib/plausible/stats/filters/filters.ex b/lib/plausible/stats/filters/filters.exindex 4c828fef1e..b693528b76 100644--- a/lib/plausible/stats/filters/filters.ex+++ b/lib/plausible/stats/filters/filters.ex@@ -3,6 +3,7 @@ defmodule Plausible.Stats.Filters doA module for parsing filters used in stat queries."""+ alias Plausible.Stats.Queryalias Plausible.Stats.Filters.QueryParseralias Plausible.Stats.Filters.StatsAPIFilterParser@@ -89,15 +90,45 @@ defmodule Plausible.Stats.Filters dodef dimensions_used_in_filters(filters, opts \\ []) domin_depth = Keyword.get(opts, :min_depth, 0)+ max_depth = Keyword.get(opts, :max_depth, 999)+ # :ignore or :only+ behavioral_filter_option = Keyword.get(opts, :behavioral_filters, nil)filters- |> traverse()- |> Enum.filter(fn {_filter, depth} -> depth >= min_depth end)+ |> traverse(+ {0, false},+ fn {depth, is_behavioral_filter}, operator ->+ {depth + 1, is_behavioral_filter or operator in [:has_done, :has_not_done]}+ end+ )+ |> Enum.filter(fn {_filter, {depth, is_behavioral_filter}} ->+ matches_behavioral_filter_option? =+ case behavioral_filter_option do+ :ignore -> not is_behavioral_filter+ :only -> is_behavioral_filter+ _ -> true+ end++ depth >= min_depth and depth <= max_depth and matches_behavioral_filter_option?+ end)|> Enum.map(fn {[_operator, dimension | _rest], _depth} -> dimension end)end- def filtering_on_dimension?(query, dimension) do- dimension in dimensions_used_in_filters(query.filters)+ def filtering_on_dimension?(query, dimension, opts \\ []) do+ filters =+ case query do+ %Query{filters: filters} -> filters+ %{filters: filters} -> filters+ filters when is_list(filters) -> filters+ end++ dimension in dimensions_used_in_filters(filters, opts)+ end++ def all_leaf_filters(filters) do+ filters+ |> traverse(nil, fn _, _ -> nil end)+ |> Enum.map(fn {filter, _} -> filter end)end@doc """@@ -144,12 +175,13 @@ defmodule Plausible.Stats.Filters dodefp transform_tree(filter, transformer) docase {transformer.(filter), filter} do# Transformer did not return that value - transform that subtree- {nil, [operation, child_filter]} when operation in [:not, :ignore_in_totals_query] ->+ {nil, [operator, child_filter]}+ when operator in [:not, :ignore_in_totals_query, :has_done, :has_not_done] ->[transformed_child] = transform_tree(child_filter, transformer)- [[operation, transformed_child]]+ [[operator, transformed_child]]- {nil, [operation, filters]} when operation in [:and, :or] ->- [[operation, transform_filters(filters, transformer)]]+ {nil, [operator, filters]} when operator in [:and, :or] ->+ [[operator, transform_filters(filters, transformer)]]# Reached a leaf node, return existing value{nil, filter} ->@@ -161,22 +193,26 @@ defmodule Plausible.Stats.Filters doendend- def traverse(filters, depth \\ -1) do+ @doc """+ Traverses a filter tree while accumulating state.+ """+ def traverse(filters, state \\ nil, state_transformer \\ fn state, _ -> state end) dofilters- |> Enum.flat_map(&traverse_tree(&1, depth + 1))+ |> Enum.flat_map(&traverse_tree(&1, state, state_transformer))end- defp traverse_tree(filter, depth) do+ defp traverse_tree(filter, state, state_transformer) docase filter do- [operation, child_filter] when operation in [:not, :ignore_in_totals_query] ->- traverse_tree(child_filter, depth + 1)+ [operation, child_filter]+ when operation in [:not, :ignore_in_totals_query, :has_done, :has_not_done] ->+ traverse_tree(child_filter, state_transformer.(state, operation), state_transformer)[operation, filters] when operation in [:and, :or] ->- traverse(filters, depth + 1)+ traverse(filters, state_transformer.(state, operation), state_transformer)# Leaf node_ ->- [{filter, depth}]+ [{filter, state}]endendenddiff --git a/lib/plausible/stats/filters/query_parser.ex b/lib/plausible/stats/filters/query_parser.exindex 0dd6a976c8..848478e7f5 100644--- a/lib/plausible/stats/filters/query_parser.ex+++ b/lib/plausible/stats/filters/query_parser.ex@@ -61,6 +61,7 @@ defmodule Plausible.Stats.Filters.QueryParser do:ok <- validate_custom_props_access(site, query),:ok <- validate_toplevel_only_filter_dimension(query),:ok <- validate_special_metrics_filters(query),+ :ok <- validate_behavioral_filters(query),:ok <- validate_filtered_goals_exist(query),:ok <- validate_revenue_metrics_access(site, query),:ok <- validate_metrics(query),@@ -110,16 +111,20 @@ defmodule Plausible.Stats.Filters.QueryParser dodefp parse_operator(["matches_wildcard_not" | _rest]), do: {:ok, :matches_wildcard_not}defp parse_operator(["contains" | _rest]), do: {:ok, :contains}defp parse_operator(["contains_not" | _rest]), do: {:ok, :contains_not}- defp parse_operator(["not" | _rest]), do: {:ok, :not}defp parse_operator(["and" | _rest]), do: {:ok, :and}defp parse_operator(["or" | _rest]), do: {:ok, :or}+ defp parse_operator(["not" | _rest]), do: {:ok, :not}+ defp parse_operator(["has_done" | _rest]), do: {:ok, :has_done}+ defp parse_operator(["has_not_done" | _rest]), do: {:ok, :has_not_done}defp parse_operator(filter), do: {:error, "Unknown operator for filter '#{i(filter)}'."}- def parse_filter_second(:not, [_, filter | _rest]), do: parse_filter(filter)-def parse_filter_second(operator, [_, filters | _rest]) when operator in [:and, :or],do: parse_filters(filters)+ def parse_filter_second(operator, [_, filter | _rest])+ when operator in [:not, :has_done, :has_not_done],+ do: parse_filter(filter)+def parse_filter_second(_operator, filter), do: parse_filter_key(filter)defp parse_filter_key([_operator, filter_key | _rest] = filter) do@@ -146,7 +151,7 @@ defmodule Plausible.Stats.Filters.QueryParser doenddefp parse_filter_rest(operator, _filter)- when operator in [:not, :and, :or],+ when operator in [:not, :and, :or, :has_done, :has_not_done],do: {:ok, []}defp parse_clauses_list([operator, filter_key, list | _rest] = filter) when is_list(list) do@@ -434,20 +439,14 @@ defmodule Plausible.Stats.Filters.QueryParser doenddef preload_goals_and_revenue(site, metrics, filters, dimensions) do- goal_filters? =- Enum.any?(filters, fn [_, filter_key | _rest] -> filter_key == "event:goal" end)+ preloaded_goals =+ Plausible.Goals.Filters.preload_needed_goals(site, dimensions, filters)- goals =- if goal_filters? or Enum.member?(dimensions, "event:goal") do- Plausible.Goals.Filters.preload_needed_goals(site, filters)- else- []- end-- {revenue_warning, revenue_currencies} = preload_revenue(site, goals, metrics, dimensions)+ {revenue_warning, revenue_currencies} =+ preload_revenue(site, preloaded_goals, metrics, dimensions){- goals,+ preloaded_goals,revenue_warning,revenue_currencies}@@ -455,9 +454,12 @@ defmodule Plausible.Stats.Filters.QueryParser do@only_toplevel ["event:goal", "event:hostname"]defp validate_toplevel_only_filter_dimension(query) do- not_toplevel = Filters.dimensions_used_in_filters(query.filters, min_depth: 1)+ not_toplevel =+ query.filters+ |> Filters.dimensions_used_in_filters(min_depth: 1, behavioral_filters: :ignore)+ |> Enum.filter(&(&1 in @only_toplevel))- if Enum.any?(not_toplevel, &(&1 in @only_toplevel)) do+ if Enum.count(not_toplevel) > 0 do{:error,"Invalid filters. Dimension `#{List.first(not_toplevel)}` can only be filtered at the top level."}else@@ -482,16 +484,53 @@ defmodule Plausible.Stats.Filters.QueryParser doendend+ defp validate_behavioral_filters(query) do+ query.filters+ |> Filters.traverse(0, fn behavioral_depth, operator ->+ if operator in [:has_done, :has_not_done] do+ behavioral_depth + 1+ else+ behavioral_depth+ end+ end)+ |> Enum.reduce_while(:ok, fn {[_operator, dimension | _rest], behavioral_depth}, :ok ->+ cond do+ behavioral_depth == 0 ->+ # ignore non-behavioral filters+ {:cont, :ok}++ behavioral_depth > 1 ->+ {:halt,+ {:error,+ "Invalid filters. Behavioral filters (has_done, has_not_done) cannot be nested."}}++ not String.starts_with?(dimension, "event:") ->+ {:halt,+ {:error,+ "Invalid filters. Behavioral filters (has_done, has_not_done) can only be used with event dimension filters."}}++ true ->+ {:cont, :ok}+ end+ end)+ end+defp validate_filtered_goals_exist(query) do- # Note: Only works since event:goal is allowed as a top level filter+ # Note: We don't check :contains goal filters since it's acceptable if they match nothing.goal_filter_clauses =- Enum.flat_map(query.filters, fn+ query.filters+ |> Filters.all_leaf_filters()+ |> Enum.flat_map(fn[:is, "event:goal", clauses] -> clauses_ -> []end)if length(goal_filter_clauses) > 0 do- validate_list(goal_filter_clauses, &validate_goal_filter(&1, query.preloaded_goals))+ configured_goal_names =+ query.preloaded_goals.all+ |> Enum.map(&Plausible.Goal.display_name/1)++ validate_list(goal_filter_clauses, &validate_goal_filter(&1, configured_goal_names))else:okend@@ -517,10 +556,7 @@ defmodule Plausible.Stats.Filters.QueryParser dodefp validate_revenue_metrics_access(_site, _query), do: :okend- defp validate_goal_filter(clause, configured_goals) do- configured_goal_names =- Enum.map(configured_goals, fn goal -> Plausible.Goal.display_name(goal) end)-+ defp validate_goal_filter(clause, configured_goal_names) doif Enum.member?(configured_goal_names, clause) do:okelse@@ -562,7 +598,7 @@ defmodule Plausible.Stats.Filters.QueryParser dodefp validate_metric(metric, query) when metric in [:conversion_rate, :group_conversion_rate] doif Enum.member?(query.dimensions, "event:goal") or- Filters.filtering_on_dimension?(query, "event:goal") do+ Filters.filtering_on_dimension?(query, "event:goal", behavioral_filters: :ignore) do:okelse{:error, "Metric `#{metric}` can only be queried with event:goal filters or dimensions."}@@ -582,7 +618,7 @@ defmodule Plausible.Stats.Filters.QueryParser dodefp validate_metric(:views_per_visit = metric, query) docond do- Filters.filtering_on_dimension?(query, "event:page") ->+ Filters.filtering_on_dimension?(query, "event:page", behavioral_filters: :ignore) ->{:error, "Metric `#{metric}` cannot be queried with a filter on `event:page`."}length(query.dimensions) > 0 ->diff --git a/lib/plausible/stats/imported/base.ex b/lib/plausible/stats/imported/base.exindex 2328a4497f..a27d6cb6f3 100644--- a/lib/plausible/stats/imported/base.ex+++ b/lib/plausible/stats/imported/base.ex@@ -8,7 +8,8 @@ defmodule Plausible.Stats.Imported.Base doalias Plausible.Importedalias Plausible.Stats.Query- import Plausible.Stats.Filters, only: [dimensions_used_in_filters: 1]+ import Plausible.Stats.Filters,+ only: [dimensions_used_in_filters: 1, dimensions_used_in_filters: 2]@property_to_table_mappings %{"visit:source" => "imported_sources",@@ -74,10 +75,18 @@ defmodule Plausible.Stats.Imported.Base doenddef decide_tables(query) do- if custom_prop_query?(query) do- do_decide_custom_prop_table(query)- else- do_decide_tables(query)+ behavioral_filters = dimensions_used_in_filters(query.filters, behavioral_filters: :only)++ cond do+ # Behavioral filters cannot be emulated via aggregated imported stats+ length(behavioral_filters) > 0 ->+ []++ custom_prop_query?(query) ->+ do_decide_custom_prop_table(query)++ true ->+ do_decide_tables(query)endend@@ -124,9 +133,9 @@ defmodule Plausible.Stats.Imported.Base do|> Enum.any?(&(&1 in special_goals_for(property)))has_unsupported_filters? =- Enum.any?(query.filters, fn [_, filter_key | _] ->- filter_key not in [property, "event:name", "event:goal"]- end)+ query.filters+ |> dimensions_used_in_filters()+ |> Enum.any?(&(&1 not in [property, "event:name", "event:goal"]))if has_required_name_filter? and not has_unsupported_filters? do["imported_custom_events"]@@ -144,7 +153,7 @@ defmodule Plausible.Stats.Imported.Base dodefp do_decide_tables(%Query{dimensions: ["event:goal"]} = query) dofilter_dimensions = dimensions_used_in_filters(query.filters)- filter_goals = query.preloaded_goals+ filter_goals = query.preloaded_goals.matching_toplevel_filtersany_event_goals? = Enum.any?(filter_goals, fn goal -> Plausible.Goal.type(goal) == :event end)@@ -177,7 +186,7 @@ defmodule Plausible.Stats.Imported.Base do|> Enum.map(&@property_to_table_mappings[&1])filter_goal_table_candidates =- query.preloaded_goals+ query.preloaded_goals.matching_toplevel_filters|> Enum.map(&Plausible.Goal.type/1)|> Enum.map(fn:event -> "imported_custom_events"diff --git a/lib/plausible/stats/imported/imported.ex b/lib/plausible/stats/imported/imported.exindex 9f7b038595..61d99373de 100644--- a/lib/plausible/stats/imported/imported.ex+++ b/lib/plausible/stats/imported/imported.ex@@ -239,7 +239,8 @@ defmodule Plausible.Stats.Imported doenddef merge_imported(q, site, %Query{dimensions: ["event:goal"]} = query, metrics) do- {events, page_regexes} = Filters.Utils.split_goals_query_expressions(query.preloaded_goals)+ {events, page_regexes} =+ Filters.Utils.split_goals_query_expressions(query.preloaded_goals.matching_toplevel_filters)Imported.Base.decide_tables(query)|> Enum.map(fndiff --git a/lib/plausible/stats/legacy/legacy_query_builder.ex b/lib/plausible/stats/legacy/legacy_query_builder.exindex 5fa518bbcf..fe81589b34 100644--- a/lib/plausible/stats/legacy/legacy_query_builder.ex+++ b/lib/plausible/stats/legacy/legacy_query_builder.ex@@ -13,7 +13,12 @@ defmodule Plausible.Stats.Legacy.QueryBuilder doquery =Query- |> struct!(now: now, debug_metadata: debug_metadata)+ |> struct!(+ now: now,+ debug_metadata: debug_metadata,+ site_id: site.id,+ site_native_stats_start_at: site.native_stats_start_at+ )|> put_period(site, params)|> put_timezone(site)|> put_dimensions(params)diff --git a/lib/plausible/stats/query.ex b/lib/plausible/stats/query.exindex 8ee940eec6..76e6ad702a 100644--- a/lib/plausible/stats/query.ex+++ b/lib/plausible/stats/query.ex@@ -22,7 +22,9 @@ defmodule Plausible.Stats.Query do# Revenue metric specific metadatarevenue_currencies: %{},revenue_warning: nil,- remove_unavailable_revenue_metrics: false+ remove_unavailable_revenue_metrics: false,+ site_id: nil,+ site_native_stats_start_at: nilrequire OpenTelemetry.Tracer, as: Traceralias Plausible.Stats.{DateTimeRange, Filters, Imported, Legacy}@@ -34,7 +36,12 @@ defmodule Plausible.Stats.Query doquery =struct!(__MODULE__, Map.to_list(query_data))|> put_imported_opts(site, %{})- |> struct!(now: DateTime.utc_now(:second), debug_metadata: debug_metadata)+ |> struct!(+ now: DateTime.utc_now(:second),+ debug_metadata: debug_metadata,+ site_id: site.id,+ site_native_stats_start_at: site.native_stats_start_at+ )on_ee doquery = Plausible.Stats.Sampling.put_threshold(query, site, params)diff --git a/lib/plausible/stats/query_runner.ex b/lib/plausible/stats/query_runner.exindex fadc683466..6af06f2154 100644--- a/lib/plausible/stats/query_runner.ex+++ b/lib/plausible/stats/query_runner.ex@@ -154,7 +154,7 @@ defmodule Plausible.Stats.QueryRunner doenddefp dimension_label("event:goal", entry, query) do- {events, paths} = Filters.Utils.split_goals(query.preloaded_goals)+ {events, paths} = Filters.Utils.split_goals(query.preloaded_goals.matching_toplevel_filters)goal_index = Map.get(entry, Util.shortname(query, "event:goal"))diff --git a/lib/plausible/stats/sql/query_builder.ex b/lib/plausible/stats/sql/query_builder.exindex ce340d9749..f0b1e2b56f 100644--- a/lib/plausible/stats/sql/query_builder.ex+++ b/lib/plausible/stats/sql/query_builder.ex@@ -37,7 +37,7 @@ defmodule Plausible.Stats.SQL.QueryBuilder doq =from(e in "events_v2",- where: ^SQL.WhereBuilder.build(:events, site, events_query),+ where: ^SQL.WhereBuilder.build(:events, events_query),select: ^select_event_metrics(events_query))@@ -46,18 +46,18 @@ defmodule Plausible.Stats.SQL.QueryBuilder doendq- |> join_sessions_if_needed(site, events_query)+ |> join_sessions_if_needed(events_query)|> build_group_by(:events, events_query)|> merge_imported(site, events_query, events_query.metrics)|> SQL.SpecialMetrics.add(site, events_query)end- defp join_sessions_if_needed(q, site, query) do+ defp join_sessions_if_needed(q, query) doif TableDecider.events_join_sessions?(query) dosessions_q =from(s in "sessions_v2",- where: ^SQL.WhereBuilder.build(:sessions, site, query),+ where: ^SQL.WhereBuilder.build(:sessions, query),where: s.sign == 1,select: %{session_id: s.session_id},group_by: s.session_id@@ -83,7 +83,7 @@ defmodule Plausible.Stats.SQL.QueryBuilder doq =from(e in "sessions_v2",- where: ^SQL.WhereBuilder.build(:sessions, site, sessions_query),+ where: ^SQL.WhereBuilder.build(:sessions, sessions_query),select: ^select_session_metrics(sessions_query))@@ -92,17 +92,17 @@ defmodule Plausible.Stats.SQL.QueryBuilder doendq- |> join_events_if_needed(site, sessions_query)+ |> join_events_if_needed(sessions_query)|> build_group_by(:sessions, sessions_query)|> merge_imported(site, sessions_query, sessions_query.metrics)|> SQL.SpecialMetrics.add(site, sessions_query)end- def join_events_if_needed(q, site, query) do+ def join_events_if_needed(q, query) doif TableDecider.sessions_join_events?(query) doevents_q =from(e in "events_v2",- where: ^SQL.WhereBuilder.build(:events, site, query),+ where: ^SQL.WhereBuilder.build(:events, query),select: %{session_id: fragment("DISTINCT ?", e.session_id),_sample_factor: fragment("_sample_factor")@@ -139,7 +139,8 @@ defmodule Plausible.Stats.SQL.QueryBuilder doenddefp dimension_group_by(q, _table, query, "event:goal" = dimension) do- {events, page_regexes} = Filters.Utils.split_goals_query_expressions(query.preloaded_goals)+ {events, page_regexes} =+ Filters.Utils.split_goals_query_expressions(query.preloaded_goals.matching_toplevel_filters)from(e in q,join: goal in Expression.event_goal_join(events, page_regexes),diff --git a/lib/plausible/stats/sql/special_metrics.ex b/lib/plausible/stats/sql/special_metrics.exindex 15eef2bed9..fd5f10c753 100644--- a/lib/plausible/stats/sql/special_metrics.ex+++ b/lib/plausible/stats/sql/special_metrics.ex@@ -58,7 +58,7 @@ defmodule Plausible.Stats.SQL.SpecialMetrics do|> Query.set(dimensions: [],include_imported: query.include_imported,- preloaded_goals: [],+ preloaded_goals: Map.put(query.preloaded_goals, :matching_toplevel_filters, []),pagination: nil)@@ -102,7 +102,7 @@ defmodule Plausible.Stats.SQL.SpecialMetrics dometrics: [:visitors],order_by: [],include_imported: query.include_imported,- preloaded_goals: [],+ preloaded_goals: Map.put(query.preloaded_goals, :matching_toplevel_filters, []),pagination: nil)diff --git a/lib/plausible/stats/sql/where_builder.ex b/lib/plausible/stats/sql/where_builder.exindex 443983521a..e19e691bd4 100644--- a/lib/plausible/stats/sql/where_builder.ex+++ b/lib/plausible/stats/sql/where_builder.ex@@ -4,7 +4,7 @@ defmodule Plausible.Stats.SQL.WhereBuilder do"""import Ecto.Query- import Plausible.Stats.Time, only: [utc_boundaries: 2]+ import Plausible.Stats.Time, only: [utc_boundaries: 1]import Plausible.Stats.Filters.Utils, only: [page_regex: 1]use Plausible.Stats.SQL.Fragments@@ -19,8 +19,8 @@ defmodule Plausible.Stats.SQL.WhereBuilder do]@doc "Builds WHERE clause for a given Query against sessions or events table"- def build(table, site, query) do- base_condition = filter_site_time_range(table, site, query)+ def build(table, query) do+ base_condition = filter_site_time_range(table, query)query.filters|> Enum.map(&add_filter(table, query, &1))@@ -41,17 +41,18 @@ defmodule Plausible.Stats.SQL.WhereBuilder doendend- defp filter_site_time_range(:events, site, query) do- {first_datetime, last_datetime} = utc_boundaries(query, site)+ defp filter_site_time_range(:events, query) do+ {first_datetime, last_datetime} = utc_boundaries(query)dynamic([e],- e.site_id == ^site.id and e.timestamp >= ^first_datetime and e.timestamp <= ^last_datetime+ e.site_id == ^query.site_id and e.timestamp >= ^first_datetime and+ e.timestamp <= ^last_datetime)end- defp filter_site_time_range(:sessions, site, query) do- {first_datetime, last_datetime} = utc_boundaries(query, site)+ defp filter_site_time_range(:sessions, query) do+ {first_datetime, last_datetime} = utc_boundaries(query)# Counts each _active_ session in time range even if they started beforedynamic(@@ -66,7 +67,7 @@ defmodule Plausible.Stats.SQL.WhereBuilder do# Without it, the sample factor would be greatly overestimated for large sites,# as query would be estimated to return _all_ rows matching other conditions# before `start == last_datetime`.- s.site_id == ^site.id and+ s.site_id == ^query.site_id ands.start >= ^NaiveDateTime.add(first_datetime, -7, :day) ands.timestamp >= ^first_datetime ands.start <= ^last_datetime@@ -93,6 +94,20 @@ defmodule Plausible.Stats.SQL.WhereBuilder do|> Enum.reduce(fn condition, acc -> dynamic([], ^acc or ^condition) end)end+ defp add_filter(_table, query, [:has_done, filter]) do+ condition =+ dynamic([], ^filter_site_time_range(:events, query) and ^add_filter(:events, query, filter))++ dynamic(+ [t],+ t.session_id in subquery(from(e in "events_v2", where: ^condition, select: e.session_id))+ )+ end++ defp add_filter(table, query, [:has_not_done, filter]) do+ dynamic([], not (^add_filter(table, query, [:has_done, filter])))+ end+defp add_filter(:events, _query, [:is, "event:name" | _rest] = filter) doin_clause(col_value(:name), filter)enddiff --git a/lib/plausible/stats/time.ex b/lib/plausible/stats/time.exindex cee037e4b0..1872928c98 100644--- a/lib/plausible/stats/time.ex+++ b/lib/plausible/stats/time.ex@@ -5,11 +5,14 @@ defmodule Plausible.Stats.Time doalias Plausible.Stats.{Query, DateTimeRange}- def utc_boundaries(%Query{utc_time_range: time_range}, site) do+ def utc_boundaries(%Query{+ utc_time_range: time_range,+ site_native_stats_start_at: native_stats_start_at+ }) dofirst =time_range.first|> DateTime.to_naive()- |> beginning_of_time(site.native_stats_start_at)+ |> beginning_of_time(native_stats_start_at)last = DateTime.to_naive(time_range.last)diff --git a/lib/plausible_web/controllers/api/stats_controller.ex b/lib/plausible_web/controllers/api/stats_controller.exindex 32430202e8..66448d79a8 100644--- a/lib/plausible_web/controllers/api/stats_controller.ex+++ b/lib/plausible_web/controllers/api/stats_controller.ex@@ -294,7 +294,7 @@ defmodule PlausibleWeb.Api.StatsController doenddefp fetch_top_stats(site, query, current_user) do- goal_filter? = Filters.filtering_on_dimension?(query, "event:goal")+ goal_filter? = toplevel_goal_filter?(query)cond doquery.period == "30m" && goal_filter? ->@@ -392,7 +392,9 @@ defmodule PlausibleWeb.Api.StatsController doenddefp fetch_other_top_stats(site, query, current_user) do- page_filter? = Filters.filtering_on_dimension?(query, "event:page")+ page_filter? =+ Filters.filtering_on_dimension?(query, "event:page", behavioral_filters: :ignore)+scroll_depth_enabled? = scroll_depth_enabled?(site, current_user)metrics = [:visitors, :visits, :pageviews, :sample_percent]@@ -477,7 +479,7 @@ defmodule PlausibleWeb.Api.StatsController do|> transform_keys(%{source: :name})if params["csv"] do- if Filters.filtering_on_dimension?(query, "event:goal") do+ if toplevel_goal_filter?(query) dores|> transform_keys(%{visitors: :conversions})|> to_csv([:name, :conversions, :conversion_rate])@@ -509,7 +511,7 @@ defmodule PlausibleWeb.Api.StatsController do|> transform_keys(%{channel: :name})if params["csv"] do- if Filters.filtering_on_dimension?(query, "event:goal") do+ if toplevel_goal_filter?(query) dores|> transform_keys(%{visitors: :conversions})|> to_csv([:name, :conversions, :conversion_rate])@@ -564,7 +566,7 @@ defmodule PlausibleWeb.Api.StatsController dodefp validate_funnel_query(query) docond do- Filters.filtering_on_dimension?(query, "event:goal") ->+ toplevel_goal_filter?(query) ->{:error, {:invalid_funnel_query, "goals"}}Filters.filtering_on_dimension?(query, "event:page") ->@@ -591,7 +593,7 @@ defmodule PlausibleWeb.Api.StatsController do|> transform_keys(%{utm_medium: :name})if params["csv"] do- if Filters.filtering_on_dimension?(query, "event:goal") do+ if toplevel_goal_filter?(query) dores|> transform_keys(%{visitors: :conversions})|> to_csv([:name, :conversions, :conversion_rate])@@ -619,7 +621,7 @@ defmodule PlausibleWeb.Api.StatsController do|> transform_keys(%{utm_campaign: :name})if params["csv"] do- if Filters.filtering_on_dimension?(query, "event:goal") do+ if toplevel_goal_filter?(query) dores|> transform_keys(%{visitors: :conversions})|> to_csv([:name, :conversions, :conversion_rate])@@ -647,7 +649,7 @@ defmodule PlausibleWeb.Api.StatsController do|> transform_keys(%{utm_content: :name})if params["csv"] do- if Filters.filtering_on_dimension?(query, "event:goal") do+ if toplevel_goal_filter?(query) dores|> transform_keys(%{visitors: :conversions})|> to_csv([:name, :conversions, :conversion_rate])@@ -675,7 +677,7 @@ defmodule PlausibleWeb.Api.StatsController do|> transform_keys(%{utm_term: :name})if params["csv"] do- if Filters.filtering_on_dimension?(query, "event:goal") do+ if toplevel_goal_filter?(query) dores|> transform_keys(%{visitors: :conversions})|> to_csv([:name, :conversions, :conversion_rate])@@ -703,7 +705,7 @@ defmodule PlausibleWeb.Api.StatsController do|> transform_keys(%{utm_source: :name})if params["csv"] do- if Filters.filtering_on_dimension?(query, "event:goal") do+ if toplevel_goal_filter?(query) dores|> transform_keys(%{visitors: :conversions})|> to_csv([:name, :conversions, :conversion_rate])@@ -731,7 +733,7 @@ defmodule PlausibleWeb.Api.StatsController do|> transform_keys(%{referrer: :name})if params["csv"] do- if Filters.filtering_on_dimension?(query, "event:goal") do+ if toplevel_goal_filter?(query) dores|> transform_keys(%{visitors: :conversions})|> to_csv([:name, :conversions, :conversion_rate])@@ -857,7 +859,7 @@ defmodule PlausibleWeb.Api.StatsController do|> transform_keys(%{page: :name})if params["csv"] do- if Filters.filtering_on_dimension?(query, "event:goal") do+ if toplevel_goal_filter?(query) dopages|> transform_keys(%{visitors: :conversions})|> to_csv([:name, :conversions, :conversion_rate])@@ -890,7 +892,7 @@ defmodule PlausibleWeb.Api.StatsController do|> transform_keys(%{entry_page: :name})if params["csv"] do- if Filters.filtering_on_dimension?(query, "event:goal") do+ if toplevel_goal_filter?(query) doto_csv(entry_pages, [:name, :visitors, :conversion_rate], [:name,:conversions,@@ -926,7 +928,7 @@ defmodule PlausibleWeb.Api.StatsController do|> transform_keys(%{exit_page: :name})if params["csv"] do- if Filters.filtering_on_dimension?(query, "event:goal") do+ if toplevel_goal_filter?(query) doto_csv(exit_pages, [:name, :visitors, :conversion_rate], [:name,:conversions,@@ -999,7 +1001,7 @@ defmodule PlausibleWeb.Api.StatsController doMap.put(country, :name, country_info.name)end)- if Filters.filtering_on_dimension?(query, "event:goal") do+ if toplevel_goal_filter?(query) docountries|> transform_keys(%{visitors: :conversions})|> to_csv([:name, :conversions, :conversion_rate])@@ -1059,7 +1061,7 @@ defmodule PlausibleWeb.Api.StatsController doend)if params["csv"] do- if Filters.filtering_on_dimension?(query, "event:goal") do+ if toplevel_goal_filter?(query) doregions|> transform_keys(%{visitors: :conversions})|> to_csv([:name, :conversions, :conversion_rate])@@ -1103,7 +1105,7 @@ defmodule PlausibleWeb.Api.StatsController doend)if params["csv"] do- if Filters.filtering_on_dimension?(query, "event:goal") do+ if toplevel_goal_filter?(query) docities|> transform_keys(%{visitors: :conversions})|> to_csv([:name, :conversions, :conversion_rate])@@ -1135,7 +1137,7 @@ defmodule PlausibleWeb.Api.StatsController do|> transform_keys(%{browser: :name})if params["csv"] do- if Filters.filtering_on_dimension?(query, "event:goal") do+ if toplevel_goal_filter?(query) dobrowsers|> transform_keys(%{visitors: :conversions})|> to_csv([:name, :conversions, :conversion_rate])@@ -1167,7 +1169,7 @@ defmodule PlausibleWeb.Api.StatsController do|> transform_keys(%{browser_version: :version})if params["csv"] do- if Filters.filtering_on_dimension?(query, "event:goal") do+ if toplevel_goal_filter?(query) doresults|> transform_keys(%{browser: :name, visitors: :conversions})|> to_csv([:name, :version, :conversions, :conversion_rate])@@ -1208,7 +1210,7 @@ defmodule PlausibleWeb.Api.StatsController do|> transform_keys(%{os: :name})if params["csv"] do- if Filters.filtering_on_dimension?(query, "event:goal") do+ if toplevel_goal_filter?(query) dosystems|> transform_keys(%{visitors: :conversions})|> to_csv([:name, :conversions, :conversion_rate])@@ -1240,7 +1242,7 @@ defmodule PlausibleWeb.Api.StatsController do|> transform_keys(%{os_version: :version})if params["csv"] do- if Filters.filtering_on_dimension?(query, "event:goal") do+ if toplevel_goal_filter?(query) doresults|> transform_keys(%{os: :name, visitors: :conversions})|> to_csv([:name, :version, :conversions, :conversion_rate])@@ -1281,7 +1283,7 @@ defmodule PlausibleWeb.Api.StatsController do|> transform_keys(%{device: :name})if params["csv"] do- if Filters.filtering_on_dimension?(query, "event:goal") do+ if toplevel_goal_filter?(query) dosizes|> transform_keys(%{visitors: :conversions})|> to_csv([:name, :conversions, :conversion_rate])@@ -1371,7 +1373,7 @@ defmodule PlausibleWeb.Api.StatsController do|> Enum.concat()percent_or_cr =- if Filters.filtering_on_dimension?(query, "event:goal"),+ if toplevel_goal_filter?(query),do: :conversion_rate,else: :percentage@@ -1388,7 +1390,7 @@ defmodule PlausibleWeb.Api.StatsController doquery = Query.from(site, params, debug_metadata(conn))metrics =- if Filters.filtering_on_dimension?(query, "event:goal") do+ if toplevel_goal_filter?(query) do[:visitors, :events, :conversion_rate] ++ @revenue_metricselse[:visitors, :events, :percentage] ++ @revenue_metrics@@ -1559,10 +1561,12 @@ defmodule PlausibleWeb.Api.StatsController doendrequires_goal_filter? = metric in [:conversion_rate, :events]- has_goal_filter? = Filters.filtering_on_dimension?(query, "event:goal")+ has_goal_filter? = toplevel_goal_filter?(query)requires_page_filter? = metric == :scroll_depth- has_page_filter? = Filters.filtering_on_dimension?(query, "event:page")++ has_page_filter? =+ Filters.filtering_on_dimension?(query, "event:page", behavioral_filters: :ignore)cond dorequires_goal_filter? and not has_goal_filter? ->@@ -1600,7 +1604,7 @@ defmodule PlausibleWeb.Api.StatsController doenddefp breakdown_metrics(query, extra_metrics \\ []) do- if Filters.filtering_on_dimension?(query, "event:goal") do+ if toplevel_goal_filter?(query) do[:visitors, :conversion_rate, :total_visitors]else[:visitors] ++ extra_metrics@@ -1625,6 +1629,10 @@ defmodule PlausibleWeb.Api.StatsController dodefp realtime_period_to_30m(params), do: params+ defp toplevel_goal_filter?(query) do+ Filters.filtering_on_dimension?(query, "event:goal", max_depth: 0)+ end+def scroll_depth_enabled?(site, user) doFunWithFlags.enabled?(:scroll_depth, for: user) ||FunWithFlags.enabled?(:scroll_depth, for: site)diff --git a/lib/plausible_web/controllers/stats_controller.ex b/lib/plausible_web/controllers/stats_controller.exindex 686567336f..336571fbd1 100644--- a/lib/plausible_web/controllers/stats_controller.ex+++ b/lib/plausible_web/controllers/stats_controller.ex@@ -194,10 +194,10 @@ defmodule PlausibleWeb.StatsController doinclude_scroll_depth? =!query.include_imported &&PlausibleWeb.Api.StatsController.scroll_depth_enabled?(site, current_user) &&- Filters.filtering_on_dimension?(query, "event:page")+ Filters.filtering_on_dimension?(query, "event:page", behavioral_filters: :ignore){metrics, column_headers} =- if Filters.filtering_on_dimension?(query, "event:goal") do+ if Filters.filtering_on_dimension?(query, "event:goal", max_depth: 0) do{[:visitors, :events, :conversion_rate],[:date, :unique_conversions, :total_conversions, :conversion_rate]diff --git a/priv/json-schemas/query-api-schema.json b/priv/json-schemas/query-api-schema.jsonindex 6a325c8d8e..0c95244ce9 100644--- a/priv/json-schemas/query-api-schema.json+++ b/priv/json-schemas/query-api-schema.json@@ -472,7 +472,11 @@"oneOf": [{ "$ref": "#/definitions/filter_entry" },{ "$ref": "#/definitions/filter_and_or" },- { "$ref": "#/definitions/filter_not" }+ { "$ref": "#/definitions/filter_not" },+ {+ "$ref": "#/definitions/filter_has_done",+ "$comment": "only :internal"+ }]},"filter_not": {@@ -480,7 +484,13 @@"additionalItems": false,"minItems": 2,"maxItems": 2,- "items": [{ "const": "not" }, { "$ref": "#/definitions/filter_tree" }]+ "items": [+ {+ "type": "string",+ "enum": ["not"]+ },+ { "$ref": "#/definitions/filter_tree" }+ ]},"filter_and_or": {"type": "array",@@ -499,6 +509,19 @@}]},+ "filter_has_done": {+ "type": "array",+ "additionalItems": false,+ "minItems": 2,+ "maxItems": 2,+ "items": [+ {+ "type": "string",+ "enum": ["has_done", "has_not_done"]+ },+ { "$ref": "#/definitions/filter_tree" }+ ]+ },"order_by_entry": {"type": "array","additionalItems": false,diff --git a/test/plausible/stats/query_parser_test.exs b/test/plausible/stats/query_parser_test.exsindex 968d04fa0f..338e071e79 100644--- a/test/plausible/stats/query_parser_test.exs+++ b/test/plausible/stats/query_parser_test.exs@@ -84,16 +84,18 @@ defmodule Plausible.Stats.Filters.QueryParserTest doenddef check_goals(actual, opts) do- preloaded_goal_names =- actual[:preloaded_goals]- |> Enum.map(& &1.display_name)- |> Enum.sort()+ assert goal_names(actual[:preloaded_goals][:all]) ==+ Enum.sort(Keyword.get(opts, :preloaded_goals)[:all])++ assert goal_names(actual[:preloaded_goals][:matching_toplevel_filters]) ==+ Enum.sort(Keyword.get(opts, :preloaded_goals)[:matching_toplevel_filters])- assert preloaded_goal_names == Keyword.get(opts, :preloaded_goals)assert actual[:revenue_warning] == Keyword.get(opts, :revenue_warning)assert actual[:revenue_currencies] == Keyword.get(opts, :revenue_currencies)end+ defp goal_names(goals), do: Enum.map(goals, & &1.display_name) |> Enum.sort()+test "parsing empty map fails", %{site: site} do%{}|> check_error(site, "#: Required properties site_id, metrics, date_range were not present.")@@ -265,7 +267,9 @@ defmodule Plausible.Stats.Filters.QueryParserTest do["or", []],["not"],["is_not"],- ["is_not", "event:name"]+ ["is_not", "event:name"],+ ["has_done"],+ ["has_not_done"]] dotest "errors on too short filter #{inspect(too_short_filter)}", %{site: site@@ -542,32 +546,97 @@ defmodule Plausible.Stats.Filters.QueryParserTest do})end- test "invalid `not` clause", %{site: site} do+ test "valid has_done and has_not_done filters", %{site: site} do+ insert(:goal, %{site: site, event_name: "Signup"})++ %{+ "site_id" => site.domain,+ "metrics" => ["visitors"],+ "date_range" => "all",+ "filters" => [+ ["has_done", ["is", "event:name", ["Signup"]]],+ [+ "has_not_done",+ [+ "or",+ [+ ["is", "event:goal", ["Signup"]],+ ["is", "event:page", ["/signup"]]+ ]+ ]+ ]+ ]+ }+ |> check_success(+ site,+ %{+ metrics: [:visitors],+ utc_time_range: @date_range_day,+ filters: [+ [:has_done, [:is, "event:name", ["Signup"]]],+ [+ :has_not_done,+ [:or, [[:is, "event:goal", ["Signup"]], [:is, "event:page", ["/signup"]]]]+ ]+ ],+ dimensions: [],+ order_by: nil,+ timezone: site.timezone,+ include: %{imports: false, time_labels: false, total_rows: false, comparisons: nil},+ pagination: %{limit: 10_000, offset: 0}+ },+ :internal+ )+ end++ test "fails when using visit filters within has_done filters", %{site: site} do%{"site_id" => site.domain,"metrics" => ["visitors"],"date_range" => "all",- "filters" => [["not", []]]+ "filters" => [+ ["has_done", ["is", "visit:browser", ["Chrome"]]]+ ]}|> check_error(site,- "#/filters/0: Invalid filter [\"not\", []]"+ "Invalid filters. Behavioral filters (has_done, has_not_done) can only be used with event dimension filters.",+ :internal)end- test "invalid `or` clause", %{site: site} do+ test "fails when nesting behavioral filters", %{site: site} do%{"site_id" => site.domain,"metrics" => ["visitors"],"date_range" => "all",- "filters" => [["or", []]]+ "filters" => [+ ["has_done", ["has_not_done", ["is", "visit:browser", ["Chrome"]]]]+ ]}|> check_error(site,- "#/filters/0: Invalid filter [\"or\", []]"+ "Invalid filters. Behavioral filters (has_done, has_not_done) cannot be nested.",+ :internal)end+ for operator <- ["not", "or", "has_done", "has_not_done"] do+ test "invalid `#{operator}` clause", %{site: site} do+ %{+ "site_id" => site.domain,+ "metrics" => ["visitors"],+ "date_range" => "all",+ "filters" => [[unquote(operator), []]]+ }+ |> check_error(+ site,+ "#/filters/0: Invalid filter [\"#{unquote(operator)}\", []]",+ :internal+ )+ end+ end+test "event:hostname filter", %{site: site} do%{"site_id" => site.domain,@@ -689,7 +758,13 @@ defmodule Plausible.Stats.Filters.QueryParserTest doinclude: %{imports: false, time_labels: false, total_rows: false, comparisons: nil},pagination: %{limit: 10_000, offset: 0}})- |> check_goals(preloaded_goals: ["Purchase", "Signup"], revenue_currencies: %{})+ |> check_goals(+ preloaded_goals: %{+ all: ["Contact", "Purchase", "Signup"],+ matching_toplevel_filters: ["Purchase", "Signup"]+ },+ revenue_currencies: %{}+ )endtest "with case insensitive match", %{site: site} do@@ -709,7 +784,13 @@ defmodule Plausible.Stats.Filters.QueryParserTest doinclude: %{imports: false, time_labels: false, total_rows: false, comparisons: nil},pagination: %{limit: 10_000, offset: 0}})- |> check_goals(preloaded_goals: ["Purchase", "Signup"], revenue_currencies: %{})+ |> check_goals(+ preloaded_goals: %{+ all: ["Contact", "Purchase", "Signup"],+ matching_toplevel_filters: ["Purchase", "Signup"]+ },+ revenue_currencies: %{}+ )endtest "with contains match", %{site: site} do@@ -729,7 +810,13 @@ defmodule Plausible.Stats.Filters.QueryParserTest doinclude: %{imports: false, time_labels: false, total_rows: false, comparisons: nil},pagination: %{limit: 10_000, offset: 0}})- |> check_goals(preloaded_goals: ["Signup"], revenue_currencies: %{})+ |> check_goals(+ preloaded_goals: %{+ all: ["Contact", "Purchase", "Signup"],+ matching_toplevel_filters: ["Signup"]+ },+ revenue_currencies: %{}+ )endtest "with case insensitive contains match", %{site: site} do@@ -749,7 +836,13 @@ defmodule Plausible.Stats.Filters.QueryParserTest doinclude: %{imports: false, time_labels: false, total_rows: false, comparisons: nil},pagination: %{limit: 10_000, offset: 0}})- |> check_goals(preloaded_goals: ["Contact", "Signup"], revenue_currencies: %{})+ |> check_goals(+ preloaded_goals: %{+ all: ["Contact", "Purchase", "Signup"],+ matching_toplevel_filters: ["Contact", "Signup"]+ },+ revenue_currencies: %{}+ )endend@@ -983,7 +1076,7 @@ defmodule Plausible.Stats.Filters.QueryParserTest doinsert(:goal, %{site: site, event_name: "Signup"})insert(:goal, %{site: site, page_path: "/thank-you"})- params = %{+ %{"site_id" => site.domain,"metrics" => ["visitors"],"date_range" => "all",@@ -991,26 +1084,29 @@ defmodule Plausible.Stats.Filters.QueryParserTest do["is", "event:goal", ["Signup", "Visit /thank-you"]]]}-- assert {:ok, res} = parse(site, :public, params, @now)- expected_timezone = site.timezone-- assert %{- metrics: [:visitors],- utc_time_range: @date_range_day,- filters: [- [:is, "event:goal", ["Signup", "Visit /thank-you"]]- ],- dimensions: [],- order_by: nil,- timezone: ^expected_timezone,- include: %{imports: false, time_labels: false, total_rows: false, comparisons: nil},- pagination: %{limit: 10_000, offset: 0},- preloaded_goals: [- %Plausible.Goal{page_path: "/thank-you"},- %Plausible.Goal{event_name: "Signup"}- ]- } = res+ |> check_success(+ site,+ %{+ metrics: [:visitors],+ utc_time_range: @date_range_day,+ filters: [+ [:is, "event:goal", ["Signup", "Visit /thank-you"]]+ ],+ dimensions: [],+ order_by: nil,+ timezone: site.timezone,+ include: %{imports: false, time_labels: false, total_rows: false, comparisons: nil},+ pagination: %{limit: 10_000, offset: 0}+ }+ )+ |> check_goals(+ preloaded_goals: %{+ all: ["Signup", "Visit /thank-you"],+ matching_toplevel_filters: ["Signup", "Visit /thank-you"]+ },+ revenue_warning: nil,+ revenue_currencies: %{}+ )endtest "invalid event filter", %{site: site} do@@ -1078,6 +1174,74 @@ defmodule Plausible.Stats.Filters.QueryParserTest do"Invalid filters. Dimension `event:goal` can only be filtered at the top level.")end++ test "allowed within behavioral filters has_done", %{site: site} do+ insert(:goal, %{site: site, event_name: "Signup"})++ %{+ "site_id" => site.domain,+ "metrics" => ["visitors"],+ "date_range" => "all",+ "filters" => [+ [+ "has_done",+ [+ "or",+ [+ ["is", "event:goal", ["Signup"]],+ ["is", "event:name", ["pageview"]]+ ]+ ]+ ]+ ]+ }+ |> check_success(+ site,+ %{+ metrics: [:visitors],+ utc_time_range: @date_range_day,+ filters: [+ [+ :has_done,+ [+ :or,+ [+ [:is, "event:goal", ["Signup"]],+ [:is, "event:name", ["pageview"]]+ ]+ ]+ ]+ ],+ dimensions: [],+ order_by: nil,+ timezone: site.timezone,+ include: %{imports: false, time_labels: false, total_rows: false, comparisons: nil},+ pagination: %{limit: 10_000, offset: 0}+ },+ :internal+ )+ |> check_goals(+ preloaded_goals: %{all: ["Signup"], matching_toplevel_filters: ["Signup"]},+ revenue_warning: nil,+ revenue_currencies: %{}+ )+ end++ test "name is checked even within behavioral filters", %{site: site} do+ insert(:goal, %{site: site, event_name: "Signup"})++ %{+ "site_id" => site.domain,+ "metrics" => ["visitors"],+ "date_range" => "all",+ "filters" => [["has_done", ["is", "event:goal", ["Unknown"]]]]+ }+ |> check_error(+ site,+ "The goal `Unknown` is not configured for this site. Find out how to configure goals here: https://plausible.io/docs/stats-api#filtering-by-goals",+ :internal+ )+ endenddescribe "date range validation" do@@ -1506,7 +1670,13 @@ defmodule Plausible.Stats.Filters.QueryParserTest doinclude: %{imports: false, time_labels: false, total_rows: false, comparisons: nil},pagination: %{limit: 10_000, offset: 0}})- |> check_goals(preloaded_goals: ["Signup"], revenue_currencies: %{})+ |> check_goals(+ preloaded_goals: %{+ all: ["Purchase", "Signup"],+ matching_toplevel_filters: ["Signup"]+ },+ revenue_currencies: %{}+ )endtest "succeeds with event:goal dimension", %{site: site} do@@ -1529,7 +1699,13 @@ defmodule Plausible.Stats.Filters.QueryParserTest doinclude: %{imports: false, time_labels: false, total_rows: false, comparisons: nil},pagination: %{limit: 10_000, offset: 0}})- |> check_goals(preloaded_goals: ["Purchase", "Signup"], revenue_currencies: %{})+ |> check_goals(+ preloaded_goals: %{+ all: ["Purchase", "Signup"],+ matching_toplevel_filters: ["Purchase", "Signup"]+ },+ revenue_currencies: %{}+ )endtest "custom properties filter with special metric", %{site: site} do@@ -1664,7 +1840,10 @@ defmodule Plausible.Stats.Filters.QueryParserTest doinclude: %{imports: false, time_labels: false, total_rows: false, comparisons: nil},pagination: %{limit: 10_000, offset: 0}})- |> check_goals(preloaded_goals: ["Signup"], revenue_currencies: %{})+ |> check_goals(+ preloaded_goals: %{all: ["Signup"], matching_toplevel_filters: ["Signup"]},+ revenue_currencies: %{}+ )endtest "fails validation if event:page filter specified", %{site: site} do@@ -1722,7 +1901,10 @@ defmodule Plausible.Stats.Filters.QueryParserTest do})|> check_goals(- preloaded_goals: [],+ preloaded_goals: %{+ all: [],+ matching_toplevel_filters: []+ },revenue_warning: :no_revenue_goals_matching,revenue_currencies: %{})@@ -1777,7 +1959,10 @@ defmodule Plausible.Stats.Filters.QueryParserTest do})|> check_goals(- preloaded_goals: ["PurchaseUSD", "Signup", "Subscription"],+ preloaded_goals: %{+ all: ["PurchaseUSD", "Signup", "Subscription", "Logout"],+ matching_toplevel_filters: ["PurchaseUSD", "Signup", "Subscription"]+ },revenue_warning: nil,revenue_currencies: %{default: :USD})@@ -1808,7 +1993,10 @@ defmodule Plausible.Stats.Filters.QueryParserTest do})|> check_goals(- preloaded_goals: ["Purchase", "Signup", "Subscription"],+ preloaded_goals: %{+ all: ["Purchase", "Signup", "Subscription"],+ matching_toplevel_filters: ["Purchase", "Signup", "Subscription"]+ },revenue_warning: :no_single_revenue_currency,revenue_currencies: %{})@@ -1839,7 +2027,10 @@ defmodule Plausible.Stats.Filters.QueryParserTest do})|> check_goals(- preloaded_goals: ["Signup"],+ preloaded_goals: %{+ all: ["Purchase", "Subscription", "Signup"],+ matching_toplevel_filters: ["Signup"]+ },revenue_warning: :no_revenue_goals_matching,revenue_currencies: %{})@@ -1870,7 +2061,10 @@ defmodule Plausible.Stats.Filters.QueryParserTest do})|> check_goals(- preloaded_goals: ["Donation", "Purchase", "Signup"],+ preloaded_goals: %{+ all: ["Donation", "Purchase", "Signup"],+ matching_toplevel_filters: ["Donation", "Purchase", "Signup"]+ },revenue_warning: nil,revenue_currencies: %{"Donation" => :EUR, "Purchase" => :USD})@@ -1903,7 +2097,10 @@ defmodule Plausible.Stats.Filters.QueryParserTest do})|> check_goals(- preloaded_goals: ["Purchase", "Signup", "Subscription"],+ preloaded_goals: %{+ all: ["Logout", "Purchase", "Signup", "Subscription"],+ matching_toplevel_filters: ["Purchase", "Signup", "Subscription"]+ },revenue_warning: nil,revenue_currencies: %{"Purchase" => :USD, "Subscription" => :EUR})@@ -1938,7 +2135,10 @@ defmodule Plausible.Stats.Filters.QueryParserTest do})|> check_goals(- preloaded_goals: ["Signup"],+ preloaded_goals: %{+ all: ["Logout", "Signup", "Subscription", "Purchase"],+ matching_toplevel_filters: ["Signup"]+ },revenue_warning: :no_revenue_goals_matching,revenue_currencies: %{})diff --git a/test/plausible/stats/query_test.exs b/test/plausible/stats/query_test.exsindex b6928122a0..fc148aa7c8 100644--- a/test/plausible/stats/query_test.exs+++ b/test/plausible/stats/query_test.exs@@ -29,11 +29,11 @@ defmodule Plausible.Stats.QueryTest do} doq1 = %{now: %DateTime{}} = Query.from(site, %{"period" => "realtime"})q2 = %{now: %DateTime{}} = Query.from(site, %{"period" => "30m"})- boundaries1 = Plausible.Stats.Time.utc_boundaries(q1, site)- boundaries2 = Plausible.Stats.Time.utc_boundaries(q2, site)+ boundaries1 = Plausible.Stats.Time.utc_boundaries(q1)+ boundaries2 = Plausible.Stats.Time.utc_boundaries(q2):timer.sleep(1500)- assert ^boundaries1 = Plausible.Stats.Time.utc_boundaries(q1, site)- assert ^boundaries2 = Plausible.Stats.Time.utc_boundaries(q2, site)+ assert ^boundaries1 = Plausible.Stats.Time.utc_boundaries(q1)+ assert ^boundaries2 = Plausible.Stats.Time.utc_boundaries(q2)endtest "parses day format", %{site: site} dodiff --git a/test/plausible_web/controllers/api/external_stats_controller/query_imported_test.exs b/test/plausible_web/controllers/api/external_stats_controller/query_imported_test.exsindex f9cac1f887..47c2035f02 100644--- a/test/plausible_web/controllers/api/external_stats_controller/query_imported_test.exs+++ b/test/plausible_web/controllers/api/external_stats_controller/query_imported_test.exs@@ -1287,4 +1287,79 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryImportedTest doassert pageviews in 1..2endend++ describe "behavioral filters" do+ setup :create_site_import++ test "imports are skipped when has_done filter is used", %{+ conn: conn,+ site: site,+ site_import: site_import+ } do+ populate_stats(site, site_import.id, [+ build(:event, name: "pageview", user_id: 1, timestamp: ~N[2021-01-01 00:00:00]),+ build(:event, name: "pageview", user_id: 2, timestamp: ~N[2021-01-01 00:00:00]),+ build(:event, name: "Conversion", user_id: 3, timestamp: ~N[2021-01-01 00:00:00]),+ build(:imported_pages,+ page: "/blog",+ pageviews: 5,+ visitors: 3,+ date: ~D[2023-01-01]+ )+ ])++ conn =+ post(conn, "/api/v2/query-internal-test", %{+ "site_id" => site.domain,+ "metrics" => ["visitors"],+ "date_range" => "all",+ "filters" => [+ ["has_done", ["is", "event:name", ["pageview"]]]+ ],+ "include" => %{"imports" => true}+ })++ assert json_response(conn, 200)["results"] == [%{"dimensions" => [], "metrics" => [2]}]+ refute json_response(conn, 200)["meta"]["imports_included"]++ assert json_response(conn, 200)["meta"]["imports_warning"] =~+ "Imported stats are not included in the results"+ end++ test "imports are skipped when has_not_done filter is used", %{+ conn: conn,+ site: site,+ site_import: site_import+ } do+ populate_stats(site, site_import.id, [+ build(:event, name: "pageview", user_id: 1, timestamp: ~N[2021-01-01 00:00:00]),+ build(:event, name: "pageview", user_id: 2, timestamp: ~N[2021-01-01 00:00:00]),+ build(:event, name: "Conversion", user_id: 3, timestamp: ~N[2021-01-01 00:00:00]),+ build(:imported_pages,+ page: "/blog",+ pageviews: 5,+ visitors: 3,+ date: ~D[2023-01-01]+ )+ ])++ conn =+ post(conn, "/api/v2/query-internal-test", %{+ "site_id" => site.domain,+ "metrics" => ["visitors"],+ "date_range" => "all",+ "dimensions" => ["event:goal"],+ "filters" => [+ ["has_not_done", ["is", "event:name", ["pageview"]]]+ ],+ "include" => %{"imports" => true}+ })++ assert json_response(conn, 200)["results"] == []+ refute json_response(conn, 200)["meta"]["imports_included"]++ assert json_response(conn, 200)["meta"]["imports_warning"] =~+ "Imported stats are not included in the results"+ end+ endenddiff --git a/test/plausible_web/controllers/api/external_stats_controller/query_test.exs b/test/plausible_web/controllers/api/external_stats_controller/query_test.exsindex 4697b94c71..33da8e25e6 100644--- a/test/plausible_web/controllers/api/external_stats_controller/query_test.exs+++ b/test/plausible_web/controllers/api/external_stats_controller/query_test.exs@@ -4578,6 +4578,296 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest doendend+ describe "behavioral (has_done/has_not_done) filters" do+ test "has_done does counting by sessions", %{conn: conn, site: site} do+ populate_stats(site, [+ # Session 1+ build(:event, name: "Conversion", user_id: 1, timestamp: ~N[2021-01-01 00:00:00]),+ build(:event, name: "pageview", user_id: 1, timestamp: ~N[2021-01-01 00:01:00]),+ # Session 2+ build(:event, name: "pageview", user_id: 1, timestamp: ~N[2021-01-02 00:00:00]),+ build(:event, name: "pageview", user_id: 1, timestamp: ~N[2021-01-02 00:01:00]),+ # Session 3+ build(:event, name: "Conversion", user_id: 1, timestamp: ~N[2021-01-03 00:00:00]),+ build(:event, name: "pageview", user_id: 1, timestamp: ~N[2021-01-03 00:01:00]),+ build(:event, name: "pageview", user_id: 1, timestamp: ~N[2021-01-03 00:02:00]),+ build(:event, name: "pageview", user_id: 1, timestamp: ~N[2021-01-03 00:03:00]),+ # Session 4+ build(:event, name: "Conversion", user_id: 2, timestamp: ~N[2021-01-01 00:00:00]),+ build(:event, name: "pageview", user_id: 2, timestamp: ~N[2021-01-01 00:01:00]),+ build(:event, name: "pageview", user_id: 2, timestamp: ~N[2021-01-01 00:02:00]),+ build(:event, name: "pageview", user_id: 2, timestamp: ~N[2021-01-01 00:03:00]),+ build(:event, name: "pageview", user_id: 2, timestamp: ~N[2021-01-01 00:04:00])+ ])++ conn =+ post(conn, "/api/v2/query-internal-test", %{+ "site_id" => site.domain,+ "metrics" => ["visitors", "visits", "events", "pageviews"],+ "date_range" => "all",+ "filters" => [["has_done", ["is", "event:name", ["Conversion"]]]]+ })++ assert json_response(conn, 200)["results"] == [+ %{"dimensions" => [], "metrics" => [2, 3, 11, 8]}+ ]+ end++ test "has_done returns all events by users who match condition if no further filters are provided",+ %{conn: conn, site: site} do+ populate_stats(site, [+ build(:event, name: "pageview", user_id: 1, timestamp: ~N[2021-01-01 00:00:00]),+ build(:event, name: "AddToCart", user_id: 1, timestamp: ~N[2021-01-01 00:00:00]),+ build(:event, name: "Purchase", user_id: 1, timestamp: ~N[2021-01-01 00:00:00]),+ build(:event, name: "pageview", user_id: 2, timestamp: ~N[2021-01-01 00:00:00]),+ build(:event, name: "Purchase", user_id: 2, timestamp: ~N[2021-01-01 00:00:00]),+ build(:event, name: "pageview", user_id: 3, timestamp: ~N[2021-01-01 00:00:00]),+ build(:event, name: "pageview", user_id: 4, timestamp: ~N[2021-01-01 00:00:00]),+ build(:event, name: "Purchase", user_id: 5, timestamp: ~N[2021-01-01 00:00:00])+ ])++ conn =+ post(conn, "/api/v2/query-internal-test", %{+ "site_id" => site.domain,+ "metrics" => ["visitors", "events", "pageviews"],+ "date_range" => "all",+ "filters" => [["has_done", ["is", "event:name", ["Purchase"]]]]+ })++ assert json_response(conn, 200)["results"] == [+ %{"dimensions" => [], "metrics" => [3, 6, 2]}+ ]+ end++ test "has_done event:page filter", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:event,+ name: "pageview",+ user_id: 1,+ pathname: "/blog/post/1",+ timestamp: ~N[2021-01-01 00:00:00]+ ),+ build(:event,+ name: "pageview",+ user_id: 1,+ pathname: "/",+ timestamp: ~N[2021-01-01 00:00:00]+ ),+ build(:event,+ name: "pageview",+ user_id: 2,+ pathname: "/",+ timestamp: ~N[2021-01-01 00:00:00]+ ),+ build(:event,+ name: "pageview",+ user_id: 3,+ pathname: "/blog/post/1",+ timestamp: ~N[2021-01-01 00:00:00]+ )+ ])++ conn =+ post(conn, "/api/v2/query-internal-test", %{+ "site_id" => site.domain,+ "metrics" => ["visitors", "pageviews"],+ "date_range" => "all",+ "filters" => [["has_done", ["contains", "event:page", ["/blog/"]]]]+ })++ assert json_response(conn, 200)["results"] == [+ %{"dimensions" => [], "metrics" => [2, 3]}+ ]+ end++ test "has_not_done event:page filter", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:event,+ name: "pageview",+ user_id: 1,+ pathname: "/blog/post/1",+ timestamp: ~N[2021-01-01 00:00:00]+ ),+ build(:event,+ name: "pageview",+ user_id: 1,+ pathname: "/",+ timestamp: ~N[2021-01-01 00:00:00]+ ),+ build(:event,+ name: "pageview",+ user_id: 2,+ pathname: "/",+ timestamp: ~N[2021-01-01 00:00:00]+ ),+ build(:event,+ name: "pageview",+ user_id: 3,+ pathname: "/blog/post/1",+ timestamp: ~N[2021-01-01 00:00:00]+ )+ ])++ conn =+ post(conn, "/api/v2/query-internal-test", %{+ "site_id" => site.domain,+ "metrics" => ["visitors", "pageviews"],+ "date_range" => "all",+ "filters" => [["has_not_done", ["contains", "event:page", ["/blog/"]]]]+ })++ assert json_response(conn, 200)["results"] == [+ %{"dimensions" => [], "metrics" => [1, 1]}+ ]+ end++ test "has_done with complex event:props and event:name filters", %{conn: conn, site: site} do+ populate_stats(site, [+ build(:event, name: "Purchase", user_id: 1, timestamp: ~N[2021-01-01 00:00:00]),+ build(:event, name: "Purchase", user_id: 1, timestamp: ~N[2021-01-01 00:10:00]),+ build(:event, name: "Purchase", user_id: 1, timestamp: ~N[2021-01-01 00:20:00]),+ build(:event, name: "pageview", user_id: 2, timestamp: ~N[2021-01-01 00:00:00]),+ build(:event, name: "Purchase", user_id: 2, timestamp: ~N[2021-01-01 00:10:00]),+ build(:event,+ name: "Signup",+ user_id: 3,+ "meta.key": ["paid"],+ "meta.value": ["true"],+ timestamp: ~N[2021-01-01 00:00:00]+ ),+ build(:event, name: "pageview", user_id: 3, timestamp: ~N[2021-01-01 00:00:00]),+ build(:event,+ name: "Signup",+ user_id: 4,+ "meta.key": ["paid"],+ "meta.value": ["true"],+ timestamp: ~N[2021-01-01 00:00:00]+ ),+ build(:event, name: "pageview", user_id: 4, timestamp: ~N[2021-01-01 00:00:00]),+ build(:event,+ name: "Signup",+ user_id: 5,+ "meta.key": ["paid"],+ "meta.value": ["false"],+ timestamp: ~N[2021-01-01 00:00:00]+ ),+ build(:event, name: "pageview", user_id: 5, timestamp: ~N[2021-01-01 00:00:00])+ ])++ conn =+ post(conn, "/api/v2/query-internal-test", %{+ "site_id" => site.domain,+ "metrics" => ["visitors", "pageviews"],+ "date_range" => "all",+ "filters" => [+ [+ "has_done",+ [+ "or",+ [+ ["is", "event:name", ["Purchase"]],+ [+ "and",+ [+ ["is", "event:name", ["Signup"]],+ ["is", "event:props:paid", ["true"]]+ ]+ ]+ ]+ ]+ ]+ ]+ })++ assert json_response(conn, 200)["results"] == [+ %{"dimensions" => [], "metrics" => [4, 3]}+ ]+ end++ test "has_done event:goal filter", %{conn: conn, site: site} do+ insert(:goal, site: site, event_name: "Conversion")++ populate_stats(site, [+ build(:event, name: "Conversion", user_id: 1, timestamp: ~N[2021-01-01 00:00:00]),+ build(:event, name: "Conversion", user_id: 1, timestamp: ~N[2021-01-01 00:00:00]),+ build(:event, name: "pageview", user_id: 1, timestamp: ~N[2021-01-01 00:00:00]),+ build(:event, name: "pageview", user_id: 1, timestamp: ~N[2021-01-01 00:00:00]),+ build(:event, name: "pageview", user_id: 1, timestamp: ~N[2021-01-01 00:00:00]),+ build(:event, name: "Conversion", user_id: 2, timestamp: ~N[2021-01-01 00:00:00]),+ build(:event, name: "pageview", user_id: 2, timestamp: ~N[2021-01-01 00:00:00]),+ build(:event, name: "Conversion", user_id: 3, timestamp: ~N[2021-01-01 00:00:00]),+ build(:event, name: "pageview", user_id: 4, timestamp: ~N[2021-01-01 00:00:00])+ ])++ conn =+ post(conn, "/api/v2/query-internal-test", %{+ "site_id" => site.domain,+ "metrics" => ["visitors", "pageviews"],+ "date_range" => "all",+ "filters" => [+ ["has_done", ["is", "event:goal", ["Conversion"]]],+ ["is", "event:name", ["pageview"]]+ ]+ })++ assert json_response(conn, 200)["results"] == [+ %{"dimensions" => [], "metrics" => [2, 4]}+ ]+ end++ test "has_not_done event:goal filter", %{conn: conn, site: site} do+ insert(:goal, site: site, event_name: "Conversion")++ populate_stats(site, [+ build(:event, name: "Conversion", user_id: 1, timestamp: ~N[2021-01-01 00:00:00]),+ build(:event, name: "Conversion", user_id: 1, timestamp: ~N[2021-01-01 00:00:00]),+ build(:event, name: "pageview", user_id: 1, timestamp: ~N[2021-01-01 00:00:00]),+ build(:event, name: "pageview", user_id: 1, timestamp: ~N[2021-01-01 00:00:00]),+ build(:event, name: "pageview", user_id: 1, timestamp: ~N[2021-01-01 00:00:00]),+ build(:event, name: "Conversion", user_id: 2, timestamp: ~N[2021-01-01 00:00:00]),+ build(:event, name: "pageview", user_id: 2, timestamp: ~N[2021-01-01 00:00:00]),+ build(:event, name: "Conversion", user_id: 3, timestamp: ~N[2021-01-01 00:00:00]),+ build(:event, name: "pageview", user_id: 4, timestamp: ~N[2021-01-01 00:00:00])+ ])++ conn =+ post(conn, "/api/v2/query-internal-test", %{+ "site_id" => site.domain,+ "metrics" => ["visitors", "pageviews"],+ "date_range" => "all",+ "filters" => [+ ["has_not_done", ["is", "event:goal", ["Conversion"]]],+ ["is", "event:name", ["pageview"]]+ ]+ })++ assert json_response(conn, 200)["results"] == [+ %{"dimensions" => [], "metrics" => [1, 1]}+ ]+ end++ test "visit filters are not allowed with has_done/has_not_done filters", %{+ conn: conn,+ site: site+ } do+ conn =+ post(conn, "/api/v2/query-internal-test", %{+ "site_id" => site.domain,+ "metrics" => ["visitors"],+ "date_range" => "all",+ "dimensions" => ["visit:browser"],+ "filters" => [+ ["has_done", ["is", "visit:browser", ["Chrome"]]]+ ]+ })++ assert %{"error" => error} = json_response(conn, 400)++ assert error =~+ "Invalid filters. Behavioral filters (has_done, has_not_done) can only be used with event dimension filters."+ end+ end+describe "segment filters" dosetup [:create_user, :create_site, :create_api_key, :use_api_key]