Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.
TypeScript analyticsanalytics-dashboardclickhouseelixirgoogle-analyticsmarketingopen-source-analyticsphoenixplausible-analyticspostgresqlprivacyprivacy-friendlyself-hostedsimple-analyticsstatisticstailwindcssweb-analyticswebsite-analyticswebsite-statswebsite-tracking
Goal join performance (#6770)
Uku Tahtd28d181a696b
diff --git a/lib/plausible/google/search_console/filters.ex b/lib/plausible/google/search_console/filters.exindex 1ea9c4a8fa..f61005a0c4 100644--- a/lib/plausible/google/search_console/filters.ex+++ b/lib/plausible/google/search_console/filters.ex@@ -1,6 +1,6 @@defmodule Plausible.Google.SearchConsole.Filters do@moduledoc false- import Plausible.Stats.Filters.Utils, only: [page_regex: 1]+ import Plausible.Stats.Filters.Utils, only: [page_regex: 1, split_page_patterns: 1]def transform(property, plausible_filters, search) dogsc_filters =@@ -35,7 +35,9 @@ defmodule Plausible.Google.SearchConsole.Filters dodefp transform_filter(property, [:matches_wildcard, "visit:entry_page", pages | _])when is_list(pages) doexpression =- Enum.map_join(pages, "|", fn page -> page_regex(property_url(property, page)) end)+ pages+ |> Enum.flat_map(&split_page_patterns/1)+ |> Enum.map_join("|", fn page -> page_regex(property_url(property, page)) end)%{dimension: "page", operator: "includingRegex", expression: expression}enddiff --git a/lib/plausible/stats/filters/utils.ex b/lib/plausible/stats/filters/utils.exindex 48dd501a87..0a2747df4c 100644--- a/lib/plausible/stats/filters/utils.ex+++ b/lib/plausible/stats/filters/utils.ex@@ -1,14 +1,23 @@defmodule Plausible.Stats.Filters.Utils do@moduledoc false+ def uses_page_regex?(path), do: String.contains?(path, ["*", "|"])++ # Preserve escaped pipes until each alternative is converted to a regex.+ def split_page_patterns(expr), do: String.split(expr, ~r/(?<!\\)\|/)+def page_regex(expr) doescaped =expr- |> Regex.escape()- |> String.replace("\\|", "|")- |> String.replace("\\*\\*", ".*")- |> String.replace("\\*", ".*")+ |> split_page_patterns()+ |> Enum.map_join("|", fn pattern ->+ pattern+ |> String.replace("\\|", "|")+ |> Regex.escape()+ |> String.replace("\\*\\*", ".*")+ |> String.replace("\\*", ".*")+ end)- "^#{escaped}$"+ "^(?:#{escaped})$"endenddiff --git a/lib/plausible/stats/goals.ex b/lib/plausible/stats/goals.exindex 7733233793..02b93015c9 100644--- a/lib/plausible/stats/goals.ex+++ b/lib/plausible/stats/goals.ex@@ -4,7 +4,7 @@ defmodule Plausible.Stats.Goals do"""import Ecto.Query- import Plausible.Stats.Filters.Utils, only: [page_regex: 1]+ import Plausible.Stats.Filters.Utils, only: [page_regex: 1, uses_page_regex?: 1]alias Plausible.Stats.Filters@@ -63,60 +63,39 @@ defmodule Plausible.Stats.Goals doend)end- @type goal_join_data() :: %{- indices: [non_neg_integer()],- event_names_by_type: [String.t()],- page_regexes: [String.t()],- scroll_thresholds: [non_neg_integer()],- custom_props_keys: [[String.t()]],- custom_props_values: [[String.t()]]- }+ @doc """+ Assigns breakdown goals 1-based indices, matching ClickHouse array positions.+ Assign indices before partitioning or excluding goals so native and imported+ queries use the same identifiers, decoded by `goal_at_index/2`.+ """+ @spec indexed_goals(Plausible.Stats.Query.t()) :: [{Plausible.Goal.t(), pos_integer()}]+ def indexed_goals(query) do+ Enum.with_index(query.preloaded_goals.matching_toplevel_filters, 1)+ end++ @doc """+ Retrieves a breakdown goal using the 1-based index assigned by `indexed_goals/1`.+ """+ @spec goal_at_index(Plausible.Stats.Query.t(), pos_integer()) :: Plausible.Goal.t() | nil+ def goal_at_index(query, index) do+ Enum.at(query.preloaded_goals.matching_toplevel_filters, index - 1)+ end@type goal_join_imported_data() :: %{- indices: [non_neg_integer()],+ indices: [pos_integer()],types: [String.t()],event_names: [String.t()],page_regexes: [String.t()]}- @doc """- Returns data needed to perform a GROUP BY on goals in an ecto query.- """- @spec goal_join_data(Plausible.Stats.Query.t()) :: goal_join_data()- def goal_join_data(query) do- {goals, indices} =- query.preloaded_goals.matching_toplevel_filters- |> Enum.with_index(1)- |> Enum.unzip()-- {custom_props_keys, custom_props_values} =- goals- |> Enum.map(fn goal -> Enum.unzip(goal.custom_props) end)- |> Enum.unzip()-- %{- indices: indices,- event_names_by_type:- Enum.map(goals, fn goal -> event_name_by_type(Plausible.Goal.type(goal), goal) end),- # Event goals are considered to match everything for the sake of efficient queries in query_builder.ex- # See also Plausible.Stats.SQL.Expression.event_goal_join/1- page_regexes:- Enum.map(goals, fn goal -> page_regex_for_goal(Plausible.Goal.type(goal), goal) end),- scroll_thresholds: Enum.map(goals, fn goal -> goal.scroll_threshold end),- custom_props_keys: custom_props_keys,- custom_props_values: custom_props_values- }- end-@doc """Returns data needed to perform a GROUP BY on goals in an imported data query."""@spec goal_join_imported_data(Plausible.Stats.Query.t()) :: goal_join_imported_data()def goal_join_imported_data(query) do{goals, indices} =- query.preloaded_goals.matching_toplevel_filters- # Keep indices aligned with the complete preloaded goal list.- |> Enum.with_index(1)+ query+ |> indexed_goals()|> Enum.reject(fn {goal, _idx} -> Plausible.Goal.has_custom_props?(goal) end)|> Enum.unzip()@@ -124,18 +103,28 @@ defmodule Plausible.Stats.Goals doindices: indices,types: Enum.map(goals, fn goal -> goal |> Plausible.Goal.type() |> to_string() end),event_names: Enum.map(goals, fn goal -> to_string(goal.event_name) end),- page_regexes:- Enum.map(goals, fn goal -> page_regex_for_goal(Plausible.Goal.type(goal), goal) end)+ page_regexes: Enum.map(goals, &pathname_regex/1)}end- defp event_name_by_type(:event, goal), do: goal.event_name- defp event_name_by_type(:page, _goal), do: "pageview"- defp event_name_by_type(:scroll, _goal), do: "engagement"+ def expected_event_name(goal) do+ case Plausible.Goal.type(goal) do+ :event -> goal.event_name+ :page -> "pageview"+ :scroll -> "engagement"+ end+ end- defp page_regex_for_goal(:event, _goal), do: ".?"- defp page_regex_for_goal(:page, goal), do: Filters.Utils.page_regex(goal.page_path)- defp page_regex_for_goal(:scroll, goal), do: Filters.Utils.page_regex(goal.page_path)+ @doc """+ The regex an event's pathname needs to match to complete the goal. Event goals+ match any pathname.+ """+ def pathname_regex(goal) do+ case Plausible.Goal.type(goal) do+ :event -> ".?"+ _page_or_scroll -> page_regex(goal.page_path)+ end+ enddef toplevel_scroll_goal_filters?(query) dogoal_filters? =@@ -259,7 +248,7 @@ defmodule Plausible.Stats.Goals dodefp page_path_condition(page_path, imported?) dodb_field = page_path_db_field(imported?)- if String.contains?(page_path, "*") do+ if uses_page_regex?(page_path) dodynamic([e], fragment("match(?, ?)", field(e, ^db_field), ^page_regex(page_path)))elsedynamic([e], field(e, ^db_field) == ^page_path)diff --git a/lib/plausible/stats/query_runner.ex b/lib/plausible/stats/query_runner.exindex 6262890992..2b509c982e 100644--- a/lib/plausible/stats/query_runner.ex+++ b/lib/plausible/stats/query_runner.ex@@ -210,8 +210,7 @@ defmodule Plausible.Stats.QueryRunner dodefp get_dimension_goal(entry, query) dogoal_index = Map.get(entry, Util.shortname(query, "event:goal"))- query.preloaded_goals.matching_toplevel_filters- |> Enum.at(goal_index - 1)+ Plausible.Stats.Goals.goal_at_index(query, goal_index)enddefp merge_with_comparison_results(results_list, runner) dodiff --git a/lib/plausible/stats/sql/expression.ex b/lib/plausible/stats/sql/expression.exindex 142ad71f1d..432e836c40 100644--- a/lib/plausible/stats/sql/expression.ex+++ b/lib/plausible/stats/sql/expression.ex@@ -164,6 +164,13 @@ defmodule Plausible.Stats.SQL.Expression dodef select_dimension(q, key, "event:name", _table, _query),do: select_merge_as(q, [t], %{key => t.name})+ # One row per goal the event completes. Events completing no goal drop out.+ def select_dimension(q, key, "event:goal", :events, query) do+ select_merge_as(q, [t], %{+ key => fragment("arrayJoin(?)", ^SQL.GoalMatching.completed_goals(query))+ })+ end+def select_dimension(q, key, "event:page", _table, _query),do: select_merge_as(q, [t], %{key => t.pathname})@@ -502,82 +509,4 @@ defmodule Plausible.Stats.SQL.Expression dodef session_metric(:percentage, _query), do: %{}def session_metric(:conversion_rate, _query), do: %{}def session_metric(:group_conversion_rate, _query), do: %{}-- @doc """- The fragment matches events to goals by:- 1. Checking if the pathname matches the goal's page regex pattern- 2. Verifying the event name matches the expected name for the goal type- 3. Validating scroll depth is within threshold (for scroll goals)- 4. Ensuring all custom properties match (if any are defined on the goal)-- Returns an array of goal indices that the event matches.- """- defmacro event_goal_join(goal_join_data) do- quote do- fragment(- """- arrayIntersect(- multiMatchAllIndices(?, ?),- arrayMap(- (expected_name, threshold, index, custom_props_keys, custom_props_values) -> if(- expected_name = ? and ? between threshold and 100 and- (empty(custom_props_keys) OR arrayAll((k, v) -> ?[indexOf(?, k)] = v, custom_props_keys, custom_props_values)),- index, -1- ),- ?,- ?,- ?,- ?,- ?- )- )- """,- e.pathname,- type(^unquote(goal_join_data).page_regexes, {:array, :string}),- e.name,- e.scroll_depth,- field(e, :"meta.value"),- field(e, :"meta.key"),- type(^unquote(goal_join_data).event_names_by_type, {:array, :string}),- type(^unquote(goal_join_data).scroll_thresholds, {:array, :integer}),- type(^unquote(goal_join_data).indices, {:array, :integer}),- type(^unquote(goal_join_data).custom_props_keys, {:array, {:array, :string}}),- type(^unquote(goal_join_data).custom_props_values, {:array, {:array, :string}})- )- end- end-- @doc """- Optimized variant of `event_goal_join/1` for use when no goals have custom- property filters. Omits all references to `meta.key` and `meta.value`,- preventing ClickHouse from reading those expensive Array(String) columns at- scan time.- """- defmacro event_goal_join_no_props(goal_join_data) do- quote do- fragment(- """- arrayIntersect(- multiMatchAllIndices(?, ?),- arrayMap(- (expected_name, threshold, index) -> if(- expected_name = ? and ? between threshold and 100,- index, -1- ),- ?,- ?,- ?- )- )- """,- e.pathname,- type(^unquote(goal_join_data).page_regexes, {:array, :string}),- e.name,- e.scroll_depth,- type(^unquote(goal_join_data).event_names_by_type, {:array, :string}),- type(^unquote(goal_join_data).scroll_thresholds, {:array, :integer}),- type(^unquote(goal_join_data).indices, {:array, :integer})- )- end- endenddiff --git a/lib/plausible/stats/sql/goal_matching.ex b/lib/plausible/stats/sql/goal_matching.exnew file mode 100644index 0000000000..cade41d370--- /dev/null+++ b/lib/plausible/stats/sql/goal_matching.ex@@ -0,0 +1,211 @@+defmodule Plausible.Stats.SQL.GoalMatching do+ @moduledoc """+ Builds the matching goal indices for an `event:goal` breakdown.++ Event names and exact page paths use lookups when no custom properties+ are required. Other goals use regex matching and additional filters.++ Goal indices are assigned by `Plausible.Stats.Goals.indexed_goals/1` and+ decoded by `Plausible.Stats.Goals.goal_at_index/2` in QueryRunner.+ """++ import Ecto.Query+ import Plausible.Stats.Filters.Utils, only: [uses_page_regex?: 1]++ alias Plausible.Goal+ alias Plausible.Stats.Goals++ @typep keyed_goal() :: {String.t(), pos_integer()}++ @doc """+ An `Array(UInt64)` expression of the indices of the goals the event completes.+ """+ def completed_goals(query) do+ %{by_name: by_name, by_pathname: by_pathname, scanned: scanned} =+ partition(Goals.indexed_goals(query))++ [+ lookup(dynamic([e], e.name), by_name),+ lookup_by_pathname(by_pathname),+ scan(scanned)+ ]+ |> Enum.reject(&is_nil/1)+ |> concat()+ end++ # Groups goals, with their indices, by how they are matched. The unique indexes+ # on goals keep each event name and exact page path to one goal without custom+ # props, so a lookup by either finds at most one goal.+ defp partition(indexed_goals) do+ classified =+ for {goal, index} <- indexed_goals, do: {matching_strategy(goal), goal, index}++ %{+ by_name: for({{:name, name}, _goal, index} <- classified, do: {name, index}),+ by_pathname: for({{:pathname, path}, _goal, index} <- classified, do: {path, index}),+ scanned: for({:scan, goal, index} <- classified, do: {goal, index})+ }+ end++ defp matching_strategy(goal) do+ case {Goal.type(goal), Goal.has_custom_props?(goal)} do+ {:event, false} ->+ {:name, goal.event_name}++ {:event, true} ->+ :scan++ {:page, false} ->+ if uses_page_regex?(goal.page_path), do: :scan, else: {:pathname, goal.page_path}++ {:page, true} ->+ :scan++ {:scroll, _has_custom_props?} ->+ :scan+ end+ end++ # Look up the goal index matching `event_field`. transform() is a hash lookup,+ # while has() on a constant array would scan it on every row.+ @spec lookup(Ecto.Query.dynamic_expr(), [keyed_goal()]) :: Ecto.Query.dynamic_expr() | nil+ defp lookup(_event_field, []), do: nil++ defp lookup(event_field, keyed_goals) do+ {keys, goal_indices} = Enum.unzip(keyed_goals)++ dynamic(+ fragment(+ "arrayFilter(i -> i != 0, [transform(?, ?, CAST(? AS Array(UInt64)), toUInt64(0))])",+ ^event_field,+ type(^keys, {:array, :string}),+ type(^goal_indices, {:array, :integer})+ )+ )+ end++ defp lookup_by_pathname([]), do: nil++ defp lookup_by_pathname(keyed_goals) do+ goal_indices_expr = lookup(dynamic([e], e.pathname), keyed_goals)++ dynamic(+ [e],+ fragment("if(?, ?, CAST([] AS Array(UInt64)))", e.name == "pageview", ^goal_indices_expr)+ )+ end++ defp scan([]), do: nil++ # Regex matches are 1-based positions within this scanned subset of goals.+ defp scan(indexed_goals) do+ {goals, goal_indices} = Enum.unzip(indexed_goals)+ expected_names = Enum.map(goals, &Goals.expected_event_name/1)+ distinct_names = Enum.uniq(expected_names)++ goal_indices_expr =+ dynamic(+ [e],+ fragment(+ "multiMatchAllIndices(?, ?)",+ e.pathname,+ type(^Enum.map(goals, &Goals.pathname_regex/1), {:array, :string})+ )+ )+ |> keep_expected_events(goals, expected_names, distinct_names)+ |> keep_matching_custom_props(goals)+ |> to_goal_indices(goal_indices)++ dynamic(+ [e],+ fragment(+ "if(?, ?, CAST([] AS Array(UInt64)))",+ e.name in ^distinct_names,+ ^goal_indices_expr+ )+ )+ end++ # The outer name guard is sufficient when all scanned goals expect the same name.+ # Omit the depth check when there are no scroll goals.+ # The upper bound excludes the missing-depth sentinel, 255.+ defp keep_expected_events(positions, goals, expected_names, distinct_names) do+ name_check =+ if length(distinct_names) > 1 do+ dynamic(+ [e],+ fragment("?[goal_pos] = ?", type(^expected_names, {:array, :string}), e.name)+ )+ end++ threshold_check =+ if Enum.any?(goals, &(Goal.type(&1) == :scroll)) do+ thresholds = Enum.map(goals, & &1.scroll_threshold)++ dynamic(+ [e],+ fragment(+ "(? BETWEEN ?[goal_pos] AND 100)",+ e.scroll_depth,+ type(^thresholds, {:array, :integer})+ )+ )+ end++ case Enum.reject([name_check, threshold_check], &is_nil/1) do+ [] ->+ positions++ checks ->+ condition = chain_conditions(checks)+ dynamic(fragment("arrayFilter(goal_pos -> ?, ?)", ^condition, ^positions))+ end+ end++ # Check custom properties only for positions that survived the earlier filters.+ # Omit meta.* references entirely when no scanned goal has custom properties.+ defp keep_matching_custom_props(positions, goals) do+ if Enum.any?(goals, &Goal.has_custom_props?/1) do+ {keys, values} =+ goals+ |> Enum.map(&Enum.unzip(&1.custom_props))+ |> Enum.unzip()++ props_check =+ dynamic(+ [e],+ fragment(+ "(empty(?[goal_pos]) OR arrayAll((k, v) -> ?[indexOf(?, k)] = v, ?[goal_pos], ?[goal_pos]))",+ type(^keys, {:array, {:array, :string}}),+ field(e, :"meta.value"),+ field(e, :"meta.key"),+ type(^keys, {:array, {:array, :string}}),+ type(^values, {:array, {:array, :string}})+ )+ )++ dynamic(fragment("arrayFilter(goal_pos -> ?, ?)", ^props_check, ^positions))+ else+ positions+ end+ end++ defp chain_conditions(conditions) do+ Enum.reduce(conditions, fn condition, acc -> dynamic(^acc and ^condition) end)+ end++ defp to_goal_indices(positions, goal_indices) do+ dynamic(+ fragment(+ "arrayMap(goal_pos -> CAST(? AS Array(UInt64))[goal_pos], ?)",+ type(^goal_indices, {:array, :integer}),+ ^positions+ )+ )+ end++ defp concat([]), do: dynamic(fragment("CAST([] AS Array(UInt64))"))+ defp concat([goal_indices_expr]), do: goal_indices_expr+ defp concat([a, b]), do: dynamic(fragment("arrayConcat(?, ?)", ^a, ^b))+ defp concat([a, b, c]), do: dynamic(fragment("arrayConcat(?, ?, ?)", ^a, ^b, ^c))+enddiff --git a/lib/plausible/stats/sql/query_builder.ex b/lib/plausible/stats/sql/query_builder.exindex 9e905e63d1..106871a406 100644--- a/lib/plausible/stats/sql/query_builder.ex+++ b/lib/plausible/stats/sql/query_builder.ex@@ -12,8 +12,6 @@ defmodule Plausible.Stats.SQL.QueryBuilder doalias Plausible.Stats.SQL.Expressionalias Plausible.Stats.Legacy.TimeOnPage- require Plausible.Stats.SQL.Expression-def build(query, site) doquery|> QueryOptimizer.split()@@ -180,32 +178,6 @@ defmodule Plausible.Stats.SQL.QueryBuilder do|> select_merge_as([..., meta], %{shortname(query, dimension) => meta.key})end- defp dimension_group_by(q, :events, query, "event:goal" = dimension) do- goal_join_data = Plausible.Stats.Goals.goal_join_data(query)-- if Enum.all?(goal_join_data.custom_props_keys, &Enum.empty?/1) do- from(e in q,- join: goal in Expression.event_goal_join_no_props(goal_join_data),- hints: "ARRAY",- on: true,- select_merge: %{- ^shortname(query, dimension) => fragment("?", goal)- },- group_by: goal- )- else- from(e in q,- join: goal in Expression.event_goal_join(goal_join_data),- hints: "ARRAY",- on: true,- select_merge: %{- ^shortname(query, dimension) => fragment("?", goal)- },- group_by: goal- )- end- end-defp dimension_group_by(q, table, query, dimension) dokey = shortname(query, dimension)diff --git a/lib/plausible/stats/sql/where_builder.ex b/lib/plausible/stats/sql/where_builder.exindex 18353d34cb..1670493c54 100644--- a/lib/plausible/stats/sql/where_builder.ex+++ b/lib/plausible/stats/sql/where_builder.ex@@ -45,13 +45,7 @@ defmodule Plausible.Stats.SQL.WhereBuilder dodefp goal_event_names(goals) dogoals- |> Enum.map(fn goal ->- case Plausible.Goal.type(goal) do- :event -> goal.event_name- :page -> "pageview"- :scroll -> "engagement"- end- end)+ |> Enum.map(&Plausible.Stats.Goals.expected_event_name/1)|> Enum.uniq()enddiff --git a/test/plausible/google/search_console/filters_test.exs b/test/plausible/google/search_console/filters_test.exsindex 12924cf7ec..e61c480b98 100644--- a/test/plausible/google/search_console/filters_test.exs+++ b/test/plausible/google/search_console/filters_test.exs@@ -35,7 +35,7 @@ defmodule Plausible.Google.SearchConsole.FiltersTest do%{dimension: "page",operator: "includingRegex",- expression: "^https://plausible\\.io.*page.*$"+ expression: "^(?:https://plausible\\.io.*page.*)$"}]}@@ -62,6 +62,27 @@ defmodule Plausible.Google.SearchConsole.FiltersTest do]end+ test "prefixes and anchors each alternative in wildcard page filters" do+ filters = [[:matches_wildcard, "visit:entry_page", ["/pricing*|/plans"]]]++ {:ok, [%{filters: [%{expression: expression}]}]} =+ Filters.transform("sc-domain:plausible.io", filters, "")++ regex = Regex.compile!(expression)++ for url <- ["https://plausible.io/pricing/team", "https://plausible.io/plans"] do+ assert Regex.match?(regex, url)+ end++ for url <- [+ "https://plausible.io/legacy/plans",+ "https://plausible.io/plans/more",+ "https://other.io/plans"+ ] do+ refute Regex.match?(regex, url)+ end+ end+test "transforms matches multiple page filter" dofilters = [[:matches_wildcard, "visit:entry_page", ["/pageA*", "/pageB*"]]@@ -75,13 +96,28 @@ defmodule Plausible.Google.SearchConsole.FiltersTest do%{dimension: "page",operator: "includingRegex",- expression: "^https://plausible\\.io/pageA.*$|^https://plausible\\.io/pageB.*$"+ expression:+ "^(?:https://plausible\\.io/pageA.*)$|^(?:https://plausible\\.io/pageB.*)$"}]}]end+ test "preserves escaped pipes in wildcard page filters" do+ filters = [[:matches_wildcard, "visit:entry_page", ["/post*\\|1|/plans"]]]++ {:ok, [%{filters: [%{expression: expression}]}]} =+ Filters.transform("sc-domain:plausible.io", filters, "")++ regex = Regex.compile!(expression)++ assert Regex.match?(regex, "https://plausible.io/post|1")+ assert Regex.match?(regex, "https://plausible.io/plans")+ refute Regex.match?(regex, "https://plausible.io/post|2")+ refute Regex.match?(regex, "https://plausible.io/1")+ end+test "transforms event:page exactly like visit:entry_page" dofilters = [[:matches_wildcard, "event:page", ["/pageA*", "/pageB*"]]@@ -95,7 +131,8 @@ defmodule Plausible.Google.SearchConsole.FiltersTest do%{dimension: "page",operator: "includingRegex",- expression: "^https://plausible\\.io/pageA.*$|^https://plausible\\.io/pageB.*$"+ expression:+ "^(?:https://plausible\\.io/pageA.*)$|^(?:https://plausible\\.io/pageB.*)$"}]}@@ -178,7 +215,7 @@ defmodule Plausible.Google.SearchConsole.FiltersTest do%{dimension: "page",operator: "includingRegex",- expression: "^https://plausible\\.io.*web\\-analytics.*$"+ expression: "^(?:https://plausible\\.io.*web\\-analytics.*)$"},%{dimension: "country", operator: "includingRegex", expression: "EST|POL"}]diff --git a/test/plausible/stats/filters_test.exs b/test/plausible/stats/filters_test.exsindex c239bfa156..d876f926b5 100644--- a/test/plausible/stats/filters_test.exs+++ b/test/plausible/stats/filters_test.exs@@ -5,6 +5,53 @@ defmodule Plausible.Stats.FiltersTest dodoctest Plausible.Stats.Filtersdoctest Plausible.Stats.Filters.Utils+ describe "page regex" do+ test "matches escaped pipes literally" do+ regex = Regex.compile!(Filters.Utils.page_regex("/post\\|1"))++ assert Regex.match?(regex, "/post|1")+ refute Regex.match?(regex, "/post")+ refute Regex.match?(regex, "1")+ refute Regex.match?(regex, "/post|1/more")+ end++ test "anchors every alternative to the whole path" do+ regex = Regex.compile!(Filters.Utils.page_regex("/pricing|/plans"))++ for path <- ["/pricing", "/plans"], do: assert(Regex.match?(regex, path))++ for path <- ["/pricing/enterprise", "/legacy/plans", "/pricing|/plans", "/about"] do+ refute Regex.match?(regex, path)+ end+ end++ test "matches * within a whole path" do+ regex = Regex.compile!(Filters.Utils.page_regex("/pricing/*"))++ assert Regex.match?(regex, "/pricing/")+ assert Regex.match?(regex, "/pricing/team")+ assert Regex.match?(regex, "/pricing/team/monthly")+ refute Regex.match?(regex, "/legacy/pricing/team")+ refute Regex.match?(regex, "/pricing")+ end++ test "matches ** within a whole path" do+ regex = Regex.compile!(Filters.Utils.page_regex("/plans/**"))++ assert Regex.match?(regex, "/plans/")+ assert Regex.match?(regex, "/plans/team/monthly")+ refute Regex.match?(regex, "/legacy/plans/team")+ end++ test "escapes other regex syntax" do+ regex = Regex.compile!(Filters.Utils.page_regex("/file.json"))++ assert Regex.match?(regex, "/file.json")+ refute Regex.match?(regex, "/fileXjson")+ refute Regex.match?(regex, "/file.json/more")+ end+ end+def assert_parsed(input, expected_output) doassert Filters.parse(input) == expected_outputenddiff --git a/test/plausible_web/controllers/api/external_stats_controller/query_goal_custom_props_test.exs b/test/plausible_web/controllers/api/external_stats_controller/query_goal_custom_props_test.exsindex 9201731780..058b185b6e 100644--- a/test/plausible_web/controllers/api/external_stats_controller/query_goal_custom_props_test.exs+++ b/test/plausible_web/controllers/api/external_stats_controller/query_goal_custom_props_test.exs@@ -366,6 +366,55 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryGoalCustomPropsTest dorefute Enum.find(results, &(&1["dimensions"] == ["Button Click"]))end++ test "custom event goals are not completed by pageviews with the same custom props", %{+ conn: conn,+ site: site+ } do+ {:ok, _} =+ Goals.create(site, %{+ "event_name" => "Signup",+ "display_name" => "Premium Signup",+ "custom_props" => %{"plan" => "premium"}+ })++ {:ok, _} = Goals.create(site, %{"page_path" => "/blog/**"})++ populate_stats(site, [+ build(:pageview,+ pathname: "/blog/post",+ "meta.key": ["plan"],+ "meta.value": ["premium"],+ user_id: 1+ ),+ build(:event,+ name: "Signup",+ "meta.key": ["plan"],+ "meta.value": ["premium"],+ user_id: 2+ ),+ build(:event,+ name: "Signup",+ "meta.key": ["plan"],+ "meta.value": ["premium"],+ user_id: 3+ ),+ build(:event, name: "Signup", "meta.key": ["plan"], "meta.value": ["free"], user_id: 4)+ ])++ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "date_range" => "all",+ "metrics" => ["visitors"],+ "dimensions" => ["event:goal"]+ })++ assert json_response(conn, 200)["results"] == [+ %{"dimensions" => ["Premium Signup"], "metrics" => [2]},+ %{"dimensions" => ["Visit /blog/**"], "metrics" => [1]}+ ]+ endenddescribe "goals with custom props and imported data" dodiff --git a/test/plausible_web/controllers/api/external_stats_controller/query_goal_dimension_test.exs b/test/plausible_web/controllers/api/external_stats_controller/query_goal_dimension_test.exsindex 9deb1ee2ce..b50d4b37db 100644--- a/test/plausible_web/controllers/api/external_stats_controller/query_goal_dimension_test.exs+++ b/test/plausible_web/controllers/api/external_stats_controller/query_goal_dimension_test.exs@@ -161,6 +161,77 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryGoalDimensionTest doassert length(results) == Plausible.Goals.max_goals_per_site()end++ test "returns pageview goals with | between alternative paths", %{conn: conn, site: site} do+ insert(:goal, %{site: site, page_path: "/pricing|/plans"})++ populate_stats(site, [+ build(:pageview, pathname: "/pricing", user_id: 1),+ build(:pageview, pathname: "/plans", user_id: 2),+ build(:pageview, pathname: "/about", user_id: 3),+ build(:pageview, pathname: "/pricing/enterprise", user_id: 4),+ build(:pageview, pathname: "/legacy/plans", user_id: 5),+ build(:pageview, pathname: "/pricing|/plans", user_id: 6),+ build(:event, name: "Signup", pathname: "/pricing", user_id: 7)+ ])++ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "date_range" => "all",+ "metrics" => ["visitors"],+ "dimensions" => ["event:goal"]+ })++ assert json_response(conn, 200)["results"] == [+ %{"dimensions" => ["Visit /pricing|/plans"], "metrics" => [2]}+ ]++ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "date_range" => "all",+ "metrics" => ["visitors"],+ "filters" => [["is", "event:goal", ["Visit /pricing|/plans"]]]+ })++ assert json_response(conn, 200)["results"] == [+ %{"dimensions" => [], "metrics" => [2]}+ ]+ end++ test "counts pageview goals on path P and custom event goals that happen on path P separately",+ %{+ conn: conn,+ site: site+ } do+ insert(:goal, %{site: site, event_name: "Signup"})+ insert(:goal, %{site: site, page_path: "/blog/**"})+ insert(:goal, %{site: site, page_path: "/pricing"})++ populate_stats(site, [+ build(:pageview, pathname: "/blog/a", user_id: 1),+ build(:pageview, pathname: "/blog/b", user_id: 2),+ build(:pageview, pathname: "/pricing", user_id: 3),+ build(:event, name: "Signup", pathname: "/blog/a", user_id: 4),+ build(:event, name: "Signup", pathname: "/pricing", user_id: 5),+ build(:event, name: "Signup", pathname: "/", user_id: 6)+ ])++ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "date_range" => "all",+ "metrics" => ["visitors"],+ "dimensions" => ["event:goal"]+ })++ assert json_response(conn, 200)["results"] == [+ %{"dimensions" => ["Signup"], "metrics" => [3]},+ %{"dimensions" => ["Visit /blog/**"], "metrics" => [2]},+ %{"dimensions" => ["Visit /pricing"], "metrics" => [1]}+ ]+ endenddescribe "page scroll goals" dodiff --git a/test/plausible_web/controllers/api/external_stats_controller/query_imported_test.exs b/test/plausible_web/controllers/api/external_stats_controller/query_imported_test.exsindex 9ed544765c..618cfcde5b 100644--- a/test/plausible_web/controllers/api/external_stats_controller/query_imported_test.exs+++ b/test/plausible_web/controllers/api/external_stats_controller/query_imported_test.exs@@ -588,6 +588,45 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryImportedTest doenddescribe "imported data" do+ test "aligns goal filters and breakdowns for alternative paths with imported pages", %{+ conn: conn,+ site: site+ } do+ insert(:goal, site: site, page_path: "/pricing|/plans")+ site_import = insert(:site_import, site: site)++ imported_pages =+ for path <- ["/pricing", "/plans", "/pricing/more", "/legacy/plans", "/pricing|/plans"] do+ build(:imported_pages, page: path, visitors: 2, pageviews: 2, date: ~D[2021-01-01])+ end++ populate_stats(site, site_import.id, [+ build(:pageview, pathname: "/pricing", timestamp: ~N[2021-01-01 00:00:01])+ | imported_pages+ ])++ for {dimensions, filters} <- [+ {["event:goal"], []},+ {[], [["is", "event:goal", ["Visit /pricing|/plans"]]]}+ ] do+ conn =+ post(conn, "/api/v2/query", %{+ "site_id" => site.domain,+ "date_range" => "all",+ "metrics" => ["visitors"],+ "dimensions" => dimensions,+ "filters" => filters,+ "include" => %{"imports" => true}+ })++ expected_dimensions = if dimensions == [], do: [], else: ["Visit /pricing|/plans"]++ assert json_response(conn, 200)["results"] == [+ %{"dimensions" => expected_dimensions, "metrics" => [5]}+ ]+ end+ end+test "returns screen sizes breakdown when filtering by screen size", %{conn: conn, site: site} dosite_import = insert(:site_import, site: site)