Skip to content

Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.

Goal join performance (#6770)

Uku Tahtd28d181a696b
diff --git a/lib/plausible/google/search_console/filters.ex b/lib/plausible/google/search_console/filters.ex
index 1ea9c4a8fa..f61005a0c4 100644
--- a/lib/plausible/google/search_console/filters.ex
+++ b/lib/plausible/google/search_console/filters.ex
@@ -1,6 +1,6 @@
defmodule Plausible.Google.SearchConsole.Filters do
@moduledoc false
- import Plausible.Stats.Filters.Utils, only: [page_regex: 1]
+ import Plausible.Stats.Filters.Utils, only: [page_regex: 1, split_page_patterns: 1]
def transform(property, plausible_filters, search) do
gsc_filters =
@@ -35,7 +35,9 @@ defmodule Plausible.Google.SearchConsole.Filters do
defp transform_filter(property, [:matches_wildcard, "visit:entry_page", pages | _])
when is_list(pages) do
expression =
- Enum.map_join(pages, "|", fn page -> page_regex(property_url(property, page)) end)
+ pages
+ |> Enum.flat_map(&split_page_patterns/1)
+ |> Enum.map_join("|", fn page -> page_regex(property_url(property, page)) end)
%{dimension: "page", operator: "includingRegex", expression: expression}
end
diff --git a/lib/plausible/stats/filters/utils.ex b/lib/plausible/stats/filters/utils.ex
index 48dd501a87..0a2747df4c 100644
--- a/lib/plausible/stats/filters/utils.ex
+++ b/lib/plausible/stats/filters/utils.ex
@@ -1,14 +1,23 @@
defmodule Plausible.Stats.Filters.Utils do
@moduledoc false
+ def uses_page_regex?(path), do: String.contains?(path, ["*", "|"])
+
+ # Preserve escaped pipes until each alternative is converted to a regex.
+ def split_page_patterns(expr), do: String.split(expr, ~r/(?<!\\)\|/)
+
def page_regex(expr) do
escaped =
expr
- |> Regex.escape()
- |> String.replace("\\|", "|")
- |> String.replace("\\*\\*", ".*")
- |> String.replace("\\*", ".*")
+ |> split_page_patterns()
+ |> Enum.map_join("|", fn pattern ->
+ pattern
+ |> String.replace("\\|", "|")
+ |> Regex.escape()
+ |> String.replace("\\*\\*", ".*")
+ |> String.replace("\\*", ".*")
+ end)
- "^#{escaped}$"
+ "^(?:#{escaped})$"
end
end
diff --git a/lib/plausible/stats/goals.ex b/lib/plausible/stats/goals.ex
index 7733233793..02b93015c9 100644
--- a/lib/plausible/stats/goals.ex
+++ b/lib/plausible/stats/goals.ex
@@ -4,7 +4,7 @@ defmodule Plausible.Stats.Goals do
"""
import Ecto.Query
- import Plausible.Stats.Filters.Utils, only: [page_regex: 1]
+ import Plausible.Stats.Filters.Utils, only: [page_regex: 1, uses_page_regex?: 1]
alias Plausible.Stats.Filters
@@ -63,60 +63,39 @@ defmodule Plausible.Stats.Goals do
end)
end
- @type goal_join_data() :: %{
- indices: [non_neg_integer()],
- event_names_by_type: [String.t()],
- page_regexes: [String.t()],
- scroll_thresholds: [non_neg_integer()],
- custom_props_keys: [[String.t()]],
- custom_props_values: [[String.t()]]
- }
+ @doc """
+ Assigns breakdown goals 1-based indices, matching ClickHouse array positions.
+ Assign indices before partitioning or excluding goals so native and imported
+ queries use the same identifiers, decoded by `goal_at_index/2`.
+ """
+ @spec indexed_goals(Plausible.Stats.Query.t()) :: [{Plausible.Goal.t(), pos_integer()}]
+ def indexed_goals(query) do
+ Enum.with_index(query.preloaded_goals.matching_toplevel_filters, 1)
+ end
+
+ @doc """
+ Retrieves a breakdown goal using the 1-based index assigned by `indexed_goals/1`.
+ """
+ @spec goal_at_index(Plausible.Stats.Query.t(), pos_integer()) :: Plausible.Goal.t() | nil
+ def goal_at_index(query, index) do
+ Enum.at(query.preloaded_goals.matching_toplevel_filters, index - 1)
+ end
@type goal_join_imported_data() :: %{
- indices: [non_neg_integer()],
+ indices: [pos_integer()],
types: [String.t()],
event_names: [String.t()],
page_regexes: [String.t()]
}
- @doc """
- Returns data needed to perform a GROUP BY on goals in an ecto query.
- """
- @spec goal_join_data(Plausible.Stats.Query.t()) :: goal_join_data()
- def goal_join_data(query) do
- {goals, indices} =
- query.preloaded_goals.matching_toplevel_filters
- |> Enum.with_index(1)
- |> Enum.unzip()
-
- {custom_props_keys, custom_props_values} =
- goals
- |> Enum.map(fn goal -> Enum.unzip(goal.custom_props) end)
- |> Enum.unzip()
-
- %{
- indices: indices,
- event_names_by_type:
- Enum.map(goals, fn goal -> event_name_by_type(Plausible.Goal.type(goal), goal) end),
- # Event goals are considered to match everything for the sake of efficient queries in query_builder.ex
- # See also Plausible.Stats.SQL.Expression.event_goal_join/1
- page_regexes:
- Enum.map(goals, fn goal -> page_regex_for_goal(Plausible.Goal.type(goal), goal) end),
- scroll_thresholds: Enum.map(goals, fn goal -> goal.scroll_threshold end),
- custom_props_keys: custom_props_keys,
- custom_props_values: custom_props_values
- }
- end
-
@doc """
Returns data needed to perform a GROUP BY on goals in an imported data query.
"""
@spec goal_join_imported_data(Plausible.Stats.Query.t()) :: goal_join_imported_data()
def goal_join_imported_data(query) do
{goals, indices} =
- query.preloaded_goals.matching_toplevel_filters
- # Keep indices aligned with the complete preloaded goal list.
- |> Enum.with_index(1)
+ query
+ |> indexed_goals()
|> Enum.reject(fn {goal, _idx} -> Plausible.Goal.has_custom_props?(goal) end)
|> Enum.unzip()
@@ -124,18 +103,28 @@ defmodule Plausible.Stats.Goals do
indices: indices,
types: Enum.map(goals, fn goal -> goal |> Plausible.Goal.type() |> to_string() end),
event_names: Enum.map(goals, fn goal -> to_string(goal.event_name) end),
- page_regexes:
- Enum.map(goals, fn goal -> page_regex_for_goal(Plausible.Goal.type(goal), goal) end)
+ page_regexes: Enum.map(goals, &pathname_regex/1)
}
end
- defp event_name_by_type(:event, goal), do: goal.event_name
- defp event_name_by_type(:page, _goal), do: "pageview"
- defp event_name_by_type(:scroll, _goal), do: "engagement"
+ def expected_event_name(goal) do
+ case Plausible.Goal.type(goal) do
+ :event -> goal.event_name
+ :page -> "pageview"
+ :scroll -> "engagement"
+ end
+ end
- defp page_regex_for_goal(:event, _goal), do: ".?"
- defp page_regex_for_goal(:page, goal), do: Filters.Utils.page_regex(goal.page_path)
- defp page_regex_for_goal(:scroll, goal), do: Filters.Utils.page_regex(goal.page_path)
+ @doc """
+ The regex an event's pathname needs to match to complete the goal. Event goals
+ match any pathname.
+ """
+ def pathname_regex(goal) do
+ case Plausible.Goal.type(goal) do
+ :event -> ".?"
+ _page_or_scroll -> page_regex(goal.page_path)
+ end
+ end
def toplevel_scroll_goal_filters?(query) do
goal_filters? =
@@ -259,7 +248,7 @@ defmodule Plausible.Stats.Goals do
defp page_path_condition(page_path, imported?) do
db_field = page_path_db_field(imported?)
- if String.contains?(page_path, "*") do
+ if uses_page_regex?(page_path) do
dynamic([e], fragment("match(?, ?)", field(e, ^db_field), ^page_regex(page_path)))
else
dynamic([e], field(e, ^db_field) == ^page_path)
diff --git a/lib/plausible/stats/query_runner.ex b/lib/plausible/stats/query_runner.ex
index 6262890992..2b509c982e 100644
--- a/lib/plausible/stats/query_runner.ex
+++ b/lib/plausible/stats/query_runner.ex
@@ -210,8 +210,7 @@ defmodule Plausible.Stats.QueryRunner do
defp get_dimension_goal(entry, query) do
goal_index = Map.get(entry, Util.shortname(query, "event:goal"))
- query.preloaded_goals.matching_toplevel_filters
- |> Enum.at(goal_index - 1)
+ Plausible.Stats.Goals.goal_at_index(query, goal_index)
end
defp merge_with_comparison_results(results_list, runner) do
diff --git a/lib/plausible/stats/sql/expression.ex b/lib/plausible/stats/sql/expression.ex
index 142ad71f1d..432e836c40 100644
--- a/lib/plausible/stats/sql/expression.ex
+++ b/lib/plausible/stats/sql/expression.ex
@@ -164,6 +164,13 @@ defmodule Plausible.Stats.SQL.Expression do
def select_dimension(q, key, "event:name", _table, _query),
do: select_merge_as(q, [t], %{key => t.name})
+ # One row per goal the event completes. Events completing no goal drop out.
+ def select_dimension(q, key, "event:goal", :events, query) do
+ select_merge_as(q, [t], %{
+ key => fragment("arrayJoin(?)", ^SQL.GoalMatching.completed_goals(query))
+ })
+ end
+
def select_dimension(q, key, "event:page", _table, _query),
do: select_merge_as(q, [t], %{key => t.pathname})
@@ -502,82 +509,4 @@ defmodule Plausible.Stats.SQL.Expression do
def session_metric(:percentage, _query), do: %{}
def session_metric(:conversion_rate, _query), do: %{}
def session_metric(:group_conversion_rate, _query), do: %{}
-
- @doc """
- The fragment matches events to goals by:
- 1. Checking if the pathname matches the goal's page regex pattern
- 2. Verifying the event name matches the expected name for the goal type
- 3. Validating scroll depth is within threshold (for scroll goals)
- 4. Ensuring all custom properties match (if any are defined on the goal)
-
- Returns an array of goal indices that the event matches.
- """
- defmacro event_goal_join(goal_join_data) do
- quote do
- fragment(
- """
- arrayIntersect(
- multiMatchAllIndices(?, ?),
- arrayMap(
- (expected_name, threshold, index, custom_props_keys, custom_props_values) -> if(
- expected_name = ? and ? between threshold and 100 and
- (empty(custom_props_keys) OR arrayAll((k, v) -> ?[indexOf(?, k)] = v, custom_props_keys, custom_props_values)),
- index, -1
- ),
- ?,
- ?,
- ?,
- ?,
- ?
- )
- )
- """,
- e.pathname,
- type(^unquote(goal_join_data).page_regexes, {:array, :string}),
- e.name,
- e.scroll_depth,
- field(e, :"meta.value"),
- field(e, :"meta.key"),
- type(^unquote(goal_join_data).event_names_by_type, {:array, :string}),
- type(^unquote(goal_join_data).scroll_thresholds, {:array, :integer}),
- type(^unquote(goal_join_data).indices, {:array, :integer}),
- type(^unquote(goal_join_data).custom_props_keys, {:array, {:array, :string}}),
- type(^unquote(goal_join_data).custom_props_values, {:array, {:array, :string}})
- )
- end
- end
-
- @doc """
- Optimized variant of `event_goal_join/1` for use when no goals have custom
- property filters. Omits all references to `meta.key` and `meta.value`,
- preventing ClickHouse from reading those expensive Array(String) columns at
- scan time.
- """
- defmacro event_goal_join_no_props(goal_join_data) do
- quote do
- fragment(
- """
- arrayIntersect(
- multiMatchAllIndices(?, ?),
- arrayMap(
- (expected_name, threshold, index) -> if(
- expected_name = ? and ? between threshold and 100,
- index, -1
- ),
- ?,
- ?,
- ?
- )
- )
- """,
- e.pathname,
- type(^unquote(goal_join_data).page_regexes, {:array, :string}),
- e.name,
- e.scroll_depth,
- type(^unquote(goal_join_data).event_names_by_type, {:array, :string}),
- type(^unquote(goal_join_data).scroll_thresholds, {:array, :integer}),
- type(^unquote(goal_join_data).indices, {:array, :integer})
- )
- end
- end
end
diff --git a/lib/plausible/stats/sql/goal_matching.ex b/lib/plausible/stats/sql/goal_matching.ex
new file mode 100644
index 0000000000..cade41d370
--- /dev/null
+++ b/lib/plausible/stats/sql/goal_matching.ex
@@ -0,0 +1,211 @@
+defmodule Plausible.Stats.SQL.GoalMatching do
+ @moduledoc """
+ Builds the matching goal indices for an `event:goal` breakdown.
+
+ Event names and exact page paths use lookups when no custom properties
+ are required. Other goals use regex matching and additional filters.
+
+ Goal indices are assigned by `Plausible.Stats.Goals.indexed_goals/1` and
+ decoded by `Plausible.Stats.Goals.goal_at_index/2` in QueryRunner.
+ """
+
+ import Ecto.Query
+ import Plausible.Stats.Filters.Utils, only: [uses_page_regex?: 1]
+
+ alias Plausible.Goal
+ alias Plausible.Stats.Goals
+
+ @typep keyed_goal() :: {String.t(), pos_integer()}
+
+ @doc """
+ An `Array(UInt64)` expression of the indices of the goals the event completes.
+ """
+ def completed_goals(query) do
+ %{by_name: by_name, by_pathname: by_pathname, scanned: scanned} =
+ partition(Goals.indexed_goals(query))
+
+ [
+ lookup(dynamic([e], e.name), by_name),
+ lookup_by_pathname(by_pathname),
+ scan(scanned)
+ ]
+ |> Enum.reject(&is_nil/1)
+ |> concat()
+ end
+
+ # Groups goals, with their indices, by how they are matched. The unique indexes
+ # on goals keep each event name and exact page path to one goal without custom
+ # props, so a lookup by either finds at most one goal.
+ defp partition(indexed_goals) do
+ classified =
+ for {goal, index} <- indexed_goals, do: {matching_strategy(goal), goal, index}
+
+ %{
+ by_name: for({{:name, name}, _goal, index} <- classified, do: {name, index}),
+ by_pathname: for({{:pathname, path}, _goal, index} <- classified, do: {path, index}),
+ scanned: for({:scan, goal, index} <- classified, do: {goal, index})
+ }
+ end
+
+ defp matching_strategy(goal) do
+ case {Goal.type(goal), Goal.has_custom_props?(goal)} do
+ {:event, false} ->
+ {:name, goal.event_name}
+
+ {:event, true} ->
+ :scan
+
+ {:page, false} ->
+ if uses_page_regex?(goal.page_path), do: :scan, else: {:pathname, goal.page_path}
+
+ {:page, true} ->
+ :scan
+
+ {:scroll, _has_custom_props?} ->
+ :scan
+ end
+ end
+
+ # Look up the goal index matching `event_field`. transform() is a hash lookup,
+ # while has() on a constant array would scan it on every row.
+ @spec lookup(Ecto.Query.dynamic_expr(), [keyed_goal()]) :: Ecto.Query.dynamic_expr() | nil
+ defp lookup(_event_field, []), do: nil
+
+ defp lookup(event_field, keyed_goals) do
+ {keys, goal_indices} = Enum.unzip(keyed_goals)
+
+ dynamic(
+ fragment(
+ "arrayFilter(i -> i != 0, [transform(?, ?, CAST(? AS Array(UInt64)), toUInt64(0))])",
+ ^event_field,
+ type(^keys, {:array, :string}),
+ type(^goal_indices, {:array, :integer})
+ )
+ )
+ end
+
+ defp lookup_by_pathname([]), do: nil
+
+ defp lookup_by_pathname(keyed_goals) do
+ goal_indices_expr = lookup(dynamic([e], e.pathname), keyed_goals)
+
+ dynamic(
+ [e],
+ fragment("if(?, ?, CAST([] AS Array(UInt64)))", e.name == "pageview", ^goal_indices_expr)
+ )
+ end
+
+ defp scan([]), do: nil
+
+ # Regex matches are 1-based positions within this scanned subset of goals.
+ defp scan(indexed_goals) do
+ {goals, goal_indices} = Enum.unzip(indexed_goals)
+ expected_names = Enum.map(goals, &Goals.expected_event_name/1)
+ distinct_names = Enum.uniq(expected_names)
+
+ goal_indices_expr =
+ dynamic(
+ [e],
+ fragment(
+ "multiMatchAllIndices(?, ?)",
+ e.pathname,
+ type(^Enum.map(goals, &Goals.pathname_regex/1), {:array, :string})
+ )
+ )
+ |> keep_expected_events(goals, expected_names, distinct_names)
+ |> keep_matching_custom_props(goals)
+ |> to_goal_indices(goal_indices)
+
+ dynamic(
+ [e],
+ fragment(
+ "if(?, ?, CAST([] AS Array(UInt64)))",
+ e.name in ^distinct_names,
+ ^goal_indices_expr
+ )
+ )
+ end
+
+ # The outer name guard is sufficient when all scanned goals expect the same name.
+ # Omit the depth check when there are no scroll goals.
+ # The upper bound excludes the missing-depth sentinel, 255.
+ defp keep_expected_events(positions, goals, expected_names, distinct_names) do
+ name_check =
+ if length(distinct_names) > 1 do
+ dynamic(
+ [e],
+ fragment("?[goal_pos] = ?", type(^expected_names, {:array, :string}), e.name)
+ )
+ end
+
+ threshold_check =
+ if Enum.any?(goals, &(Goal.type(&1) == :scroll)) do
+ thresholds = Enum.map(goals, & &1.scroll_threshold)
+
+ dynamic(
+ [e],
+ fragment(
+ "(? BETWEEN ?[goal_pos] AND 100)",
+ e.scroll_depth,
+ type(^thresholds, {:array, :integer})
+ )
+ )
+ end
+
+ case Enum.reject([name_check, threshold_check], &is_nil/1) do
+ [] ->
+ positions
+
+ checks ->
+ condition = chain_conditions(checks)
+ dynamic(fragment("arrayFilter(goal_pos -> ?, ?)", ^condition, ^positions))
+ end
+ end
+
+ # Check custom properties only for positions that survived the earlier filters.
+ # Omit meta.* references entirely when no scanned goal has custom properties.
+ defp keep_matching_custom_props(positions, goals) do
+ if Enum.any?(goals, &Goal.has_custom_props?/1) do
+ {keys, values} =
+ goals
+ |> Enum.map(&Enum.unzip(&1.custom_props))
+ |> Enum.unzip()
+
+ props_check =
+ dynamic(
+ [e],
+ fragment(
+ "(empty(?[goal_pos]) OR arrayAll((k, v) -> ?[indexOf(?, k)] = v, ?[goal_pos], ?[goal_pos]))",
+ type(^keys, {:array, {:array, :string}}),
+ field(e, :"meta.value"),
+ field(e, :"meta.key"),
+ type(^keys, {:array, {:array, :string}}),
+ type(^values, {:array, {:array, :string}})
+ )
+ )
+
+ dynamic(fragment("arrayFilter(goal_pos -> ?, ?)", ^props_check, ^positions))
+ else
+ positions
+ end
+ end
+
+ defp chain_conditions(conditions) do
+ Enum.reduce(conditions, fn condition, acc -> dynamic(^acc and ^condition) end)
+ end
+
+ defp to_goal_indices(positions, goal_indices) do
+ dynamic(
+ fragment(
+ "arrayMap(goal_pos -> CAST(? AS Array(UInt64))[goal_pos], ?)",
+ type(^goal_indices, {:array, :integer}),
+ ^positions
+ )
+ )
+ end
+
+ defp concat([]), do: dynamic(fragment("CAST([] AS Array(UInt64))"))
+ defp concat([goal_indices_expr]), do: goal_indices_expr
+ defp concat([a, b]), do: dynamic(fragment("arrayConcat(?, ?)", ^a, ^b))
+ defp concat([a, b, c]), do: dynamic(fragment("arrayConcat(?, ?, ?)", ^a, ^b, ^c))
+end
diff --git a/lib/plausible/stats/sql/query_builder.ex b/lib/plausible/stats/sql/query_builder.ex
index 9e905e63d1..106871a406 100644
--- a/lib/plausible/stats/sql/query_builder.ex
+++ b/lib/plausible/stats/sql/query_builder.ex
@@ -12,8 +12,6 @@ defmodule Plausible.Stats.SQL.QueryBuilder do
alias Plausible.Stats.SQL.Expression
alias Plausible.Stats.Legacy.TimeOnPage
- require Plausible.Stats.SQL.Expression
-
def build(query, site) do
query
|> QueryOptimizer.split()
@@ -180,32 +178,6 @@ defmodule Plausible.Stats.SQL.QueryBuilder do
|> select_merge_as([..., meta], %{shortname(query, dimension) => meta.key})
end
- defp dimension_group_by(q, :events, query, "event:goal" = dimension) do
- goal_join_data = Plausible.Stats.Goals.goal_join_data(query)
-
- if Enum.all?(goal_join_data.custom_props_keys, &Enum.empty?/1) do
- from(e in q,
- join: goal in Expression.event_goal_join_no_props(goal_join_data),
- hints: "ARRAY",
- on: true,
- select_merge: %{
- ^shortname(query, dimension) => fragment("?", goal)
- },
- group_by: goal
- )
- else
- from(e in q,
- join: goal in Expression.event_goal_join(goal_join_data),
- hints: "ARRAY",
- on: true,
- select_merge: %{
- ^shortname(query, dimension) => fragment("?", goal)
- },
- group_by: goal
- )
- end
- end
-
defp dimension_group_by(q, table, query, dimension) do
key = shortname(query, dimension)
diff --git a/lib/plausible/stats/sql/where_builder.ex b/lib/plausible/stats/sql/where_builder.ex
index 18353d34cb..1670493c54 100644
--- a/lib/plausible/stats/sql/where_builder.ex
+++ b/lib/plausible/stats/sql/where_builder.ex
@@ -45,13 +45,7 @@ defmodule Plausible.Stats.SQL.WhereBuilder do
defp goal_event_names(goals) do
goals
- |> Enum.map(fn goal ->
- case Plausible.Goal.type(goal) do
- :event -> goal.event_name
- :page -> "pageview"
- :scroll -> "engagement"
- end
- end)
+ |> Enum.map(&Plausible.Stats.Goals.expected_event_name/1)
|> Enum.uniq()
end
diff --git a/test/plausible/google/search_console/filters_test.exs b/test/plausible/google/search_console/filters_test.exs
index 12924cf7ec..e61c480b98 100644
--- a/test/plausible/google/search_console/filters_test.exs
+++ b/test/plausible/google/search_console/filters_test.exs
@@ -35,7 +35,7 @@ defmodule Plausible.Google.SearchConsole.FiltersTest do
%{
dimension: "page",
operator: "includingRegex",
- expression: "^https://plausible\\.io.*page.*$"
+ expression: "^(?:https://plausible\\.io.*page.*)$"
}
]
}
@@ -62,6 +62,27 @@ defmodule Plausible.Google.SearchConsole.FiltersTest do
]
end
+ test "prefixes and anchors each alternative in wildcard page filters" do
+ filters = [[:matches_wildcard, "visit:entry_page", ["/pricing*|/plans"]]]
+
+ {:ok, [%{filters: [%{expression: expression}]}]} =
+ Filters.transform("sc-domain:plausible.io", filters, "")
+
+ regex = Regex.compile!(expression)
+
+ for url <- ["https://plausible.io/pricing/team", "https://plausible.io/plans"] do
+ assert Regex.match?(regex, url)
+ end
+
+ for url <- [
+ "https://plausible.io/legacy/plans",
+ "https://plausible.io/plans/more",
+ "https://other.io/plans"
+ ] do
+ refute Regex.match?(regex, url)
+ end
+ end
+
test "transforms matches multiple page filter" do
filters = [
[:matches_wildcard, "visit:entry_page", ["/pageA*", "/pageB*"]]
@@ -75,13 +96,28 @@ defmodule Plausible.Google.SearchConsole.FiltersTest do
%{
dimension: "page",
operator: "includingRegex",
- expression: "^https://plausible\\.io/pageA.*$|^https://plausible\\.io/pageB.*$"
+ expression:
+ "^(?:https://plausible\\.io/pageA.*)$|^(?:https://plausible\\.io/pageB.*)$"
}
]
}
]
end
+ test "preserves escaped pipes in wildcard page filters" do
+ filters = [[:matches_wildcard, "visit:entry_page", ["/post*\\|1|/plans"]]]
+
+ {:ok, [%{filters: [%{expression: expression}]}]} =
+ Filters.transform("sc-domain:plausible.io", filters, "")
+
+ regex = Regex.compile!(expression)
+
+ assert Regex.match?(regex, "https://plausible.io/post|1")
+ assert Regex.match?(regex, "https://plausible.io/plans")
+ refute Regex.match?(regex, "https://plausible.io/post|2")
+ refute Regex.match?(regex, "https://plausible.io/1")
+ end
+
test "transforms event:page exactly like visit:entry_page" do
filters = [
[:matches_wildcard, "event:page", ["/pageA*", "/pageB*"]]
@@ -95,7 +131,8 @@ defmodule Plausible.Google.SearchConsole.FiltersTest do
%{
dimension: "page",
operator: "includingRegex",
- expression: "^https://plausible\\.io/pageA.*$|^https://plausible\\.io/pageB.*$"
+ expression:
+ "^(?:https://plausible\\.io/pageA.*)$|^(?:https://plausible\\.io/pageB.*)$"
}
]
}
@@ -178,7 +215,7 @@ defmodule Plausible.Google.SearchConsole.FiltersTest do
%{
dimension: "page",
operator: "includingRegex",
- expression: "^https://plausible\\.io.*web\\-analytics.*$"
+ expression: "^(?:https://plausible\\.io.*web\\-analytics.*)$"
},
%{dimension: "country", operator: "includingRegex", expression: "EST|POL"}
]
diff --git a/test/plausible/stats/filters_test.exs b/test/plausible/stats/filters_test.exs
index c239bfa156..d876f926b5 100644
--- a/test/plausible/stats/filters_test.exs
+++ b/test/plausible/stats/filters_test.exs
@@ -5,6 +5,53 @@ defmodule Plausible.Stats.FiltersTest do
doctest Plausible.Stats.Filters
doctest Plausible.Stats.Filters.Utils
+ describe "page regex" do
+ test "matches escaped pipes literally" do
+ regex = Regex.compile!(Filters.Utils.page_regex("/post\\|1"))
+
+ assert Regex.match?(regex, "/post|1")
+ refute Regex.match?(regex, "/post")
+ refute Regex.match?(regex, "1")
+ refute Regex.match?(regex, "/post|1/more")
+ end
+
+ test "anchors every alternative to the whole path" do
+ regex = Regex.compile!(Filters.Utils.page_regex("/pricing|/plans"))
+
+ for path <- ["/pricing", "/plans"], do: assert(Regex.match?(regex, path))
+
+ for path <- ["/pricing/enterprise", "/legacy/plans", "/pricing|/plans", "/about"] do
+ refute Regex.match?(regex, path)
+ end
+ end
+
+ test "matches * within a whole path" do
+ regex = Regex.compile!(Filters.Utils.page_regex("/pricing/*"))
+
+ assert Regex.match?(regex, "/pricing/")
+ assert Regex.match?(regex, "/pricing/team")
+ assert Regex.match?(regex, "/pricing/team/monthly")
+ refute Regex.match?(regex, "/legacy/pricing/team")
+ refute Regex.match?(regex, "/pricing")
+ end
+
+ test "matches ** within a whole path" do
+ regex = Regex.compile!(Filters.Utils.page_regex("/plans/**"))
+
+ assert Regex.match?(regex, "/plans/")
+ assert Regex.match?(regex, "/plans/team/monthly")
+ refute Regex.match?(regex, "/legacy/plans/team")
+ end
+
+ test "escapes other regex syntax" do
+ regex = Regex.compile!(Filters.Utils.page_regex("/file.json"))
+
+ assert Regex.match?(regex, "/file.json")
+ refute Regex.match?(regex, "/fileXjson")
+ refute Regex.match?(regex, "/file.json/more")
+ end
+ end
+
def assert_parsed(input, expected_output) do
assert Filters.parse(input) == expected_output
end
diff --git a/test/plausible_web/controllers/api/external_stats_controller/query_goal_custom_props_test.exs b/test/plausible_web/controllers/api/external_stats_controller/query_goal_custom_props_test.exs
index 9201731780..058b185b6e 100644
--- a/test/plausible_web/controllers/api/external_stats_controller/query_goal_custom_props_test.exs
+++ b/test/plausible_web/controllers/api/external_stats_controller/query_goal_custom_props_test.exs
@@ -366,6 +366,55 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryGoalCustomPropsTest do
refute Enum.find(results, &(&1["dimensions"] == ["Button Click"]))
end
+
+ test "custom event goals are not completed by pageviews with the same custom props", %{
+ conn: conn,
+ site: site
+ } do
+ {:ok, _} =
+ Goals.create(site, %{
+ "event_name" => "Signup",
+ "display_name" => "Premium Signup",
+ "custom_props" => %{"plan" => "premium"}
+ })
+
+ {:ok, _} = Goals.create(site, %{"page_path" => "/blog/**"})
+
+ populate_stats(site, [
+ build(:pageview,
+ pathname: "/blog/post",
+ "meta.key": ["plan"],
+ "meta.value": ["premium"],
+ user_id: 1
+ ),
+ build(:event,
+ name: "Signup",
+ "meta.key": ["plan"],
+ "meta.value": ["premium"],
+ user_id: 2
+ ),
+ build(:event,
+ name: "Signup",
+ "meta.key": ["plan"],
+ "meta.value": ["premium"],
+ user_id: 3
+ ),
+ build(:event, name: "Signup", "meta.key": ["plan"], "meta.value": ["free"], user_id: 4)
+ ])
+
+ conn =
+ post(conn, "/api/v2/query", %{
+ "site_id" => site.domain,
+ "date_range" => "all",
+ "metrics" => ["visitors"],
+ "dimensions" => ["event:goal"]
+ })
+
+ assert json_response(conn, 200)["results"] == [
+ %{"dimensions" => ["Premium Signup"], "metrics" => [2]},
+ %{"dimensions" => ["Visit /blog/**"], "metrics" => [1]}
+ ]
+ end
end
describe "goals with custom props and imported data" do
diff --git a/test/plausible_web/controllers/api/external_stats_controller/query_goal_dimension_test.exs b/test/plausible_web/controllers/api/external_stats_controller/query_goal_dimension_test.exs
index 9deb1ee2ce..b50d4b37db 100644
--- a/test/plausible_web/controllers/api/external_stats_controller/query_goal_dimension_test.exs
+++ b/test/plausible_web/controllers/api/external_stats_controller/query_goal_dimension_test.exs
@@ -161,6 +161,77 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryGoalDimensionTest do
assert length(results) == Plausible.Goals.max_goals_per_site()
end
+
+ test "returns pageview goals with | between alternative paths", %{conn: conn, site: site} do
+ insert(:goal, %{site: site, page_path: "/pricing|/plans"})
+
+ populate_stats(site, [
+ build(:pageview, pathname: "/pricing", user_id: 1),
+ build(:pageview, pathname: "/plans", user_id: 2),
+ build(:pageview, pathname: "/about", user_id: 3),
+ build(:pageview, pathname: "/pricing/enterprise", user_id: 4),
+ build(:pageview, pathname: "/legacy/plans", user_id: 5),
+ build(:pageview, pathname: "/pricing|/plans", user_id: 6),
+ build(:event, name: "Signup", pathname: "/pricing", user_id: 7)
+ ])
+
+ conn =
+ post(conn, "/api/v2/query", %{
+ "site_id" => site.domain,
+ "date_range" => "all",
+ "metrics" => ["visitors"],
+ "dimensions" => ["event:goal"]
+ })
+
+ assert json_response(conn, 200)["results"] == [
+ %{"dimensions" => ["Visit /pricing|/plans"], "metrics" => [2]}
+ ]
+
+ conn =
+ post(conn, "/api/v2/query", %{
+ "site_id" => site.domain,
+ "date_range" => "all",
+ "metrics" => ["visitors"],
+ "filters" => [["is", "event:goal", ["Visit /pricing|/plans"]]]
+ })
+
+ assert json_response(conn, 200)["results"] == [
+ %{"dimensions" => [], "metrics" => [2]}
+ ]
+ end
+
+ test "counts pageview goals on path P and custom event goals that happen on path P separately",
+ %{
+ conn: conn,
+ site: site
+ } do
+ insert(:goal, %{site: site, event_name: "Signup"})
+ insert(:goal, %{site: site, page_path: "/blog/**"})
+ insert(:goal, %{site: site, page_path: "/pricing"})
+
+ populate_stats(site, [
+ build(:pageview, pathname: "/blog/a", user_id: 1),
+ build(:pageview, pathname: "/blog/b", user_id: 2),
+ build(:pageview, pathname: "/pricing", user_id: 3),
+ build(:event, name: "Signup", pathname: "/blog/a", user_id: 4),
+ build(:event, name: "Signup", pathname: "/pricing", user_id: 5),
+ build(:event, name: "Signup", pathname: "/", user_id: 6)
+ ])
+
+ conn =
+ post(conn, "/api/v2/query", %{
+ "site_id" => site.domain,
+ "date_range" => "all",
+ "metrics" => ["visitors"],
+ "dimensions" => ["event:goal"]
+ })
+
+ assert json_response(conn, 200)["results"] == [
+ %{"dimensions" => ["Signup"], "metrics" => [3]},
+ %{"dimensions" => ["Visit /blog/**"], "metrics" => [2]},
+ %{"dimensions" => ["Visit /pricing"], "metrics" => [1]}
+ ]
+ end
end
describe "page scroll goals" do
diff --git a/test/plausible_web/controllers/api/external_stats_controller/query_imported_test.exs b/test/plausible_web/controllers/api/external_stats_controller/query_imported_test.exs
index 9ed544765c..618cfcde5b 100644
--- a/test/plausible_web/controllers/api/external_stats_controller/query_imported_test.exs
+++ b/test/plausible_web/controllers/api/external_stats_controller/query_imported_test.exs
@@ -588,6 +588,45 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryImportedTest do
end
describe "imported data" do
+ test "aligns goal filters and breakdowns for alternative paths with imported pages", %{
+ conn: conn,
+ site: site
+ } do
+ insert(:goal, site: site, page_path: "/pricing|/plans")
+ site_import = insert(:site_import, site: site)
+
+ imported_pages =
+ for path <- ["/pricing", "/plans", "/pricing/more", "/legacy/plans", "/pricing|/plans"] do
+ build(:imported_pages, page: path, visitors: 2, pageviews: 2, date: ~D[2021-01-01])
+ end
+
+ populate_stats(site, site_import.id, [
+ build(:pageview, pathname: "/pricing", timestamp: ~N[2021-01-01 00:00:01])
+ | imported_pages
+ ])
+
+ for {dimensions, filters} <- [
+ {["event:goal"], []},
+ {[], [["is", "event:goal", ["Visit /pricing|/plans"]]]}
+ ] do
+ conn =
+ post(conn, "/api/v2/query", %{
+ "site_id" => site.domain,
+ "date_range" => "all",
+ "metrics" => ["visitors"],
+ "dimensions" => dimensions,
+ "filters" => filters,
+ "include" => %{"imports" => true}
+ })
+
+ expected_dimensions = if dimensions == [], do: [], else: ["Visit /pricing|/plans"]
+
+ assert json_response(conn, 200)["results"] == [
+ %{"dimensions" => expected_dimensions, "metrics" => [5]}
+ ]
+ end
+ end
+
test "returns screen sizes breakdown when filtering by screen size", %{conn: conn, site: site} do
site_import = insert(:site_import, site: site)