Skip to content

Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.

Make exploration implicit wildcard togglable (#6284)

Adrian Gruntkowski74f3eef39f00
diff --git a/extra/lib/plausible/stats/exploration.ex b/extra/lib/plausible/stats/exploration.ex
index 4fa1263bc9..1a9b551d4f 100644
--- a/extra/lib/plausible/stats/exploration.ex
+++ b/extra/lib/plausible/stats/exploration.ex
@@ -66,7 +66,12 @@ defmodule Plausible.Stats.Exploration do
@max_steps 20
@max_candidates 20
- @next_steps_defaults [search_term: "", direction: :forward, max_candidates: 10]
+ @next_steps_defaults [
+ search_term: "",
+ direction: :forward,
+ max_candidates: 10,
+ include_wildcard?: true
+ ]
@spec max_steps() :: pos_integer()
def max_steps, do: @max_steps
@@ -84,10 +89,11 @@ defmodule Plausible.Stats.Exploration do
direction = Keyword.fetch!(opts, :direction)
search_term = Keyword.fetch!(opts, :search_term)
max_candidates = min(Keyword.fetch!(opts, :max_candidates), @max_candidates)
+ include_wilcard? = Keyword.fetch!(opts, :include_wildcard?)
query
|> Base.base_event_query()
- |> next_steps_query(journey, search_term, direction, max_candidates)
+ |> next_steps_query(journey, search_term, direction, max_candidates, include_wilcard?)
# We pass the query struct to record query metadata for
# the CH debug console.
|> ClickhouseRepo.all(query: query)
@@ -133,6 +139,9 @@ defmodule Plausible.Stats.Exploration do
* `:max_steps` - maximum number of funnel steps to build (default: `6`)
* `:max_candidates` - passed to `next_steps/3`, limiting
how many candidate next steps are fetched per step (default: `10`)
+ * `:include_wildcard?` - passed to `next_steps/3`, deciding whether
+ to include implicit wildcard pathnames in suggestions or not
+ (default: true)
"""
@spec interesting_funnel(Query.t(), keyword()) ::
{:ok, [funnel_step()]} | {:error, :not_found}
@@ -140,23 +149,34 @@ defmodule Plausible.Stats.Exploration do
max_steps = min(Keyword.get(opts, :max_steps, 6), @max_steps)
max_candidates = min(Keyword.get(opts, :max_candidates, 10), @max_candidates)
- case build_interesting_journey(query, max_steps, max_candidates) do
+ include_wildcard? =
+ Keyword.get(
+ opts,
+ :include_wildcard?,
+ Keyword.fetch!(@next_steps_defaults, :include_wildcard?)
+ )
+
+ case build_interesting_journey(query, max_steps, max_candidates, include_wildcard?) do
[] -> {:error, :not_found}
journey -> journey_funnel(query, journey)
end
end
- defp build_interesting_journey(query, max_steps, max_candidates) do
- do_build_journey(query, [], MapSet.new(), max_steps, max_candidates)
+ defp build_interesting_journey(query, max_steps, max_candidates, include_wildcard?) do
+ do_build_journey(query, [], MapSet.new(), max_steps, max_candidates, include_wildcard?)
end
- defp do_build_journey(_query, journey, _seen, max_steps, _max_candidates)
+ defp do_build_journey(_query, journey, _seen, max_steps, _max_candidates, _include_wildcard?)
when length(journey) >= max_steps do
journey
end
- defp do_build_journey(query, journey, seen, max_steps, max_candidates) do
- {:ok, candidates} = next_steps(query, journey, max_candidates: max_candidates)
+ defp do_build_journey(query, journey, seen, max_steps, max_candidates, include_wildcard?) do
+ {:ok, candidates} =
+ next_steps(query, journey,
+ max_candidates: max_candidates,
+ include_wildcard?: include_wildcard?
+ )
case find_unseen_step(candidates, seen) do
nil ->
@@ -164,7 +184,15 @@ defmodule Plausible.Stats.Exploration do
step ->
new_seen = MapSet.put(seen, normalize_step_key(step))
- do_build_journey(query, journey ++ [step], new_seen, max_steps, max_candidates)
+
+ do_build_journey(
+ query,
+ journey ++ [step],
+ new_seen,
+ max_steps,
+ max_candidates,
+ include_wildcard?
+ )
end
end
@@ -181,14 +209,7 @@ defmodule Plausible.Stats.Exploration do
defp normalize_pathname("/"), do: "/"
defp normalize_pathname(pathname), do: String.trim_trailing(pathname, "/")
- @wildcard_array_join """
- if(? = 'pageview', arrayFold(
- acc, x -> arrayPushBack(acc, concat(acc[-1], '/', x)),
- arraySlice(splitByChar('/', ?) AS split_pathname, 2),
- arraySlice(split_pathname, 1, 1)), [?])
- """
-
- defp next_steps_query(query, steps, search_term, direction, max_candidates)
+ defp next_steps_query(query, steps, search_term, direction, max_candidates, include_wildcard?)
when is_direction(direction) do
next_step_idx = length(steps) + 1
q_steps = steps_query(query, next_step_idx, direction)
@@ -214,41 +235,13 @@ defmodule Plausible.Stats.Exploration do
from(s in q, where: ^step_condition)
end)
- # Expand each (name, pathname, user_id) row into all prefix paths via
- # ARRAY JOIN, then aggregate once to get both exact and wildcard visitor
- # counts in a single scan of events_v2.
- #
- # The arrayFold expansion includes the original pathname as the last
- # element, so uniqIf(user_id, original_pathname = prefix_pathname) gives the
- # exact-match count for free, alongside the wildcard uniq(user_id) and the
- # uniq(original_pathname) subpath count — all in one GROUP BY.
- #
- # Non-pageview events are included in the expansion but produce only a
- # single prefix (their exact pathname), so they naturally get
- # subpaths_count = 1 and are only emitted as exact rows.
q_per_user_matches =
from(m in q_matches,
select_merge: %{user_id: m.user_id, _sample_factor: fragment("any(?)", m._sample_factor)},
group_by: [selected_as(:name), selected_as(:pathname), m.user_id]
)
- q_combined =
- from(em in subquery(q_per_user_matches),
- join: pname in fragment(@wildcard_array_join, em.name, em.pathname, em.pathname),
- on: true,
- hints: "ARRAY",
- where: selected_as(:pathname) != "" and selected_as(:pathname) != "/",
- select: %{
- name: em.name,
- pathname: selected_as(fragment("?", pname), :pathname),
- exact_visitors:
- scale_sample(fragment("uniqIf(?, ? = ?)", em.user_id, em.pathname, pname)),
- wildcard_visitors:
- selected_as(scale_sample(fragment("uniq(?)", em.user_id)), :wildcard_visitors),
- subpaths_count: scale_sample(fragment("uniq(?)", em.pathname))
- },
- group_by: [em.name, selected_as(:pathname)]
- )
+ q_combined = combined_query(q_per_user_matches, include_wildcard?)
# Fan out each q_combined row into up to two output rows (exact + wildcard)
# using ARRAY JOIN over a small boolean array.
@@ -264,11 +257,12 @@ defmodule Plausible.Stats.Exploration do
is_wildcard in fragment(
"""
arrayFilter(
- x -> x = false OR (? = 'pageview' AND ? > 1 AND ? != ?),
+ x -> x = false OR (? = 'pageview' AND ? != '/' AND ? > 1 AND ? != ?),
[false, true]
)
""",
m.name,
+ m.pathname,
m.subpaths_count,
m.wildcard_visitors,
m.exact_visitors
@@ -315,6 +309,59 @@ defmodule Plausible.Stats.Exploration do
|> maybe_search(search_term)
end
+ # Expand each (name, pathname, user_id) row into all prefix paths via
+ # ARRAY JOIN, then aggregate once to get both exact and wildcard visitor
+ # counts in a single scan of events_v2.
+ #
+ # The arrayFold expansion includes the original pathname as the last
+ # element, so uniqIf(user_id, original_pathname = prefix_pathname) gives the
+ # exact-match count for free, alongside the wildcard uniq(user_id) and the
+ # uniq(original_pathname) subpath count — all in one GROUP BY.
+ #
+ # Non-pageview events are included in the expansion but produce only a
+ # single prefix (their exact pathname), so they naturally get
+ # subpaths_count = 1 and are only emitted as exact rows.
+ @wildcard_array_join """
+ if(? = 'pageview', arrayFold(
+ acc, x -> arrayPushBack(acc, concat(acc[-1], '/', x)),
+ arraySlice(splitByChar('/', ?) AS split_pathname, 2),
+ arraySlice(split_pathname, 1, 1)), [?])
+ """
+
+ defp combined_query(q_matches, true = _include_wildcard?) do
+ from(em in subquery(q_matches),
+ join: pname in fragment(@wildcard_array_join, em.name, em.pathname, em.pathname),
+ on: true,
+ hints: "ARRAY",
+ where: selected_as(:pathname) != "",
+ select: %{
+ name: em.name,
+ pathname: selected_as(fragment("?", pname), :pathname),
+ exact_visitors:
+ scale_sample(fragment("uniqIf(?, ? = ?)", em.user_id, em.pathname, pname)),
+ wildcard_visitors:
+ selected_as(scale_sample(fragment("uniq(?)", em.user_id)), :wildcard_visitors),
+ subpaths_count: scale_sample(fragment("uniq(?)", em.pathname))
+ },
+ group_by: [em.name, selected_as(:pathname)]
+ )
+ end
+
+ defp combined_query(q_matches, false = _include_wildcard?) do
+ from(em in subquery(q_matches),
+ where: selected_as(:pathname) != "",
+ select: %{
+ name: em.name,
+ pathname: selected_as(em.pathname, :pathname),
+ exact_visitors:
+ selected_as(scale_sample(fragment("uniq(?)", em.user_id)), :exact_visitors),
+ wildcard_visitors: selected_as(:exact_visitors),
+ subpaths_count: 1
+ },
+ group_by: [em.name, selected_as(:pathname)]
+ )
+ end
+
defp journey_funnel_query(query, steps, direction) do
q_steps = steps_query(query, length(steps), direction)
diff --git a/lib/plausible_web/controllers/api/stats_controller.ex b/lib/plausible_web/controllers/api/stats_controller.ex
index d8584106c4..398c067916 100644
--- a/lib/plausible_web/controllers/api/stats_controller.ex
+++ b/lib/plausible_web/controllers/api/stats_controller.ex
@@ -137,6 +137,8 @@ defmodule PlausibleWeb.Api.StatsController do
end
on_ee do
+ @exploration_wildcard_disabled_flag :exploration_wildcard_disabled
+
def exploration_next(conn, %{"journey" => steps} = params) do
site = conn.assigns.site
search_term = params["search_term"] || ""
@@ -147,7 +149,9 @@ defmodule PlausibleWeb.Api.StatsController do
{:ok, next_steps} <-
Plausible.Stats.Exploration.next_steps(query, journey,
search_term: search_term,
- direction: direction
+ direction: direction,
+ include_wildcard?:
+ not FunWithFlags.enabled?(@exploration_wildcard_disabled_flag, for: site)
) do
json(conn, next_steps)
else
@@ -180,7 +184,9 @@ defmodule PlausibleWeb.Api.StatsController do
case Plausible.Stats.Exploration.interesting_funnel(query,
max_steps: params["max_steps"],
- max_candidates: params["max_candidates"]
+ max_candidates: params["max_candidates"],
+ include_wildcard?:
+ not FunWithFlags.enabled?(@exploration_wildcard_disabled_flag, for: site)
) do
{:ok, funnel} -> json(conn, funnel)
{:error, :not_found} -> json(conn, [])
diff --git a/test/plausible/stats/exploration_test.exs b/test/plausible/stats/exploration_test.exs
index ba886a18fa..49b14205a4 100644
--- a/test/plausible/stats/exploration_test.exs
+++ b/test/plausible/stats/exploration_test.exs
@@ -525,6 +525,41 @@ defmodule Plausible.Stats.ExplorationTest do
assert next_step.visitors == 1
end
+ test "includes root path (/) in suggestions" do
+ site = new_site()
+
+ now = DateTime.utc_now()
+
+ populate_stats(site, [
+ build(:pageview,
+ user_id: 122,
+ pathname: "/",
+ timestamp: DateTime.shift(now, minute: -320)
+ ),
+ build(:pageview,
+ user_id: 123,
+ pathname: "/",
+ timestamp: DateTime.shift(now, minute: -320)
+ ),
+ build(:event,
+ user_id: 123,
+ name: "Signup",
+ pathname: "/register",
+ timestamp: DateTime.shift(now, minute: -300)
+ )
+ ])
+
+ query = QueryBuilder.build!(site, input_date_range: :all)
+
+ assert {:ok, [next_step1, next_step2]} = Exploration.next_steps(query, [])
+
+ assert next_step1.step.label == "/"
+ assert next_step1.visitors == 2
+
+ assert next_step2.step.label == "Signup /register"
+ assert next_step2.visitors == 1
+ end
+
test "allows to filter according to how label is rendered" do
site = new_site()
@@ -867,105 +902,132 @@ defmodule Plausible.Stats.ExplorationTest do
end
end
- test "implicit wildcard path visitors computation is correct and consistent between next_step and journey_funnel" do
- now = DateTime.utc_now()
- site = new_site()
+ describe "implicit wildcard pathnames" do
+ setup do
+ now = DateTime.utc_now()
+ site = new_site()
- populate_stats(site, [
- build(:pageview,
- user_id: 122,
- pathname: "/aa",
- timestamp: DateTime.shift(now, minute: -300)
- ),
- build(:pageview,
- user_id: 123,
- pathname: "/a",
- timestamp: DateTime.shift(now, minute: -300)
- ),
- build(:pageview,
- user_id: 124,
- pathname: "/a",
- timestamp: DateTime.shift(now, minute: -300)
- ),
- build(:pageview,
- user_id: 125,
- pathname: "/a",
- timestamp: DateTime.shift(now, minute: -300)
- ),
- build(:pageview,
- user_id: 126,
- pathname: "/a",
- timestamp: DateTime.shift(now, minute: -300)
- ),
- build(:pageview,
- user_id: 127,
- pathname: "/a",
- timestamp: DateTime.shift(now, minute: -300)
- ),
- build(:pageview,
- user_id: 123,
- pathname: "/a/b",
- timestamp: DateTime.shift(now, minute: -270)
- ),
- build(:pageview,
- user_id: 124,
- pathname: "/a/b",
- timestamp: DateTime.shift(now, minute: -270)
- ),
- build(:pageview,
- user_id: 126,
- pathname: "/a/d",
- timestamp: DateTime.shift(now, minute: -270)
- ),
- build(:pageview,
- user_id: 127,
- pathname: "/a/d",
- timestamp: DateTime.shift(now, minute: -270)
- ),
- build(:pageview,
- user_id: 123,
- pathname: "/a/b/c",
- timestamp: DateTime.shift(now, minute: -240)
- ),
- build(:pageview,
- user_id: 128,
- pathname: "/a/b/c",
- timestamp: DateTime.shift(now, minute: -240)
- )
- ])
+ populate_stats(site, [
+ build(:pageview,
+ user_id: 122,
+ pathname: "/aa",
+ timestamp: DateTime.shift(now, minute: -300)
+ ),
+ build(:pageview,
+ user_id: 123,
+ pathname: "/a",
+ timestamp: DateTime.shift(now, minute: -300)
+ ),
+ build(:pageview,
+ user_id: 124,
+ pathname: "/a",
+ timestamp: DateTime.shift(now, minute: -300)
+ ),
+ build(:pageview,
+ user_id: 125,
+ pathname: "/a",
+ timestamp: DateTime.shift(now, minute: -300)
+ ),
+ build(:pageview,
+ user_id: 126,
+ pathname: "/a",
+ timestamp: DateTime.shift(now, minute: -300)
+ ),
+ build(:pageview,
+ user_id: 127,
+ pathname: "/a",
+ timestamp: DateTime.shift(now, minute: -300)
+ ),
+ build(:pageview,
+ user_id: 123,
+ pathname: "/a/b",
+ timestamp: DateTime.shift(now, minute: -270)
+ ),
+ build(:pageview,
+ user_id: 124,
+ pathname: "/a/b",
+ timestamp: DateTime.shift(now, minute: -270)
+ ),
+ build(:pageview,
+ user_id: 126,
+ pathname: "/a/d",
+ timestamp: DateTime.shift(now, minute: -270)
+ ),
+ build(:pageview,
+ user_id: 127,
+ pathname: "/a/d",
+ timestamp: DateTime.shift(now, minute: -270)
+ ),
+ build(:pageview,
+ user_id: 123,
+ pathname: "/a/b/c",
+ timestamp: DateTime.shift(now, minute: -240)
+ ),
+ build(:pageview,
+ user_id: 128,
+ pathname: "/a/b/c",
+ timestamp: DateTime.shift(now, minute: -240)
+ )
+ ])
+
+ {:ok, site: site}
+ end
+
+ test "implicit wildcard path visitors computation is correct and consistent between next_step and journey_funnel",
+ %{site: site} do
+ query = QueryBuilder.build!(site, input_date_range: :all)
- query = QueryBuilder.build!(site, input_date_range: :all)
-
- result = Exploration.next_steps(query, [])
-
- assert {:ok,
- [
- %{step: %{label: "/a", includes_subpaths: true, subpaths_count: 4}, visitors: 6},
- %{step: %{label: "/a", includes_subpaths: false}, visitors: 5},
- %{
- step: %{label: "/a/b", includes_subpaths: true, subpaths_count: 2},
- visitors: 3
- },
- %{step: %{label: "/a/b", includes_subpaths: false}, visitors: 2},
- %{step: %{label: "/a/b/c"}, visitors: 2},
- %{step: %{label: "/a/d"}, visitors: 2},
- %{step: %{label: "/aa"}, visitors: 1}
- ]} = result
-
- journey = [
- %Exploration.Journey.Step{
- name: "pageview",
- pathname: "/a",
- includes_subpaths: true,
- subpaths_count: 4
- }
- ]
-
- assert {:ok, [step1]} = Exploration.journey_funnel(query, journey)
-
- assert step1.step.label == "/a"
- assert step1.step.includes_subpaths == true
- assert step1.visitors == 6
+ result = Exploration.next_steps(query, [])
+
+ assert {:ok,
+ [
+ %{
+ step: %{label: "/a", includes_subpaths: true, subpaths_count: 4},
+ visitors: 6
+ },
+ %{step: %{label: "/a", includes_subpaths: false}, visitors: 5},
+ %{
+ step: %{label: "/a/b", includes_subpaths: true, subpaths_count: 2},
+ visitors: 3
+ },
+ %{step: %{label: "/a/b", includes_subpaths: false}, visitors: 2},
+ %{step: %{label: "/a/b/c"}, visitors: 2},
+ %{step: %{label: "/a/d"}, visitors: 2},
+ %{step: %{label: "/aa"}, visitors: 1}
+ ]} = result
+
+ journey = [
+ %Exploration.Journey.Step{
+ name: "pageview",
+ pathname: "/a",
+ includes_subpaths: true,
+ subpaths_count: 4
+ }
+ ]
+
+ assert {:ok, [step1]} = Exploration.journey_funnel(query, journey)
+
+ assert step1.step.label == "/a"
+ assert step1.step.includes_subpaths == true
+ assert step1.visitors == 6
+ end
+
+ test "implicit wildcard paths are not returned in suggestions when explicitly disabled", %{
+ site: site
+ } do
+ query = QueryBuilder.build!(site, input_date_range: :all)
+
+ result = Exploration.next_steps(query, [], include_wildcard?: false)
+
+ assert {:ok,
+ [
+ %{step: %{label: "/a", includes_subpaths: false}, visitors: 5},
+ %{step: %{label: "/a/b", includes_subpaths: false}, visitors: 2},
+ %{step: %{label: "/a/b/c"}, visitors: 2},
+ %{step: %{label: "/a/d"}, visitors: 2},
+ %{step: %{label: "/aa"}, visitors: 1}
+ ]} = result
+ end
end
end
end