Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.
TypeScript analyticsanalytics-dashboardclickhouseelixirgoogle-analyticsmarketingopen-source-analyticsphoenixplausible-analyticspostgresqlprivacyprivacy-friendlyself-hostedsimple-analyticsstatisticstailwindcssweb-analyticswebsite-analyticswebsite-statswebsite-tracking
Make exploration implicit wildcard togglable (#6284)
Adrian Gruntkowski74f3eef39f00
diff --git a/extra/lib/plausible/stats/exploration.ex b/extra/lib/plausible/stats/exploration.exindex 4fa1263bc9..1a9b551d4f 100644--- a/extra/lib/plausible/stats/exploration.ex+++ b/extra/lib/plausible/stats/exploration.ex@@ -66,7 +66,12 @@ defmodule Plausible.Stats.Exploration do@max_steps 20@max_candidates 20- @next_steps_defaults [search_term: "", direction: :forward, max_candidates: 10]+ @next_steps_defaults [+ search_term: "",+ direction: :forward,+ max_candidates: 10,+ include_wildcard?: true+ ]@spec max_steps() :: pos_integer()def max_steps, do: @max_steps@@ -84,10 +89,11 @@ defmodule Plausible.Stats.Exploration dodirection = Keyword.fetch!(opts, :direction)search_term = Keyword.fetch!(opts, :search_term)max_candidates = min(Keyword.fetch!(opts, :max_candidates), @max_candidates)+ include_wilcard? = Keyword.fetch!(opts, :include_wildcard?)query|> Base.base_event_query()- |> next_steps_query(journey, search_term, direction, max_candidates)+ |> next_steps_query(journey, search_term, direction, max_candidates, include_wilcard?)# We pass the query struct to record query metadata for# the CH debug console.|> ClickhouseRepo.all(query: query)@@ -133,6 +139,9 @@ defmodule Plausible.Stats.Exploration do* `:max_steps` - maximum number of funnel steps to build (default: `6`)* `:max_candidates` - passed to `next_steps/3`, limitinghow many candidate next steps are fetched per step (default: `10`)+ * `:include_wildcard?` - passed to `next_steps/3`, deciding whether+ to include implicit wildcard pathnames in suggestions or not+ (default: true)"""@spec interesting_funnel(Query.t(), keyword()) ::{:ok, [funnel_step()]} | {:error, :not_found}@@ -140,23 +149,34 @@ defmodule Plausible.Stats.Exploration domax_steps = min(Keyword.get(opts, :max_steps, 6), @max_steps)max_candidates = min(Keyword.get(opts, :max_candidates, 10), @max_candidates)- case build_interesting_journey(query, max_steps, max_candidates) do+ include_wildcard? =+ Keyword.get(+ opts,+ :include_wildcard?,+ Keyword.fetch!(@next_steps_defaults, :include_wildcard?)+ )++ case build_interesting_journey(query, max_steps, max_candidates, include_wildcard?) do[] -> {:error, :not_found}journey -> journey_funnel(query, journey)endend- defp build_interesting_journey(query, max_steps, max_candidates) do- do_build_journey(query, [], MapSet.new(), max_steps, max_candidates)+ defp build_interesting_journey(query, max_steps, max_candidates, include_wildcard?) do+ do_build_journey(query, [], MapSet.new(), max_steps, max_candidates, include_wildcard?)end- defp do_build_journey(_query, journey, _seen, max_steps, _max_candidates)+ defp do_build_journey(_query, journey, _seen, max_steps, _max_candidates, _include_wildcard?)when length(journey) >= max_steps dojourneyend- defp do_build_journey(query, journey, seen, max_steps, max_candidates) do- {:ok, candidates} = next_steps(query, journey, max_candidates: max_candidates)+ defp do_build_journey(query, journey, seen, max_steps, max_candidates, include_wildcard?) do+ {:ok, candidates} =+ next_steps(query, journey,+ max_candidates: max_candidates,+ include_wildcard?: include_wildcard?+ )case find_unseen_step(candidates, seen) donil ->@@ -164,7 +184,15 @@ defmodule Plausible.Stats.Exploration dostep ->new_seen = MapSet.put(seen, normalize_step_key(step))- do_build_journey(query, journey ++ [step], new_seen, max_steps, max_candidates)++ do_build_journey(+ query,+ journey ++ [step],+ new_seen,+ max_steps,+ max_candidates,+ include_wildcard?+ )endend@@ -181,14 +209,7 @@ defmodule Plausible.Stats.Exploration dodefp normalize_pathname("/"), do: "/"defp normalize_pathname(pathname), do: String.trim_trailing(pathname, "/")- @wildcard_array_join """- if(? = 'pageview', arrayFold(- acc, x -> arrayPushBack(acc, concat(acc[-1], '/', x)),- arraySlice(splitByChar('/', ?) AS split_pathname, 2),- arraySlice(split_pathname, 1, 1)), [?])- """-- defp next_steps_query(query, steps, search_term, direction, max_candidates)+ defp next_steps_query(query, steps, search_term, direction, max_candidates, include_wildcard?)when is_direction(direction) donext_step_idx = length(steps) + 1q_steps = steps_query(query, next_step_idx, direction)@@ -214,41 +235,13 @@ defmodule Plausible.Stats.Exploration dofrom(s in q, where: ^step_condition)end)- # Expand each (name, pathname, user_id) row into all prefix paths via- # ARRAY JOIN, then aggregate once to get both exact and wildcard visitor- # counts in a single scan of events_v2.- #- # The arrayFold expansion includes the original pathname as the last- # element, so uniqIf(user_id, original_pathname = prefix_pathname) gives the- # exact-match count for free, alongside the wildcard uniq(user_id) and the- # uniq(original_pathname) subpath count — all in one GROUP BY.- #- # Non-pageview events are included in the expansion but produce only a- # single prefix (their exact pathname), so they naturally get- # subpaths_count = 1 and are only emitted as exact rows.q_per_user_matches =from(m in q_matches,select_merge: %{user_id: m.user_id, _sample_factor: fragment("any(?)", m._sample_factor)},group_by: [selected_as(:name), selected_as(:pathname), m.user_id])- q_combined =- from(em in subquery(q_per_user_matches),- join: pname in fragment(@wildcard_array_join, em.name, em.pathname, em.pathname),- on: true,- hints: "ARRAY",- where: selected_as(:pathname) != "" and selected_as(:pathname) != "/",- select: %{- name: em.name,- pathname: selected_as(fragment("?", pname), :pathname),- exact_visitors:- scale_sample(fragment("uniqIf(?, ? = ?)", em.user_id, em.pathname, pname)),- wildcard_visitors:- selected_as(scale_sample(fragment("uniq(?)", em.user_id)), :wildcard_visitors),- subpaths_count: scale_sample(fragment("uniq(?)", em.pathname))- },- group_by: [em.name, selected_as(:pathname)]- )+ q_combined = combined_query(q_per_user_matches, include_wildcard?)# Fan out each q_combined row into up to two output rows (exact + wildcard)# using ARRAY JOIN over a small boolean array.@@ -264,11 +257,12 @@ defmodule Plausible.Stats.Exploration dois_wildcard in fragment("""arrayFilter(- x -> x = false OR (? = 'pageview' AND ? > 1 AND ? != ?),+ x -> x = false OR (? = 'pageview' AND ? != '/' AND ? > 1 AND ? != ?),[false, true])""",m.name,+ m.pathname,m.subpaths_count,m.wildcard_visitors,m.exact_visitors@@ -315,6 +309,59 @@ defmodule Plausible.Stats.Exploration do|> maybe_search(search_term)end+ # Expand each (name, pathname, user_id) row into all prefix paths via+ # ARRAY JOIN, then aggregate once to get both exact and wildcard visitor+ # counts in a single scan of events_v2.+ #+ # The arrayFold expansion includes the original pathname as the last+ # element, so uniqIf(user_id, original_pathname = prefix_pathname) gives the+ # exact-match count for free, alongside the wildcard uniq(user_id) and the+ # uniq(original_pathname) subpath count — all in one GROUP BY.+ #+ # Non-pageview events are included in the expansion but produce only a+ # single prefix (their exact pathname), so they naturally get+ # subpaths_count = 1 and are only emitted as exact rows.+ @wildcard_array_join """+ if(? = 'pageview', arrayFold(+ acc, x -> arrayPushBack(acc, concat(acc[-1], '/', x)),+ arraySlice(splitByChar('/', ?) AS split_pathname, 2),+ arraySlice(split_pathname, 1, 1)), [?])+ """++ defp combined_query(q_matches, true = _include_wildcard?) do+ from(em in subquery(q_matches),+ join: pname in fragment(@wildcard_array_join, em.name, em.pathname, em.pathname),+ on: true,+ hints: "ARRAY",+ where: selected_as(:pathname) != "",+ select: %{+ name: em.name,+ pathname: selected_as(fragment("?", pname), :pathname),+ exact_visitors:+ scale_sample(fragment("uniqIf(?, ? = ?)", em.user_id, em.pathname, pname)),+ wildcard_visitors:+ selected_as(scale_sample(fragment("uniq(?)", em.user_id)), :wildcard_visitors),+ subpaths_count: scale_sample(fragment("uniq(?)", em.pathname))+ },+ group_by: [em.name, selected_as(:pathname)]+ )+ end++ defp combined_query(q_matches, false = _include_wildcard?) do+ from(em in subquery(q_matches),+ where: selected_as(:pathname) != "",+ select: %{+ name: em.name,+ pathname: selected_as(em.pathname, :pathname),+ exact_visitors:+ selected_as(scale_sample(fragment("uniq(?)", em.user_id)), :exact_visitors),+ wildcard_visitors: selected_as(:exact_visitors),+ subpaths_count: 1+ },+ group_by: [em.name, selected_as(:pathname)]+ )+ end+defp journey_funnel_query(query, steps, direction) doq_steps = steps_query(query, length(steps), direction)diff --git a/lib/plausible_web/controllers/api/stats_controller.ex b/lib/plausible_web/controllers/api/stats_controller.exindex d8584106c4..398c067916 100644--- a/lib/plausible_web/controllers/api/stats_controller.ex+++ b/lib/plausible_web/controllers/api/stats_controller.ex@@ -137,6 +137,8 @@ defmodule PlausibleWeb.Api.StatsController doendon_ee do+ @exploration_wildcard_disabled_flag :exploration_wildcard_disabled+def exploration_next(conn, %{"journey" => steps} = params) dosite = conn.assigns.sitesearch_term = params["search_term"] || ""@@ -147,7 +149,9 @@ defmodule PlausibleWeb.Api.StatsController do{:ok, next_steps} <-Plausible.Stats.Exploration.next_steps(query, journey,search_term: search_term,- direction: direction+ direction: direction,+ include_wildcard?:+ not FunWithFlags.enabled?(@exploration_wildcard_disabled_flag, for: site)) dojson(conn, next_steps)else@@ -180,7 +184,9 @@ defmodule PlausibleWeb.Api.StatsController docase Plausible.Stats.Exploration.interesting_funnel(query,max_steps: params["max_steps"],- max_candidates: params["max_candidates"]+ max_candidates: params["max_candidates"],+ include_wildcard?:+ not FunWithFlags.enabled?(@exploration_wildcard_disabled_flag, for: site)) do{:ok, funnel} -> json(conn, funnel){:error, :not_found} -> json(conn, [])diff --git a/test/plausible/stats/exploration_test.exs b/test/plausible/stats/exploration_test.exsindex ba886a18fa..49b14205a4 100644--- a/test/plausible/stats/exploration_test.exs+++ b/test/plausible/stats/exploration_test.exs@@ -525,6 +525,41 @@ defmodule Plausible.Stats.ExplorationTest doassert next_step.visitors == 1end+ test "includes root path (/) in suggestions" do+ site = new_site()++ now = DateTime.utc_now()++ populate_stats(site, [+ build(:pageview,+ user_id: 122,+ pathname: "/",+ timestamp: DateTime.shift(now, minute: -320)+ ),+ build(:pageview,+ user_id: 123,+ pathname: "/",+ timestamp: DateTime.shift(now, minute: -320)+ ),+ build(:event,+ user_id: 123,+ name: "Signup",+ pathname: "/register",+ timestamp: DateTime.shift(now, minute: -300)+ )+ ])++ query = QueryBuilder.build!(site, input_date_range: :all)++ assert {:ok, [next_step1, next_step2]} = Exploration.next_steps(query, [])++ assert next_step1.step.label == "/"+ assert next_step1.visitors == 2++ assert next_step2.step.label == "Signup /register"+ assert next_step2.visitors == 1+ end+test "allows to filter according to how label is rendered" dosite = new_site()@@ -867,105 +902,132 @@ defmodule Plausible.Stats.ExplorationTest doendend- test "implicit wildcard path visitors computation is correct and consistent between next_step and journey_funnel" do- now = DateTime.utc_now()- site = new_site()+ describe "implicit wildcard pathnames" do+ setup do+ now = DateTime.utc_now()+ site = new_site()- populate_stats(site, [- build(:pageview,- user_id: 122,- pathname: "/aa",- timestamp: DateTime.shift(now, minute: -300)- ),- build(:pageview,- user_id: 123,- pathname: "/a",- timestamp: DateTime.shift(now, minute: -300)- ),- build(:pageview,- user_id: 124,- pathname: "/a",- timestamp: DateTime.shift(now, minute: -300)- ),- build(:pageview,- user_id: 125,- pathname: "/a",- timestamp: DateTime.shift(now, minute: -300)- ),- build(:pageview,- user_id: 126,- pathname: "/a",- timestamp: DateTime.shift(now, minute: -300)- ),- build(:pageview,- user_id: 127,- pathname: "/a",- timestamp: DateTime.shift(now, minute: -300)- ),- build(:pageview,- user_id: 123,- pathname: "/a/b",- timestamp: DateTime.shift(now, minute: -270)- ),- build(:pageview,- user_id: 124,- pathname: "/a/b",- timestamp: DateTime.shift(now, minute: -270)- ),- build(:pageview,- user_id: 126,- pathname: "/a/d",- timestamp: DateTime.shift(now, minute: -270)- ),- build(:pageview,- user_id: 127,- pathname: "/a/d",- timestamp: DateTime.shift(now, minute: -270)- ),- build(:pageview,- user_id: 123,- pathname: "/a/b/c",- timestamp: DateTime.shift(now, minute: -240)- ),- build(:pageview,- user_id: 128,- pathname: "/a/b/c",- timestamp: DateTime.shift(now, minute: -240)- )- ])+ populate_stats(site, [+ build(:pageview,+ user_id: 122,+ pathname: "/aa",+ timestamp: DateTime.shift(now, minute: -300)+ ),+ build(:pageview,+ user_id: 123,+ pathname: "/a",+ timestamp: DateTime.shift(now, minute: -300)+ ),+ build(:pageview,+ user_id: 124,+ pathname: "/a",+ timestamp: DateTime.shift(now, minute: -300)+ ),+ build(:pageview,+ user_id: 125,+ pathname: "/a",+ timestamp: DateTime.shift(now, minute: -300)+ ),+ build(:pageview,+ user_id: 126,+ pathname: "/a",+ timestamp: DateTime.shift(now, minute: -300)+ ),+ build(:pageview,+ user_id: 127,+ pathname: "/a",+ timestamp: DateTime.shift(now, minute: -300)+ ),+ build(:pageview,+ user_id: 123,+ pathname: "/a/b",+ timestamp: DateTime.shift(now, minute: -270)+ ),+ build(:pageview,+ user_id: 124,+ pathname: "/a/b",+ timestamp: DateTime.shift(now, minute: -270)+ ),+ build(:pageview,+ user_id: 126,+ pathname: "/a/d",+ timestamp: DateTime.shift(now, minute: -270)+ ),+ build(:pageview,+ user_id: 127,+ pathname: "/a/d",+ timestamp: DateTime.shift(now, minute: -270)+ ),+ build(:pageview,+ user_id: 123,+ pathname: "/a/b/c",+ timestamp: DateTime.shift(now, minute: -240)+ ),+ build(:pageview,+ user_id: 128,+ pathname: "/a/b/c",+ timestamp: DateTime.shift(now, minute: -240)+ )+ ])++ {:ok, site: site}+ end++ test "implicit wildcard path visitors computation is correct and consistent between next_step and journey_funnel",+ %{site: site} do+ query = QueryBuilder.build!(site, input_date_range: :all)- query = QueryBuilder.build!(site, input_date_range: :all)-- result = Exploration.next_steps(query, [])-- assert {:ok,- [- %{step: %{label: "/a", includes_subpaths: true, subpaths_count: 4}, visitors: 6},- %{step: %{label: "/a", includes_subpaths: false}, visitors: 5},- %{- step: %{label: "/a/b", includes_subpaths: true, subpaths_count: 2},- visitors: 3- },- %{step: %{label: "/a/b", includes_subpaths: false}, visitors: 2},- %{step: %{label: "/a/b/c"}, visitors: 2},- %{step: %{label: "/a/d"}, visitors: 2},- %{step: %{label: "/aa"}, visitors: 1}- ]} = result-- journey = [- %Exploration.Journey.Step{- name: "pageview",- pathname: "/a",- includes_subpaths: true,- subpaths_count: 4- }- ]-- assert {:ok, [step1]} = Exploration.journey_funnel(query, journey)-- assert step1.step.label == "/a"- assert step1.step.includes_subpaths == true- assert step1.visitors == 6+ result = Exploration.next_steps(query, [])++ assert {:ok,+ [+ %{+ step: %{label: "/a", includes_subpaths: true, subpaths_count: 4},+ visitors: 6+ },+ %{step: %{label: "/a", includes_subpaths: false}, visitors: 5},+ %{+ step: %{label: "/a/b", includes_subpaths: true, subpaths_count: 2},+ visitors: 3+ },+ %{step: %{label: "/a/b", includes_subpaths: false}, visitors: 2},+ %{step: %{label: "/a/b/c"}, visitors: 2},+ %{step: %{label: "/a/d"}, visitors: 2},+ %{step: %{label: "/aa"}, visitors: 1}+ ]} = result++ journey = [+ %Exploration.Journey.Step{+ name: "pageview",+ pathname: "/a",+ includes_subpaths: true,+ subpaths_count: 4+ }+ ]++ assert {:ok, [step1]} = Exploration.journey_funnel(query, journey)++ assert step1.step.label == "/a"+ assert step1.step.includes_subpaths == true+ assert step1.visitors == 6+ end++ test "implicit wildcard paths are not returned in suggestions when explicitly disabled", %{+ site: site+ } do+ query = QueryBuilder.build!(site, input_date_range: :all)++ result = Exploration.next_steps(query, [], include_wildcard?: false)++ assert {:ok,+ [+ %{step: %{label: "/a", includes_subpaths: false}, visitors: 5},+ %{step: %{label: "/a/b", includes_subpaths: false}, visitors: 2},+ %{step: %{label: "/a/b/c"}, visitors: 2},+ %{step: %{label: "/a/d"}, visitors: 2},+ %{step: %{label: "/aa"}, visitors: 1}+ ]} = result+ endendendend