Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.
TypeScript analyticsanalytics-dashboardclickhouseelixirgoogle-analyticsmarketingopen-source-analyticsphoenixplausible-analyticspostgresqlprivacyprivacy-friendlyself-hostedsimple-analyticsstatisticstailwindcssweb-analyticswebsite-analyticswebsite-statswebsite-tracking
Add custom and pageview goals to exploration suggestions (#6299)
Adrian Gruntkowski4a9f46dba1e7
diff --git a/assets/js/dashboard/extra/exploration.js b/assets/js/dashboard/extra/exploration.jsindex e6bd9f1975..09aecc2467 100644--- a/assets/js/dashboard/extra/exploration.js+++ b/assets/js/dashboard/extra/exploration.js@@ -77,12 +77,15 @@ function dashboardStateForQuery(dashboardState, steps) {// Serialize steps into the wire format expected by the API.function stepsToJourneyParam(steps) {return JSON.stringify(- steps.map(({ name, pathname, includes_subpaths, subpaths_count }) => ({- name,- pathname,- includes_subpaths,- subpaths_count- }))+ steps.map(+ ({ name, pathname, includes_subpaths, subpaths_count, is_goal }) => ({+ name,+ pathname,+ includes_subpaths,+ subpaths_count,+ is_goal+ })+ ))}diff --git a/extra/lib/plausible/stats/exploration.ex b/extra/lib/plausible/stats/exploration.exindex 6eaa5f77f2..1bf0a54373 100644--- a/extra/lib/plausible/stats/exploration.ex+++ b/extra/lib/plausible/stats/exploration.ex@@ -8,16 +8,22 @@ defmodule Plausible.Stats.Exploration do@type t() :: %__MODULE__{}- @derive {Jason.Encoder, only: [:name, :pathname, :label, :includes_subpaths, :subpaths_count]}- defstruct name: nil, pathname: "", label: nil, includes_subpaths: false, subpaths_count: 0+ @derive {Jason.Encoder,+ only: [:name, :pathname, :label, :includes_subpaths, :subpaths_count, :is_goal]}+ defstruct name: nil,+ pathname: "",+ label: nil,+ includes_subpaths: false,+ subpaths_count: 0,+ is_goal: false@spec from(map()) :: t()def from(step) do- new(step.name, step.pathname, step.includes_subpaths, step.subpaths_count)+ new(step.name, step.pathname, step.includes_subpaths, step.subpaths_count, step.is_goal)end- @spec new(String.t(), String.t(), boolean(), non_neg_integer()) :: t()- def new(name, pathname, includes_subpaths \\ false, subpaths_count \\ 0)+ @spec new(String.t(), String.t(), boolean(), non_neg_integer(), boolean()) :: t()+ def new(name, pathname, includes_subpaths \\ false, subpaths_count \\ 0, is_goal \\ false)when is_boolean(includes_subpaths) and is_integer(subpaths_count) dolabel =if name != "pageview" do@@ -31,7 +37,8 @@ defmodule Plausible.Stats.Exploration doname: name,pathname: pathname,includes_subpaths: includes_subpaths,- subpaths_count: subpaths_count+ subpaths_count: subpaths_count,+ is_goal: is_goal}endend@@ -42,6 +49,7 @@ defmodule Plausible.Stats.Exploration doalias Plausible.ClickhouseRepoalias Plausible.Stats.Base+ alias Plausible.Stats.Filtersalias Plausible.Stats.Query@type journey() :: [Journey.Step.t()]@@ -76,24 +84,29 @@ defmodule Plausible.Stats.Exploration do@spec max_steps() :: pos_integer()def max_steps, do: @max_steps- @spec next_steps(Query.t(), journey(), keyword()) ::+ @spec next_steps(Plausible.Site.t(), Query.t(), journey(), keyword()) ::{:ok, [next_step()]} | {:error, :journey_too_long}- def next_steps(query, journey, opts \\ [])+ def next_steps(site, query, journey, opts \\ [])- def next_steps(_query, journey, _opts) when length(journey) >= @max_steps do+ def next_steps(_site, _query, journey, _opts) when length(journey) >= @max_steps do{:error, :journey_too_long}end- def next_steps(query, journey, opts) do+ def next_steps(site, query, journey, opts) doopts = Keyword.merge(@next_steps_defaults, opts)direction = Keyword.fetch!(opts, :direction)search_term = Keyword.fetch!(opts, :search_term)max_candidates = min(Keyword.fetch!(opts, :max_candidates), @max_candidates)include_wilcard? = Keyword.fetch!(opts, :include_wildcard?)+ goals =+ site+ |> Plausible.Goals.for_site(include_goals_with_custom_props?: false)+ |> filter_eligible_goals()+query|> Base.base_event_query()- |> next_steps_query(journey, search_term, direction, max_candidates, include_wilcard?)+ |> next_steps_query(journey, search_term, direction, max_candidates, include_wilcard?, goals)# We pass the query struct to record query metadata for# the CH debug console.|> ClickhouseRepo.all(query: query)@@ -143,9 +156,9 @@ defmodule Plausible.Stats.Exploration doto include implicit wildcard pathnames in suggestions or not(default: true)"""- @spec interesting_funnel(Query.t(), keyword()) ::+ @spec interesting_funnel(Plausible.Site.t(), Query.t(), keyword()) ::{:ok, %{funnel: [funnel_step()], candidates: [next_step()]}} | {:error, :not_found}- def interesting_funnel(query, opts \\ []) do+ def interesting_funnel(site, query, opts \\ []) domax_steps = min(Keyword.get(opts, :max_steps, 6), @max_steps)max_candidates = min(Keyword.get(opts, :max_candidates, 10), @max_candidates)@@ -157,14 +170,15 @@ defmodule Plausible.Stats.Exploration do)with {:ok, result} <-- build_interesting_journey(query, max_steps, max_candidates, include_wildcard?),+ build_interesting_journey(site, query, max_steps, max_candidates, include_wildcard?),{:ok, funnel} <- journey_funnel(query, result.journey) do{:ok, %{funnel: funnel, candidates: result.candidates}}endend- defp build_interesting_journey(query, max_steps, max_candidates, include_wildcard?) do+ defp build_interesting_journey(site, query, max_steps, max_candidates, include_wildcard?) docase do_build_journey(+ site,query,[],[],@@ -179,6 +193,7 @@ defmodule Plausible.Stats.Exploration doenddefp do_build_journey(+ _site,_query,journey,step_candidates,@@ -192,6 +207,7 @@ defmodule Plausible.Stats.Exploration doenddefp do_build_journey(+ site,query,journey,step_candidates,@@ -201,7 +217,7 @@ defmodule Plausible.Stats.Exploration doinclude_wildcard?) do{:ok, candidates} =- next_steps(query, journey,+ next_steps(site, query, journey,max_candidates: max_candidates,include_wildcard?: include_wildcard?)@@ -214,6 +230,7 @@ defmodule Plausible.Stats.Exploration donew_seen = MapSet.put(seen, normalize_step_key(step))do_build_journey(+ site,query,journey ++ [step],step_candidates ++ [candidates],@@ -238,7 +255,22 @@ defmodule Plausible.Stats.Exploration dodefp normalize_pathname("/"), do: "/"defp normalize_pathname(pathname), do: String.trim_trailing(pathname, "/")- defp next_steps_query(query, steps, search_term, direction, max_candidates, include_wildcard?)+ defp filter_eligible_goals(goals) do+ Enum.reject(goals, fn g ->+ Plausible.Goal.Revenue.revenue?(g) or g.scroll_threshold > -1 or+ Plausible.Goal.has_custom_props?(g)+ end)+ end++ defp next_steps_query(+ query,+ steps,+ search_term,+ direction,+ max_candidates,+ include_wildcard?,+ goals+ )when is_direction(direction) donext_step_idx = length(steps) + 1q_steps = steps_query(query, next_step_idx, direction)@@ -250,9 +282,12 @@ defmodule Plausible.Stats.Exploration dofrom(s in subquery(q_steps),where: selected_as(:name) != "",select: %{+ user_id: s.user_id,name: selected_as(field(s, ^next_name), :name),- pathname: selected_as(field(s, ^next_pathname), :pathname)- }+ pathname: selected_as(field(s, ^next_pathname), :pathname),+ _sample_factor: fragment("any(?)", s._sample_factor)+ },+ group_by: [selected_as(:name), selected_as(:pathname), s.user_id])q_matches =@@ -264,14 +299,6 @@ defmodule Plausible.Stats.Exploration dofrom(s in q, where: ^step_condition)end)- q_per_user_matches =- from(m in q_matches,- select_merge: %{user_id: m.user_id, _sample_factor: fragment("any(?)", m._sample_factor)},- group_by: [selected_as(:name), selected_as(:pathname), m.user_id]- )-- q_combined = combined_query(q_per_user_matches, include_wildcard?)-# Fan out each q_combined row into up to two output rows (exact + wildcard)# using ARRAY JOIN over a small boolean array.#@@ -280,55 +307,29 @@ defmodule Plausible.Stats.Exploration do# subpath, or same visitor count as exact). ARRAY JOIN then emits one or more# rows per group. The joined boolean `is_wildcard` selects which values to# use for visitors / includes_subpaths / subpaths_count.- q_all_matches =- from(m in subquery(q_combined),- join:- is_wildcard in fragment(- """- arrayFilter(- x -> x = false OR (? = 'pageview' AND ? != '/' AND ? > 1 AND ? != ?),- [false, true]- )- """,- m.name,- m.pathname,- m.subpaths_count,- m.wildcard_visitors,- m.exact_visitors- ),- on: true,- hints: "ARRAY",- where: selected_as(:visitors) > 0,- select: %{- name: m.name,- pathname: m.pathname,- visitors:- selected_as(- fragment("if(?, ?, ?)", is_wildcard, m.wildcard_visitors, m.exact_visitors),- :visitors- ),- includes_subpaths: fragment("CAST(?, 'Bool')", is_wildcard),- subpaths_count: fragment("if(?, ?, 0)", is_wildcard, m.subpaths_count)- }- )-- from(m in subquery(q_all_matches),+ q_wildcard_combined_matches =+ q_matches+ |> combined_wildcard_query(include_wildcard?)+ |> combined_wildcard_matches_query()++ q_all_combined_matches =+ if q_goal_matches = goals_query(q_matches, goals) do+ q_wildcard_combined_matches+ |> exclude_goal_matches(goals)+ |> union_all(^q_goal_matches)+ else+ q_wildcard_combined_matches+ end++ from(m in subquery(q_all_combined_matches),select: %{step: %Journey.Step{- label:- selected_as(- fragment(- "if(? != 'pageview', ?, ?)",- m.name,- m.name,- m.pathname- ),- :label- ),+ label: selected_as(m.label, :label),name: m.name,pathname: m.pathname,includes_subpaths: m.includes_subpaths,- subpaths_count: m.subpaths_count+ subpaths_count: m.subpaths_count,+ is_goal: m.is_goal},visitors: m.visitors},@@ -342,6 +343,81 @@ defmodule Plausible.Stats.Exploration do|> maybe_search(search_term)end+ defp goals_query(_, []), do: nil++ defp goals_query(q_matches, goals) do+ values =+ Enum.map(goals, fn g ->+ pathname = g.page_path || ""++ regex_pathname =+ if String.contains?(pathname, "*") do+ Filters.Utils.page_regex(pathname)+ else+ ""+ end++ %{+ label: g.display_name,+ name: g.event_name || "pageview",+ pathname: pathname,+ regex_pathname: regex_pathname+ }+ end)++ types = %{label: :string, name: :string, pathname: :string, regex_pathname: :string}++ query =+ from(m in subquery(q_matches),+ inner_join: g in values(values, types),+ on:+ g.name == m.name and+ (g.name != "pageview" or+ (g.name == "pageview" and+ fragment(+ "if(? != '', match(?, ?), ? = ?)",+ g.regex_pathname,+ m.pathname,+ g.regex_pathname,+ m.pathname,+ g.pathname+ ))),+ select: %{+ label: selected_as(g.label, :label),+ name: selected_as(g.name, :name),+ pathname: selected_as(g.pathname, :pathname),+ visitors: scale_sample(fragment("uniq(?)", m.user_id)),+ includes_subpaths: fragment("CAST(?, 'Bool')", false),+ subpaths_count: 0,+ is_goal: fragment("CAST(?, 'Bool')", true)+ },+ group_by: [selected_as(:label), selected_as(:name), selected_as(:pathname)]+ )++ from(m in subquery(query),+ where: m.visitors > 0+ )+ end++ defp exclude_goal_matches(query, goals) do+ to_exclude =+ goals+ |> Enum.filter(fn g -> is_nil(g.page_path) or not String.contains?(g.page_path, "*") end)+ |> Enum.map(fn g ->+ %{+ name: g.event_name || "pageview",+ pathname: g.page_path || ""+ }+ end)++ types = %{name: :string, pathname: :string}++ from m in subquery(query),+ left_join: g in values(to_exclude, types),+ on: g.name == m.name and g.pathname == m.pathname,+ where: g.name == "" or m.includes_subpaths+ end+# Expand each (name, pathname, user_id) row into all prefix paths via# ARRAY JOIN, then aggregate once to get both exact and wildcard visitor# counts in a single scan of events_v2.@@ -361,7 +437,7 @@ defmodule Plausible.Stats.Exploration doarraySlice(split_pathname, 1, 1)), [?])"""- defp combined_query(q_matches, true = _include_wildcard?) do+ defp combined_wildcard_query(q_matches, true = _include_wildcard?) dofrom(em in subquery(q_matches),join: pname in fragment(@wildcard_array_join, em.name, em.pathname, em.pathname),on: true,@@ -380,7 +456,7 @@ defmodule Plausible.Stats.Exploration do)end- defp combined_query(q_matches, false = _include_wildcard?) do+ defp combined_wildcard_query(q_matches, false = _include_wildcard?) dofrom(em in subquery(q_matches),where: em.name != "pageview" or selected_as(:pathname) != "",select: %{@@ -395,6 +471,51 @@ defmodule Plausible.Stats.Exploration do)end+ defp combined_wildcard_matches_query(q_wildcard_combined) do+ from(m in subquery(q_wildcard_combined),+ join:+ is_wildcard in fragment(+ """+ arrayFilter(+ x -> x = false OR (? = 'pageview' AND ? != '/' AND ? > 1 AND ? != ?),+ [false, true]+ )+ """,+ m.name,+ m.pathname,+ m.subpaths_count,+ m.wildcard_visitors,+ m.exact_visitors+ ),+ on: true,+ hints: "ARRAY",+ where: selected_as(:visitors) > 0,+ select: %{+ label:+ selected_as(+ fragment(+ "if(? != 'pageview', ?, ?)",+ m.name,+ m.name,+ m.pathname+ ),+ :label+ ),+ name: selected_as(m.name, :name),+ pathname: selected_as(m.pathname, :pathname),+ visitors:+ selected_as(+ fragment("if(?, ?, ?)", is_wildcard, m.wildcard_visitors, m.exact_visitors),+ :visitors+ ),+ includes_subpaths:+ selected_as(fragment("CAST(?, 'Bool')", is_wildcard), :includes_subpaths),+ subpaths_count: fragment("if(?, ?, 0)", is_wildcard, m.subpaths_count),+ is_goal: fragment("CAST(?, 'Bool')", false)+ }+ )+ end+defp journey_funnel_query(query, steps, direction) doq_steps = steps_query(query, length(steps), direction)@@ -524,22 +645,33 @@ defmodule Plausible.Stats.Exploration doenddefp step_condition(step, count) when count <= @max_steps do- if step.includes_subpaths do- escaped = Regex.escape(step.pathname)+ cond do+ step.includes_subpaths ->+ escaped = Regex.escape(step.pathname)- pattern = "^#{escaped}(/.+)?$"+ pattern = "^#{escaped}(/.+)?$"- dynamic(- [s],- field(s, ^:"name#{count}") == ^step.name and- fragment("match(?, ?)", field(s, ^:"pathname#{count}"), ^pattern)- )- else- dynamic(- [s],- field(s, ^:"name#{count}") == ^step.name and- field(s, ^:"pathname#{count}") == ^step.pathname- )+ dynamic(+ [s],+ field(s, ^:"name#{count}") == ^step.name and+ fragment("match(?, ?)", field(s, ^:"pathname#{count}"), ^pattern)+ )++ step.is_goal and step.name == "pageview" and String.contains?(step.pathname, "*") ->+ pattern = Filters.Utils.page_regex(step.pathname)++ dynamic(+ [s],+ field(s, ^:"name#{count}") == ^step.name and+ fragment("match(?, ?)", field(s, ^:"pathname#{count}"), ^pattern)+ )++ true ->+ dynamic(+ [s],+ field(s, ^:"name#{count}") == ^step.name and+ field(s, ^:"pathname#{count}") == ^step.pathname+ )endenddiff --git a/lib/plausible_web/controllers/api/stats_controller.ex b/lib/plausible_web/controllers/api/stats_controller.exindex 3aeca5b420..8d723d33e7 100644--- a/lib/plausible_web/controllers/api/stats_controller.ex+++ b/lib/plausible_web/controllers/api/stats_controller.ex@@ -151,7 +151,7 @@ defmodule PlausibleWeb.Api.StatsController doinclude_wildcard? =not FunWithFlags.enabled?(@exploration_wildcard_disabled_flag, for: site),{:ok, next_steps} <-- Exploration.next_steps(query, journey,+ Exploration.next_steps(site, query, journey,search_term: search_term,direction: direction,include_wildcard?: include_wildcard?@@ -188,7 +188,7 @@ defmodule PlausibleWeb.Api.StatsController doinclude_wildcard? =not FunWithFlags.enabled?(@exploration_wildcard_disabled_flag, for: site)- case Exploration.interesting_funnel(query,+ case Exploration.interesting_funnel(site, query,max_steps: params["max_steps"],max_candidates: params["max_candidates"],include_wildcard?: include_wildcard?@@ -209,7 +209,7 @@ defmodule PlausibleWeb.Api.StatsController doinclude_wildcard? =not FunWithFlags.enabled?(@exploration_wildcard_disabled_flag, for: site),{:ok, next_steps} <-- Exploration.next_steps(query, journey,+ Exploration.next_steps(site, query, journey,search_term: search_term,direction: direction,include_wildcard?: include_wildcard?@@ -242,13 +242,15 @@ defmodule PlausibleWeb.Api.StatsController do"name" => name,"pathname" => pathname,"includes_subpaths" => includes_subpaths,- "subpaths_count" => subpaths_count+ "subpaths_count" => subpaths_count,+ "is_goal" => is_goal}) doExploration.Journey.Step.new(name,pathname,includes_subpaths,- subpaths_count+ subpaths_count,+ is_goal)enddiff --git a/test/plausible/stats/exploration_test.exs b/test/plausible/stats/exploration_test.exsindex e2deaf5dbc..4cb916f594 100644--- a/test/plausible/stats/exploration_test.exs+++ b/test/plausible/stats/exploration_test.exs@@ -238,6 +238,51 @@ defmodule Plausible.Stats.ExplorationTest doassert step3.conversion_rate == "50"assert step3.conversion_rate_step == "100"end++ test "handles goal with a pattern" do+ journey = [+ %Exploration.Journey.Step{name: "pageview", pathname: "/site*", is_goal: true},+ %Exploration.Journey.Step{name: "pageview", pathname: "/dashboard"}+ ]++ site = new_site()+ now = DateTime.utc_now()++ populate_stats(site, [+ build(:pageview,+ user_id: 123,+ pathname: "/sites",+ timestamp: DateTime.shift(now, minute: -50)+ ),+ build(:pageview,+ user_id: 123,+ pathname: "/dashboard",+ timestamp: DateTime.shift(now, minute: -40)+ ),+ build(:pageview,+ user_id: 124,+ pathname: "/sites/settings",+ timestamp: DateTime.shift(now, minute: -50)+ )+ ])++ query = QueryBuilder.build!(site, input_date_range: :all)++ assert {:ok, [step1, step2]} = Exploration.journey_funnel(query, journey)++ assert step1.step.pathname == "/site*"+ assert step1.visitors == 2+ assert step1.dropoff == 0+ assert step1.dropoff_percentage == "0"+ assert step1.conversion_rate == "100"+ assert step1.conversion_rate_step == "0"+ assert step2.step.pathname == "/dashboard"+ assert step2.visitors == 1+ assert step2.dropoff == 1+ assert step2.dropoff_percentage == "50"+ assert step2.conversion_rate == "50"+ assert step2.conversion_rate_step == "50"+ endenddescribe "interesting_funnel" do@@ -245,7 +290,7 @@ defmodule Plausible.Stats.ExplorationTest doquery = QueryBuilder.build!(site, input_date_range: :all)assert {:ok, %{funnel: [step1, step2, step3, step4]}} =- Exploration.interesting_funnel(query)+ Exploration.interesting_funnel(site, query)assert step1.step.pathname == "/home"assert step1.visitors == 2@@ -264,7 +309,7 @@ defmodule Plausible.Stats.ExplorationTest doquery = QueryBuilder.build!(site, input_date_range: :all)assert {:ok, %{funnel: [step1, step2]}} =- Exploration.interesting_funnel(query, max_steps: 2)+ Exploration.interesting_funnel(site, query, max_steps: 2)assert step1.step.pathname == "/home"assert step2.step.pathname == "/login"@@ -274,7 +319,7 @@ defmodule Plausible.Stats.ExplorationTest doempty_site = new_site()query = QueryBuilder.build!(empty_site, input_date_range: :all)- assert {:error, :not_found} = Exploration.interesting_funnel(query)+ assert {:error, :not_found} = Exploration.interesting_funnel(empty_site, query)endtest "stops when no more unseen steps are available" do@@ -291,7 +336,8 @@ defmodule Plausible.Stats.ExplorationTest doquery = QueryBuilder.build!(site, input_date_range: :all)- assert {:ok, %{funnel: [step1]}} = Exploration.interesting_funnel(query, max_steps: 6)+ assert {:ok, %{funnel: [step1]}} =+ Exploration.interesting_funnel(site, query, max_steps: 6)assert step1.step.pathname == "/only-page"assert step1.visitors == 1@@ -304,7 +350,7 @@ defmodule Plausible.Stats.ExplorationTest dofilters: [[:is, "visit:browser", ["Firefox"]]])- assert {:ok, result} = Exploration.interesting_funnel(query)+ assert {:ok, result} = Exploration.interesting_funnel(site, query)pathnames = Enum.map(result.funnel, & &1.step.pathname)assert pathnames == ["/docs", "/logout"]@@ -361,7 +407,7 @@ defmodule Plausible.Stats.ExplorationTest doquery = QueryBuilder.build!(site, input_date_range: :all)- assert {:ok, result} = Exploration.interesting_funnel(query)+ assert {:ok, result} = Exploration.interesting_funnel(site, query)pathnames = Enum.map(result.funnel, & &1.step.pathname)assert pathnames == ["/a", "/b", "/c"]@@ -378,7 +424,7 @@ defmodule Plausible.Stats.ExplorationTest do]assert {:ok, [next_step1, next_step2, next_step3]} =- Exploration.next_steps(query, journey)+ Exploration.next_steps(site, query, journey)assert next_step1.step.label == "/docs"assert next_step1.step.pathname == "/docs"@@ -400,7 +446,7 @@ defmodule Plausible.Stats.ExplorationTest do]assert {:ok, [%{step: %{pathname: "/docs"}}]} =- Exploration.next_steps(query, journey, max_candidates: 1)+ Exploration.next_steps(site, query, journey, max_candidates: 1)endtest "returns error on too long journey", %{site: site} do@@ -411,14 +457,14 @@ defmodule Plausible.Stats.ExplorationTest do%Exploration.Journey.Step{name: "pageview", pathname: "/page#{idx}"}end)- assert {:error, :journey_too_long} = Exploration.next_steps(query, journey)+ assert {:error, :journey_too_long} = Exploration.next_steps(site, query, journey)endtest "suggests the first step in the journey", %{site: site} doquery = QueryBuilder.build!(site, input_date_range: :all)assert {:ok, [next_step1, next_step2, next_step3, next_step4]} =- Exploration.next_steps(query, [])+ Exploration.next_steps(site, query, [])assert next_step1.step.pathname == "/home"assert next_step1.visitors == 2@@ -438,7 +484,7 @@ defmodule Plausible.Stats.ExplorationTest do)assert {:ok, [next_step1, next_step2, next_step3, next_step4]} =- Exploration.next_steps(query, [])+ Exploration.next_steps(site, query, [])assert next_step1.step.pathname == "/docs"assert next_step1.visitors == 1@@ -458,7 +504,8 @@ defmodule Plausible.Stats.ExplorationTest do%Exploration.Journey.Step{name: "pageview", pathname: "/login"}]- assert {:ok, [next_step]} = Exploration.next_steps(query, journey, search_term: "doc")+ assert {:ok, [next_step]} =+ Exploration.next_steps(site, query, journey, search_term: "doc")assert next_step.step.pathname == "/docs"assert next_step.visitors == 1@@ -490,7 +537,7 @@ defmodule Plausible.Stats.ExplorationTest doquery = QueryBuilder.build!(site, input_date_range: :all)- assert {:ok, [next_step1, next_step2]} = Exploration.next_steps(query, [])+ assert {:ok, [next_step1, next_step2]} = Exploration.next_steps(site, query, [])assert next_step1.step.label == "/"assert next_step1.visitors == 2@@ -530,7 +577,7 @@ defmodule Plausible.Stats.ExplorationTest do]assert {:ok, [next_step]} =- Exploration.next_steps(query, journey, search_term: "up")+ Exploration.next_steps(site, query, journey, search_term: "up")assert next_step.step.label == "Signup"assert next_step.step.name == "Signup"@@ -546,7 +593,7 @@ defmodule Plausible.Stats.ExplorationTest do]assert {:ok, [next_step1, next_step2]} =- Exploration.next_steps(query, journey, direction: :backward)+ Exploration.next_steps(site, query, journey, direction: :backward)assert next_step1.visitors == 1assert next_step2.step.pathname == "/login"@@ -605,13 +652,19 @@ defmodule Plausible.Stats.ExplorationTest do[%{step: %{pathname: "/:dashboard"}}]} =- Exploration.next_steps(query, journey, search_term: "", direction: :forward)+ Exploration.next_steps(site, query, journey,+ search_term: "",+ direction: :forward+ )assert {:ok,[%{step: %{pathname: "/:dashboard"}}]} =- Exploration.next_steps(query, journey, search_term: "", direction: :backward)+ Exploration.next_steps(site, query, journey,+ search_term: "",+ direction: :backward+ )endtest "treats identical sequence of events as a single step" do@@ -690,7 +743,7 @@ defmodule Plausible.Stats.ExplorationTest do]assert {:ok, [next_step1, next_step2, next_step3]} =- Exploration.next_steps(query, journey)+ Exploration.next_steps(site, query, journey)assert next_step1.step.pathname == "/docs"assert next_step1.visitors == 1@@ -705,7 +758,7 @@ defmodule Plausible.Stats.ExplorationTest do%Exploration.Journey.Step{name: "pageview", pathname: "/docs"}]- assert {:ok, [next_step]} = Exploration.next_steps(query, journey)+ assert {:ok, [next_step]} = Exploration.next_steps(site, query, journey)assert next_step.step.pathname == "/logout"assert next_step.visitors == 1@@ -760,7 +813,8 @@ defmodule Plausible.Stats.ExplorationTest do%Exploration.Journey.Step{name: "pageview", pathname: "/logout"}]- assert {:ok, [next_step]} = Exploration.next_steps(query, journey, direction: :backward)+ assert {:ok, [next_step]} =+ Exploration.next_steps(site, query, journey, direction: :backward)assert next_step.step.pathname == "/login"assert next_step.visitors == 1@@ -811,7 +865,8 @@ defmodule Plausible.Stats.ExplorationTest doquery = QueryBuilder.build!(site, input_date_range: :all)- assert {:ok, [next_step1, next_step2, next_step3]} = Exploration.next_steps(query, [])+ assert {:ok, [next_step1, next_step2, next_step3]} =+ Exploration.next_steps(site, query, [])assert next_step1.step.pathname == "/home"assert next_step1.visitors == 2@@ -824,7 +879,7 @@ defmodule Plausible.Stats.ExplorationTest do%Exploration.Journey.Step{name: "pageview", pathname: "/home"}]- assert {:ok, [next_step]} = Exploration.next_steps(query, journey)+ assert {:ok, [next_step]} = Exploration.next_steps(site, query, journey)assert next_step.step.pathname == "/login"assert next_step.visitors == 2@@ -834,11 +889,173 @@ defmodule Plausible.Stats.ExplorationTest do%Exploration.Journey.Step{name: "pageview", pathname: "/login"}]- assert {:ok, [next_step]} = Exploration.next_steps(query, journey)+ assert {:ok, [next_step]} = Exploration.next_steps(site, query, journey)assert next_step.step.pathname == "/logout"assert next_step.visitors == 2end++ test "considers existing goals in the listing" do+ now = DateTime.utc_now()+ site = new_site()++ Plausible.Goals.create(site, %{"page_path" => "/home"})+ Plausible.Goals.create(site, %{"event_name" => "Signup"})++ Plausible.Goals.create(site, %{+ "page_path" => "/sites/new",+ "display_name" => "Create a site"+ })++ Plausible.Goals.create(site, %{"page_path" => "/site*"})++ populate_stats(site, [+ build(:pageview,+ user_id: 123,+ pathname: "/home",+ timestamp: DateTime.shift(now, minute: -300)+ ),+ build(:pageview,+ user_id: 123,+ pathname: "/register",+ timestamp: DateTime.shift(now, minute: -290)+ ),+ build(:event,+ user_id: 123,+ name: "Signup",+ pathname: "/register",+ timestamp: DateTime.shift(now, minute: -280)+ ),+ build(:pageview,+ user_id: 123,+ pathname: "/activate",+ timestamp: DateTime.shift(now, minute: -270)+ ),+ build(:pageview,+ user_id: 123,+ pathname: "/sites/new",+ timestamp: DateTime.shift(now, minute: -260)+ ),+ build(:pageview,+ user_id: 123,+ pathname: "/sites",+ timestamp: DateTime.shift(now, minute: -250)+ ),+ build(:pageview,+ user_id: 124,+ pathname: "/home",+ timestamp: DateTime.shift(now, minute: -300)+ ),+ build(:pageview,+ user_id: 124,+ pathname: "/register",+ timestamp: DateTime.shift(now, minute: -290)+ ),+ build(:event,+ user_id: 124,+ name: "Signup",+ pathname: "/register",+ timestamp: DateTime.shift(now, minute: -280)+ ),+ build(:pageview,+ user_id: 124,+ pathname: "/activate",+ timestamp: DateTime.shift(now, minute: -270)+ ),+ build(:pageview,+ user_id: 124,+ pathname: "/sites",+ timestamp: DateTime.shift(now, minute: -250)+ ),+ build(:pageview,+ user_id: 125,+ pathname: "/home",+ timestamp: DateTime.shift(now, minute: -300)+ ),+ build(:pageview,+ user_id: 125,+ pathname: "/register",+ timestamp: DateTime.shift(now, minute: -290)+ ),+ build(:event,+ user_id: 125,+ name: "Signup",+ pathname: "/register",+ timestamp: DateTime.shift(now, minute: -280)+ ),+ build(:pageview,+ user_id: 126,+ pathname: "/home",+ timestamp: DateTime.shift(now, minute: -300)+ ),+ build(:pageview,+ user_id: 126,+ pathname: "/register",+ timestamp: DateTime.shift(now, minute: -290)+ ),+ build(:pageview,+ user_id: 127,+ pathname: "/home",+ timestamp: DateTime.shift(now, minute: -300)+ )+ ])++ query = QueryBuilder.build!(site, input_date_range: :all)++ assert {:ok,+ [+ next_step1,+ next_step2,+ next_step3,+ next_step4,+ next_step5,+ next_step6,+ next_step7+ ]} =+ Exploration.next_steps(site, query, [])++ assert next_step1.step.label == "Visit /home"+ assert next_step1.step.name == "pageview"+ assert next_step1.step.pathname == "/home"+ assert next_step1.step.is_goal+ assert next_step1.visitors == 5++ assert next_step2.step.label == "/register"+ assert next_step2.step.name == "pageview"+ assert next_step2.step.pathname == "/register"+ refute next_step2.step.is_goal+ assert next_step2.visitors == 4++ assert next_step3.step.label == "Signup"+ assert next_step3.step.name == "Signup"+ assert next_step3.step.pathname == ""+ assert next_step3.step.is_goal+ assert next_step3.visitors == 3++ assert next_step4.step.label == "/activate"+ assert next_step4.step.name == "pageview"+ assert next_step4.step.pathname == "/activate"+ refute next_step4.step.is_goal+ assert next_step4.visitors == 2++ assert next_step5.step.label == "Visit /site*"+ assert next_step5.step.name == "pageview"+ assert next_step5.step.pathname == "/site*"+ assert next_step5.step.is_goal+ assert next_step5.visitors == 2++ assert next_step6.step.label == "/sites"+ assert next_step6.step.name == "pageview"+ assert next_step6.step.pathname == "/sites"+ refute next_step6.step.is_goal+ assert next_step6.visitors == 2++ assert next_step7.step.label == "Create a site"+ assert next_step7.step.name == "pageview"+ assert next_step7.step.pathname == "/sites/new"+ assert next_step7.step.is_goal+ assert next_step7.visitors == 1+ endenddescribe "implicit wildcard pathnames" do@@ -916,7 +1133,7 @@ defmodule Plausible.Stats.ExplorationTest do%{site: site} doquery = QueryBuilder.build!(site, input_date_range: :all)- result = Exploration.next_steps(query, [])+ result = Exploration.next_steps(site, query, [])assert {:ok,[@@ -956,7 +1173,7 @@ defmodule Plausible.Stats.ExplorationTest do} doquery = QueryBuilder.build!(site, input_date_range: :all)- result = Exploration.next_steps(query, [], include_wildcard?: false)+ result = Exploration.next_steps(site, query, [], include_wildcard?: false)assert {:ok,[