Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.
TypeScript analyticsanalytics-dashboardclickhouseelixirgoogle-analyticsmarketingopen-source-analyticsphoenixplausible-analyticspostgresqlprivacyprivacy-friendlyself-hostedsimple-analyticsstatisticstailwindcssweb-analyticswebsite-analyticswebsite-statswebsite-tracking
Add implicit wildcard matches to exploration suggestions (#6271)
Adrian Gruntkowskibe2d839f771b
diff --git a/assets/js/dashboard/extra/exploration.js b/assets/js/dashboard/extra/exploration.jsindex 87dd6b9a48..ca7e8d4a67 100644--- a/assets/js/dashboard/extra/exploration.js+++ b/assets/js/dashboard/extra/exploration.js@@ -1,10 +1,14 @@import React, { useState, useEffect, useRef } from 'react'import * as api from '../api'import * as url from '../util/url'+import { Tooltip } from '../util/tooltip'import { useDebounce } from '../custom-hooks'import { useSiteContext } from '../site-context'import { useDashboardStateContext } from '../dashboard-state-context'-import { numberShortFormatter } from '../util/number-formatter'+import {+ numberShortFormatter,+ numberLongFormatter+} from '../util/number-formatter'const PAGE_FILTER_KEYS = ['page', 'entry_page', 'exit_page']const EXPLORATION_DIRECTIONS = {@@ -26,7 +30,12 @@ function stateWithApplicableFilters(dashboardState, steps) {}function toJourney(steps) {- return steps.map((s) => ({ name: s.name, pathname: s.pathname }))+ return steps.map((s) => ({+ name: s.name,+ pathname: s.pathname,+ includes_subpaths: s.includes_subpaths,+ subpaths_count: s.subpaths_count+ }))}function fetchNextWithFunnel(@@ -61,7 +70,11 @@ function fetchInterestingFunnel(site, dashboardState) {}function isSameStep(step, otherStep) {- return step.name === otherStep.name && step.pathname === otherStep.pathname+ return (+ step.name === otherStep.name &&+ step.pathname === otherStep.pathname &&+ step.includes_subpaths === otherStep.includes_subpaths+ )}function ExplorationColumn({@@ -148,9 +161,12 @@ function ExplorationColumn({isSelected && selectedConversionRate !== null? selectedConversionRate: Math.round((visitors / stepMaxVisitors) * 100)+ const label = step.includes_subpaths+ ? `${step.label}… (${step.subpaths_count} pages)`+ : step.labelreturn (- <li key={step.label}>+ <li key={label}><buttonclassName={`group w-full border text-left px-2.5 pt-2 pb-2.5 text-sm rounded-md focus:outline-none ${isSelected@@ -162,12 +178,14 @@ function ExplorationColumn({<div className="flex items-center justify-between gap-2 mb-1"><spanclassName="truncate font-medium text-gray-800 dark:text-gray-200"- title={step.label}+ title={label}>- {step.label}+ {label}</span><span className="shrink-0 text-gray-800 dark:text-gray-200 tabular-nums">- {numberShortFormatter(visitorsToShow)}+ <Tooltip info={numberLongFormatter(visitorsToShow)}>+ {numberShortFormatter(visitorsToShow)}+ </Tooltip></span></div><divdiff --git a/extra/lib/plausible/stats/exploration.ex b/extra/lib/plausible/stats/exploration.exindex 95debb658c..8b9e4f0390 100644--- a/extra/lib/plausible/stats/exploration.ex+++ b/extra/lib/plausible/stats/exploration.ex@@ -8,20 +8,30 @@ defmodule Plausible.Stats.Exploration do@type t() :: %__MODULE__{}- @derive {Jason.Encoder, only: [:name, :pathname, :label]}- defstruct [:name, :pathname, :label]+ @derive {Jason.Encoder, only: [:name, :pathname, :label, :includes_subpaths, :subpaths_count]}+ defstruct name: nil, pathname: nil, label: nil, includes_subpaths: false, subpaths_count: 0@spec from(map()) :: t()def from(step) do- new(step.name, step.pathname)+ new(step.name, step.pathname, step.includes_subpaths, step.subpaths_count)end- @spec new(String.t(), String.t()) :: t()- def new(name, pathname) do+ @spec new(String.t(), String.t(), boolean(), non_neg_integer()) :: t()+ def new(name, pathname, includes_subpaths \\ false, subpaths_count \\ 0)+ when is_boolean(includes_subpaths) and is_integer(subpaths_count) do+ label =+ if name != "pageview" do+ name <> " " <> pathname+ else+ pathname+ end+%__MODULE__{- label: if(name == "pageview", do: "Visit", else: name) <> " " <> pathname,+ label: label,name: name,- pathname: pathname+ pathname: pathname,+ includes_subpaths: includes_subpaths,+ subpaths_count: subpaths_count}endend@@ -53,14 +63,27 @@ defmodule Plausible.Stats.Exploration doconversion_rate_step: String.t()}+ @max_steps 20+ @max_candidates 20+@next_steps_defaults [search_term: "", direction: :forward, max_candidates: 10]- @spec next_steps(Query.t(), journey(), keyword()) :: {:ok, [next_step()]}- def next_steps(query, journey, opts \\ []) do+ @spec max_steps() :: pos_integer()+ def max_steps, do: @max_steps++ @spec next_steps(Query.t(), journey(), keyword()) ::+ {:ok, [next_step()]} | {:error, :journey_too_long}+ def next_steps(query, journey, opts \\ [])++ def next_steps(_query, journey, _opts) when length(journey) >= @max_steps do+ {:error, :journey_too_long}+ end++ def next_steps(query, journey, opts) doopts = Keyword.merge(@next_steps_defaults, opts)direction = Keyword.fetch!(opts, :direction)search_term = Keyword.fetch!(opts, :search_term)- max_candidates = min(Keyword.fetch!(opts, :max_candidates), 20)+ max_candidates = min(Keyword.fetch!(opts, :max_candidates), @max_candidates)query|> Base.base_event_query()@@ -72,11 +95,15 @@ defmodule Plausible.Stats.Exploration doend@spec journey_funnel(Query.t(), journey(), direction()) ::- {:ok, [funnel_step()]} | {:error, :empty_journey}+ {:ok, [funnel_step()]} | {:error, :empty_journey | :journey_too_long}def journey_funnel(query, journey, direction \\ :forward)def journey_funnel(_query, [], _direction), do: {:error, :empty_journey}+ def journey_funnel(_query, journey, _direction) when length(journey) > @max_steps do+ {:error, :journey_too_long}+ end+def journey_funnel(query, journey, direction) when is_direction(direction) doquery|> Base.base_event_query()@@ -110,8 +137,8 @@ defmodule Plausible.Stats.Exploration do@spec interesting_funnel(Query.t(), keyword()) ::{:ok, [funnel_step()]} | {:error, :not_found}def interesting_funnel(query, opts \\ []) do- max_steps = min(Keyword.get(opts, :max_steps, 6), 20)- max_candidates = min(Keyword.get(opts, :max_candidates, 10), 20)+ max_steps = min(Keyword.get(opts, :max_steps, 6), @max_steps)+ max_candidates = min(Keyword.get(opts, :max_candidates, 10), @max_candidates)case build_interesting_journey(query, max_steps, max_candidates) do[] -> {:error, :not_found}@@ -154,6 +181,13 @@ defmodule Plausible.Stats.Exploration dodefp normalize_pathname("/"), do: "/"defp normalize_pathname(pathname), do: String.trim_trailing(pathname, "/")+ @wildcard_array_join """+ arrayFold(+ acc, x -> arrayPushBack(acc, concat(acc[-1], '/', x)),+ arraySlice(splitByChar('/', ?) AS split_pathname, 2),+ arraySlice(split_pathname, 1, 1))+ """+defp next_steps_query(query, steps, search_term, direction, max_candidates)when is_direction(direction) donext_step_idx = length(steps) + 1@@ -162,80 +196,147 @@ defmodule Plausible.Stats.Exploration donext_name = :"name#{next_step_idx}"next_pathname = :"pathname#{next_step_idx}"- q_next =+ q_matches =from(s in subquery(q_steps),- where: selected_as(:next_name) != "",+ where: selected_as(:name) != "",select: %{- step: %Journey.Step{- label:- selected_as(- fragment(- "concat(CASE WHEN ? = ? THEN ? ELSE ? END, ' ', ?)",- selected_as(:next_name),- "pageview",- "Visit",- selected_as(:next_name),- selected_as(:next_pathname)- ),- :next_label- ),- name: selected_as(field(s, ^next_name), :next_name),- pathname: selected_as(field(s, ^next_pathname), :next_pathname)- },- visitors: selected_as(scale_sample(fragment("uniq(?)", s.user_id)), :count)- },- group_by: [selected_as(:next_name), selected_as(:next_pathname)],- order_by: [- desc: selected_as(:count),- asc: selected_as(:next_pathname),- asc: selected_as(:next_name)- ],- limit: ^max_candidates+ name: selected_as(field(s, ^next_name), :name),+ pathname: selected_as(field(s, ^next_pathname), :pathname)+ })- |> maybe_search(search_term)- steps- |> Enum.with_index()- |> Enum.reduce(q_next, fn {step, idx}, q ->- name = :"name#{idx + 1}"- pathname = :"pathname#{idx + 1}"+ q_matches =+ steps+ |> Enum.with_index()+ |> Enum.reduce(q_matches, fn {step, idx}, q ->+ step_condition = step_condition(step, idx + 1)++ from(s in q, where: ^step_condition)+ end)- from(s in q,- where: field(s, ^name) == ^step.name and field(s, ^pathname) == ^step.pathname+ q_exact_matches =+ from(m in q_matches,+ select_merge: %{+ visitors: scale_sample(fragment("uniq(?)", m.user_id)),+ includes_subpaths: type(^false, :boolean),+ subpaths_count: 0+ },+ group_by: [selected_as(:name), selected_as(:pathname)])- end)- end- defp journey_funnel_query(query, steps, direction) do- q_steps = steps_query(query, length(steps), direction)+ q_per_user_matches =+ from(m in q_matches,+ select_merge: %{user_id: m.user_id, _sample_factor: fragment("any(?)", m._sample_factor)},+ group_by: [selected_as(:name), selected_as(:pathname), m.user_id]+ )- [first_step | steps] = steps+ q_wildcard_matches =+ from(em in subquery(q_per_user_matches),+ join: pname in fragment(@wildcard_array_join, em.pathname),+ on: true,+ hints: "ARRAY",+ where: em.name == "pageview",+ where: selected_as(:pathname) != "" and selected_as(:pathname) != "/",+ select: %{+ name: em.name,+ pathname: selected_as(fragment("?", pname), :pathname),+ visitors: selected_as(scale_sample(fragment("uniq(?)", em.user_id)), :visitors),+ unique_paths: scale_sample(fragment("uniq(?)", em.pathname))+ },+ group_by: [em.name, selected_as(:pathname)]+ )- q_funnel =- from(s in subquery(q_steps),- where: s.name1 == ^first_step.name and s.pathname1 == ^first_step.pathname,+ q_wildcard_filtered_matches =+ from(wm in subquery(q_wildcard_matches),+ left_join: emx in subquery(q_exact_matches),+ on: emx.name == wm.name and emx.pathname == wm.pathname,+ where: wm.unique_paths > 1 and (is_nil(emx.name) or wm.visitors != emx.visitors),select: %{- 1 => scale_sample(fragment("uniq(?)", s.user_id))+ name: wm.name,+ pathname: wm.pathname,+ visitors: wm.visitors,+ includes_subpaths: type(^true, :boolean),+ subpaths_count: wm.unique_paths})- steps- |> Enum.with_index()- |> Enum.reduce(q_funnel, fn {_step, idx}, q ->- current_steps = Enum.take(steps, idx + 1)+ q_all_matches =+ q_exact_matches+ |> union_all(^q_wildcard_filtered_matches)++ from(m in subquery(q_all_matches),+ select: %{+ step: %Journey.Step{+ label:+ selected_as(+ fragment(+ "if(? != 'pageview', concat(?, ' ', ?), ?)",+ m.name,+ m.name,+ m.pathname,+ m.pathname+ ),+ :label+ ),+ name: m.name,+ pathname: m.pathname,+ includes_subpaths: m.includes_subpaths,+ subpaths_count: m.subpaths_count+ },+ visitors: m.visitors+ },+ order_by: [+ desc: m.visitors,+ asc: m.pathname,+ asc: m.name+ ],+ limit: ^max_candidates+ )+ |> maybe_search(search_term)+ end++ defp journey_funnel_query(query, steps, direction) do+ q_steps = steps_query(query, length(steps), direction)- step_conditions =- current_steps- |> Enum.with_index()- |> Enum.reduce(dynamic(true), fn {step, idx}, acc ->- step_condition = step_condition(step, idx + 2)- dynamic([q], fragment("? and ?", ^acc, ^step_condition))- end)+ q_funnel = from(s in subquery(q_steps), select: %{})- step_count =- dynamic([e], scale_sample(fragment("uniqIf(?, ?)", e.user_id, ^step_conditions)))+ steps+ |> Enum.with_index()+ |> Enum.reduce(q_funnel, fn+ {step, 0}, q ->+ step_condition = step_condition(step, 1)- from(e in q, select_merge: ^%{(idx + 2) => step_count})+ from(e in q,+ select_merge: %{+ 1 => scale_sample(fragment("uniq(?)", e.user_id))+ },+ where: ^step_condition+ )++ {_step, idx}, q ->+ current_steps = Enum.take(steps, idx + 1)++ step_conditions =+ current_steps+ |> Enum.with_index()+ |> Enum.reduce(dynamic(true), fn {step, idx}, acc ->+ step_condition = step_condition(step, idx + 1)+ dynamic([q], fragment("? and ?", ^acc, ^step_condition))+ end)++ step_count =+ dynamic(+ [e],+ scale_sample(+ fragment(+ "uniqIf(?, ?)",+ e.user_id,+ ^step_conditions+ )+ )+ )++ from(e in q, select_merge: ^%{(idx + 1) => step_count})end)end@@ -255,7 +356,8 @@ defmodule Plausible.Stats.Exploration douser_id: e.user_id,_sample_factor: e._sample_factor,name: e.name,- pathname: e.pathname,+ pathname:+ fragment("if(? = '/', ?, trimRight(?, '/'))", e.pathname, e.pathname, e.pathname),timestamp: e.timestamp},where: e.name != "engagement",@@ -288,7 +390,9 @@ defmodule Plausible.Stats.Exploration dodefp select_previous(query, :forward) dofrom(e in query,select_merge: %{- prev_pathname: lag(e.pathname) |> over(:session_window),+ prev_pathname:+ lag(fragment("if(? = '/', ?, trimRight(?, '/'))", e.pathname, e.pathname, e.pathname))+ |> over(:session_window),prev_name: lag(e.name) |> over(:session_window)})@@ -297,7 +401,9 @@ defmodule Plausible.Stats.Exploration dodefp select_previous(query, :backward) dofrom(e in query,select_merge: %{- prev_pathname: lead(e.pathname) |> over(:session_window),+ prev_pathname:+ lead(fragment("if(? = '/', ?, trimRight(?, '/'))", e.pathname, e.pathname, e.pathname))+ |> over(:session_window),prev_name: lead(e.name) |> over(:session_window)})@@ -321,18 +427,30 @@ defmodule Plausible.Stats.Exploration do)end- defp step_condition(step, count) do- dynamic(- [s],- field(s, ^:"name#{count}") == ^step.name and- field(s, ^:"pathname#{count}") == ^step.pathname- )+ defp step_condition(step, count) when count <= @max_steps do+ if step.includes_subpaths do+ escaped = Regex.escape(step.pathname)++ pattern = "^#{escaped}(/.+)?$"++ dynamic(+ [s],+ field(s, ^:"name#{count}") == ^step.name and+ fragment("match(?, ?)", field(s, ^:"pathname#{count}"), ^pattern)+ )+ else+ dynamic(+ [s],+ field(s, ^:"name#{count}") == ^step.name and+ field(s, ^:"pathname#{count}") == ^step.pathname+ )+ endenddefp maybe_search(query, search_term) docase String.trim(search_term) doterm when byte_size(term) > 2 ->- from(s in query, where: ilike(selected_as(:next_label), ^"%#{term}%"))+ from(s in query, where: ilike(selected_as(:label), ^"%#{term}%"))_ ->querydiff --git a/lib/plausible_web/controllers/api/stats_controller.ex b/lib/plausible_web/controllers/api/stats_controller.exindex b281213ccf..d8584106c4 100644--- a/lib/plausible_web/controllers/api/stats_controller.ex+++ b/lib/plausible_web/controllers/api/stats_controller.ex@@ -151,8 +151,8 @@ defmodule PlausibleWeb.Api.StatsController do) dojson(conn, next_steps)else- _ ->- bad_request(conn, "There was an error with your request")+ {:error, :journey_too_long} ->+ bad_request(conn, "The journey is too long")endend@@ -167,13 +167,10 @@ defmodule PlausibleWeb.Api.StatsController dojson(conn, funnel)else{:error, :empty_journey} ->- bad_request(- conn,- "We are unable to show funnels when journey is empty",- %{- level: :normal- }- )+ bad_request(conn, "We are unable to show funnels when journey is empty")++ {:error, :journey_too_long} ->+ bad_request(conn, "The journey is too long")endend@@ -227,8 +224,18 @@ defmodule PlausibleWeb.Api.StatsController do|> then(&{:ok, &1})end- defp parse_journey_step(%{"name" => name, "pathname" => pathname}) do- Plausible.Stats.Exploration.Journey.Step.new(name, pathname)+ defp parse_journey_step(%{+ "name" => name,+ "pathname" => pathname,+ "includes_subpaths" => includes_subpaths,+ "subpaths_count" => subpaths_count+ }) do+ Plausible.Stats.Exploration.Journey.Step.new(+ name,+ pathname,+ includes_subpaths,+ subpaths_count+ )enddefp parse_exploration_direction("backward"), do: {:ok, :backward}diff --git a/test/plausible/stats/exploration_test.exs b/test/plausible/stats/exploration_test.exsindex 817f7474ff..ba886a18fa 100644--- a/test/plausible/stats/exploration_test.exs+++ b/test/plausible/stats/exploration_test.exs@@ -149,9 +149,9 @@ defmodule Plausible.Stats.ExplorationTest doassert {:ok, [step1, step2, step3, step4]} = Exploration.journey_funnel(query, journey)- assert step1.step.label == "Visit /register"+ assert step1.step.label == "/register"assert step2.step.label == "Signup /register"- assert step3.step.label == "Visit /activate"+ assert step3.step.label == "/activate"assert step4.step.label == "Create site /sites/new"end@@ -196,6 +196,17 @@ defmodule Plausible.Stats.ExplorationTest doassert {:error, :empty_journey} = Exploration.journey_funnel(query, [])end+ test "returns error on too long journey", %{site: site} do+ query = QueryBuilder.build!(site, input_date_range: :all)++ journey =+ Enum.map(1..21, fn idx ->+ %Exploration.Journey.Step{name: "pageview", pathname: "/page#{idx}"}+ end)++ assert {:error, :journey_too_long} = Exploration.journey_funnel(query, journey)+ end+test "supports backward journey funnel", %{site: site} doquery = QueryBuilder.build!(site, input_date_range: :all)@@ -324,7 +335,7 @@ defmodule Plausible.Stats.ExplorationTest doassert {:ok, funnel} = Exploration.interesting_funnel(query)pathnames = Enum.map(funnel, & &1.step.pathname)- assert pathnames == ["/home", "/about/", "/contact"]+ assert pathnames == ["/about", "/contact"]endtest "stops when no more unseen steps are available" do@@ -430,13 +441,13 @@ defmodule Plausible.Stats.ExplorationTest doassert {:ok, [next_step1, next_step2, next_step3]} =Exploration.next_steps(query, journey)- assert next_step1.step.label == "Visit /docs"+ assert next_step1.step.label == "/docs"assert next_step1.step.pathname == "/docs"assert next_step1.visitors == 1- assert next_step2.step.label == "Visit /home"+ assert next_step2.step.label == "/home"assert next_step2.step.pathname == "/home"assert next_step2.visitors == 1- assert next_step3.step.label == "Visit /logout"+ assert next_step3.step.label == "/logout"assert next_step3.step.pathname == "/logout"assert next_step3.visitors == 1end@@ -453,6 +464,17 @@ defmodule Plausible.Stats.ExplorationTest doExploration.next_steps(query, journey, max_candidates: 1)end+ test "returns error on too long journey", %{site: site} do+ query = QueryBuilder.build!(site, input_date_range: :all)++ journey =+ Enum.map(1..20, fn idx ->+ %Exploration.Journey.Step{name: "pageview", pathname: "/page#{idx}"}+ end)++ assert {:error, :journey_too_long} = Exploration.next_steps(query, journey)+ end+test "suggests the first step in the journey", %{site: site} doquery = QueryBuilder.build!(site, input_date_range: :all)@@ -503,18 +525,42 @@ defmodule Plausible.Stats.ExplorationTest doassert next_step.visitors == 1end- test "allows to filter according to how label is rendered", %{site: site} do+ test "allows to filter according to how label is rendered" do+ site = new_site()++ now = DateTime.utc_now()++ populate_stats(site, [+ build(:pageview,+ user_id: 123,+ pathname: "/home",+ timestamp: DateTime.shift(now, minute: -320)+ ),+ build(:event,+ user_id: 123,+ name: "Signup",+ pathname: "/register",+ timestamp: DateTime.shift(now, minute: -300)+ ),+ build(:pageview,+ user_id: 123,+ pathname: "/sites/new",+ timestamp: DateTime.shift(now, minute: -270)+ )+ ])+query = QueryBuilder.build!(site, input_date_range: :all)journey = [- %Exploration.Journey.Step{name: "pageview", pathname: "/home"},- %Exploration.Journey.Step{name: "pageview", pathname: "/login"}+ %Exploration.Journey.Step{name: "pageview", pathname: "/home"}]assert {:ok, [next_step]} =- Exploration.next_steps(query, journey, search_term: "isit /doc")+ Exploration.next_steps(query, journey, search_term: "up /regi")- assert next_step.step.pathname == "/docs"+ assert next_step.step.label == "Signup /register"+ assert next_step.step.name == "Signup"+ assert next_step.step.pathname == "/register"assert next_step.visitors == 1end@@ -581,11 +627,17 @@ defmodule Plausible.Stats.ExplorationTest doquery = QueryBuilder.build!(site, input_date_range: :all)- assert {:ok, [%{step: %{pathname: "/:dashboard"}}, %{step: %{pathname: "/:dashboard/"}}]} =- Exploration.next_steps(query, journey, direction: :forward)-- assert {:ok, [%{step: %{pathname: "/:dashboard"}}, %{step: %{pathname: "/:dashboard/"}}]} =- Exploration.next_steps(query, journey, direction: :backward)+ assert {:ok,+ [+ %{step: %{pathname: "/:dashboard"}}+ ]} =+ Exploration.next_steps(query, journey, search_term: "", direction: :forward)++ assert {:ok,+ [+ %{step: %{pathname: "/:dashboard"}}+ ]} =+ Exploration.next_steps(query, journey, search_term: "", direction: :backward)endtest "treats identical sequence of events as a single step" do@@ -814,5 +866,106 @@ defmodule Plausible.Stats.ExplorationTest doassert next_step.visitors == 2endend++ test "implicit wildcard path visitors computation is correct and consistent between next_step and journey_funnel" do+ now = DateTime.utc_now()+ site = new_site()++ populate_stats(site, [+ build(:pageview,+ user_id: 122,+ pathname: "/aa",+ timestamp: DateTime.shift(now, minute: -300)+ ),+ build(:pageview,+ user_id: 123,+ pathname: "/a",+ timestamp: DateTime.shift(now, minute: -300)+ ),+ build(:pageview,+ user_id: 124,+ pathname: "/a",+ timestamp: DateTime.shift(now, minute: -300)+ ),+ build(:pageview,+ user_id: 125,+ pathname: "/a",+ timestamp: DateTime.shift(now, minute: -300)+ ),+ build(:pageview,+ user_id: 126,+ pathname: "/a",+ timestamp: DateTime.shift(now, minute: -300)+ ),+ build(:pageview,+ user_id: 127,+ pathname: "/a",+ timestamp: DateTime.shift(now, minute: -300)+ ),+ build(:pageview,+ user_id: 123,+ pathname: "/a/b",+ timestamp: DateTime.shift(now, minute: -270)+ ),+ build(:pageview,+ user_id: 124,+ pathname: "/a/b",+ timestamp: DateTime.shift(now, minute: -270)+ ),+ build(:pageview,+ user_id: 126,+ pathname: "/a/d",+ timestamp: DateTime.shift(now, minute: -270)+ ),+ build(:pageview,+ user_id: 127,+ pathname: "/a/d",+ timestamp: DateTime.shift(now, minute: -270)+ ),+ build(:pageview,+ user_id: 123,+ pathname: "/a/b/c",+ timestamp: DateTime.shift(now, minute: -240)+ ),+ build(:pageview,+ user_id: 128,+ pathname: "/a/b/c",+ timestamp: DateTime.shift(now, minute: -240)+ )+ ])++ query = QueryBuilder.build!(site, input_date_range: :all)++ result = Exploration.next_steps(query, [])++ assert {:ok,+ [+ %{step: %{label: "/a", includes_subpaths: true, subpaths_count: 4}, visitors: 6},+ %{step: %{label: "/a", includes_subpaths: false}, visitors: 5},+ %{+ step: %{label: "/a/b", includes_subpaths: true, subpaths_count: 2},+ visitors: 3+ },+ %{step: %{label: "/a/b", includes_subpaths: false}, visitors: 2},+ %{step: %{label: "/a/b/c"}, visitors: 2},+ %{step: %{label: "/a/d"}, visitors: 2},+ %{step: %{label: "/aa"}, visitors: 1}+ ]} = result++ journey = [+ %Exploration.Journey.Step{+ name: "pageview",+ pathname: "/a",+ includes_subpaths: true,+ subpaths_count: 4+ }+ ]++ assert {:ok, [step1]} = Exploration.journey_funnel(query, journey)++ assert step1.step.label == "/a"+ assert step1.step.includes_subpaths == true+ assert step1.visitors == 6+ endendenddiff --git a/test/plausible_web/controllers/api/stats_controller/exploration_test.exs b/test/plausible_web/controllers/api/stats_controller/exploration_test.exsindex 12e561fa23..589a0cc99c 100644--- a/test/plausible_web/controllers/api/stats_controller/exploration_test.exs+++ b/test/plausible_web/controllers/api/stats_controller/exploration_test.exs@@ -3,6 +3,8 @@ defmodule PlausibleWeb.Api.StatsController.ExplorationTest douse Plausibleon_ee do+ alias Plausible.Stats.Exploration.Journey+setup [:create_user, :log_in, :create_site]setup %{user: user, site: site} do@@ -62,11 +64,11 @@ defmodule PlausibleWeb.Api.StatsController.ExplorationTest doenddescribe "exploration_next/2" do- test "it works", %{conn: conn, site: site} do+ test "returns suggestions for the next step", %{conn: conn, site: site} dojourney =Jason.encode!([- %{name: "pageview", pathname: "/home"},- %{name: "pageview", pathname: "/login"}+ Journey.Step.new("pageview", "/home"),+ Journey.Step.new("pageview", "/login")])resp =@@ -78,22 +80,39 @@ defmodule PlausibleWeb.Api.StatsController.ExplorationTest do|> json_response(200)assert [next_step1, next_step2, next_step3] = resp- assert next_step1["step"]["label"] == "Visit /docs"+ assert next_step1["step"]["label"] == "/docs"assert next_step1["step"]["pathname"] == "/docs"assert next_step1["visitors"] == 1- assert next_step2["step"]["label"] == "Visit /home"+ assert next_step2["step"]["label"] == "/home"assert next_step2["step"]["pathname"] == "/home"assert next_step2["visitors"] == 1- assert next_step3["step"]["label"] == "Visit /logout"+ assert next_step3["step"]["label"] == "/logout"assert next_step3["step"]["pathname"] == "/logout"assert next_step3["visitors"] == 1end- test "it filters", %{conn: conn, site: site} do+ test "returns error on too long journey", %{conn: conn, site: site} do+ journey =+ Jason.encode!(+ Enum.map(1..20, fn idx -> Journey.Step.new("pageview", "/page#{idx}") end)+ )++ resp =+ conn+ |> post("/api/stats/#{site.domain}/exploration/next/", %{+ "journey" => journey,+ "period" => "24h"+ })+ |> json_response(400)++ assert resp["error"] == "The journey is too long"+ end++ test "allows filtering suggestions", %{conn: conn, site: site} dojourney =Jason.encode!([- %{name: "pageview", pathname: "/home"},- %{name: "pageview", pathname: "/login"}+ Journey.Step.new("pageview", "/home"),+ Journey.Step.new("pageview", "/login")])resp =@@ -110,8 +129,8 @@ defmodule PlausibleWeb.Api.StatsController.ExplorationTest doassert next_step["visitors"] == 1end- test "it supports backward direction", %{conn: conn, site: site} do- journey = Jason.encode!([%{name: "pageview", pathname: "/logout"}])+ test "supports backward direction", %{conn: conn, site: site} do+ journey = Jason.encode!([Journey.Step.new("pageview", "/logout")])resp =conn@@ -131,12 +150,12 @@ defmodule PlausibleWeb.Api.StatsController.ExplorationTest doenddescribe "exploration_funnel/2" do- test "it works", %{conn: conn, site: site} do+ test "returns a funnel", %{conn: conn, site: site} dojourney =Jason.encode!([- %{name: "pageview", pathname: "/home"},- %{name: "pageview", pathname: "/login"},- %{name: "pageview", pathname: "/logout"}+ Journey.Step.new("pageview", "/home"),+ Journey.Step.new("pageview", "/login"),+ Journey.Step.new("pageview", "/logout")])resp =@@ -149,21 +168,21 @@ defmodule PlausibleWeb.Api.StatsController.ExplorationTest doassert [step1, step2, step3] = resp- assert step1["step"]["label"] == "Visit /home"+ assert step1["step"]["label"] == "/home"assert step1["step"]["pathname"] == "/home"assert step1["visitors"] == 2assert step1["dropoff"] == 0assert step1["dropoff_percentage"] == "0"assert step1["conversion_rate"] == "100"assert step1["conversion_rate_step"] == "0"- assert step2["step"]["label"] == "Visit /login"+ assert step2["step"]["label"] == "/login"assert step2["step"]["pathname"] == "/login"assert step2["visitors"] == 2assert step2["dropoff"] == 0assert step2["dropoff_percentage"] == "0"assert step2["conversion_rate"] == "100"assert step2["conversion_rate_step"] == "100"- assert step3["step"]["label"] == "Visit /logout"+ assert step3["step"]["label"] == "/logout"assert step3["step"]["pathname"] == "/logout"assert step3["visitors"] == 1assert step3["dropoff"] == 1@@ -172,12 +191,41 @@ defmodule PlausibleWeb.Api.StatsController.ExplorationTest doassert step3["conversion_rate_step"] == "50"end- test "it supports backward direction", %{conn: conn, site: site} do+ test "returns error on too long journey", %{conn: conn, site: site} do+ journey =+ Jason.encode!(+ Enum.map(1..21, fn idx -> Journey.Step.new("pageview", "/page#{idx}") end)+ )++ resp =+ conn+ |> post("/api/stats/#{site.domain}/exploration/funnel/", %{+ "journey" => journey,+ "period" => "24h"+ })+ |> json_response(400)++ assert resp["error"] == "The journey is too long"+ end++ test "returns error on empty journey", %{conn: conn, site: site} do+ resp =+ conn+ |> post("/api/stats/#{site.domain}/exploration/funnel/", %{+ "journey" => Jason.encode!([]),+ "period" => "24h"+ })+ |> json_response(400)++ assert resp["error"] == "We are unable to show funnels when journey is empty"+ end++ test "supports backward direction", %{conn: conn, site: site} dojourney =Jason.encode!([- %{name: "pageview", pathname: "/logout"},- %{name: "pageview", pathname: "/login"},- %{name: "pageview", pathname: "/home"}+ Journey.Step.new("pageview", "/logout"),+ Journey.Step.new("pageview", "/login"),+ Journey.Step.new("pageview", "/home")])resp =@@ -216,8 +264,8 @@ defmodule PlausibleWeb.Api.StatsController.ExplorationTest dotest "returns next steps without funnel by default", %{conn: conn, site: site} dojourney =Jason.encode!([- %{name: "pageview", pathname: "/home"},- %{name: "pageview", pathname: "/login"}+ Journey.Step.new("pageview", "/home"),+ Journey.Step.new("pageview", "/login")])resp =@@ -229,13 +277,13 @@ defmodule PlausibleWeb.Api.StatsController.ExplorationTest do|> json_response(200)assert [next_step1, next_step2, next_step3] = resp["next"]- assert next_step1["step"]["label"] == "Visit /docs"+ assert next_step1["step"]["label"] == "/docs"assert next_step1["step"]["pathname"] == "/docs"assert next_step1["visitors"] == 1- assert next_step2["step"]["label"] == "Visit /home"+ assert next_step2["step"]["label"] == "/home"assert next_step2["step"]["pathname"] == "/home"assert next_step2["visitors"] == 1- assert next_step3["step"]["label"] == "Visit /logout"+ assert next_step3["step"]["label"] == "/logout"assert next_step3["step"]["pathname"] == "/logout"assert next_step3["visitors"] == 1@@ -245,8 +293,8 @@ defmodule PlausibleWeb.Api.StatsController.ExplorationTest dotest "filters next steps by search_term", %{conn: conn, site: site} dojourney =Jason.encode!([- %{name: "pageview", pathname: "/home"},- %{name: "pageview", pathname: "/login"}+ Journey.Step.new("pageview", "/home"),+ Journey.Step.new("pageview", "/login")])resp =@@ -265,7 +313,7 @@ defmodule PlausibleWeb.Api.StatsController.ExplorationTest doendtest "supports backward direction for next steps", %{conn: conn, site: site} do- journey = Jason.encode!([%{name: "pageview", pathname: "/logout"}])+ journey = Jason.encode!([Journey.Step.new("pageview", "/logout")])resp =conn@@ -287,9 +335,9 @@ defmodule PlausibleWeb.Api.StatsController.ExplorationTest dotest "includes funnel when include_funnel is true", %{conn: conn, site: site} dojourney =Jason.encode!([- %{name: "pageview", pathname: "/home"},- %{name: "pageview", pathname: "/login"},- %{name: "pageview", pathname: "/logout"}+ Journey.Step.new("pageview", "/home"),+ Journey.Step.new("pageview", "/login"),+ Journey.Step.new("pageview", "/logout")])resp =@@ -303,21 +351,21 @@ defmodule PlausibleWeb.Api.StatsController.ExplorationTest doassert [step1, step2, step3] = resp["funnel"]- assert step1["step"]["label"] == "Visit /home"+ assert step1["step"]["label"] == "/home"assert step1["step"]["pathname"] == "/home"assert step1["visitors"] == 2assert step1["dropoff"] == 0assert step1["dropoff_percentage"] == "0"assert step1["conversion_rate"] == "100"assert step1["conversion_rate_step"] == "0"- assert step2["step"]["label"] == "Visit /login"+ assert step2["step"]["label"] == "/login"assert step2["step"]["pathname"] == "/login"assert step2["visitors"] == 2assert step2["dropoff"] == 0assert step2["dropoff_percentage"] == "0"assert step2["conversion_rate"] == "100"assert step2["conversion_rate_step"] == "100"- assert step3["step"]["label"] == "Visit /logout"+ assert step3["step"]["label"] == "/logout"assert step3["step"]["pathname"] == "/logout"assert step3["visitors"] == 1assert step3["dropoff"] == 1@@ -332,9 +380,9 @@ defmodule PlausibleWeb.Api.StatsController.ExplorationTest do} dojourney =Jason.encode!([- %{name: "pageview", pathname: "/logout"},- %{name: "pageview", pathname: "/login"},- %{name: "pageview", pathname: "/home"}+ Journey.Step.new("pageview", "/logout"),+ Journey.Step.new("pageview", "/login"),+ Journey.Step.new("pageview", "/home")])resp =