Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.
TypeScript analyticsanalytics-dashboardclickhouseelixirgoogle-analyticsmarketingopen-source-analyticsphoenixplausible-analyticspostgresqlprivacyprivacy-friendlyself-hostedsimple-analyticsstatisticstailwindcssweb-analyticswebsite-analyticswebsite-statswebsite-tracking
Suggest a journey (#6257)
Adam Rutkowski0eaaa498b971
diff --git a/assets/js/dashboard/stats/exploration.js b/assets/js/dashboard/stats/exploration.jsindex d304ed5926..df8292b86e 100644--- a/assets/js/dashboard/stats/exploration.js+++ b/assets/js/dashboard/stats/exploration.js@@ -51,6 +51,18 @@ function fetchFunnelData(site, dashboardState, steps, direction) {})}+function fetchSuggestedJourney(site, dashboardState) {+ return api.post(+ url.apiPath(site, '/exploration/interesting-funnel'),+ dashboardState,+ {}+ )+}++function isSameStep(step, otherStep) {+ return step.name === otherStep.name && step.pathname === otherStep.pathname+}+function ExplorationColumn({header,steps,@@ -63,7 +75,7 @@ function ExplorationColumn({direction}) {const site = useSiteContext()- const [loading, setLoading] = useState(steps !== null)+ const [loading, setLoading] = useState(steps !== null && !selected)const [results, setResults] = useState([])const [filter, setFilter] = useState('')const stepsFingerprint =@@ -76,10 +88,6 @@ function ExplorationColumn({)useEffect(() => {- if (selected) {- return- }-if (steps === null) {setFilter('')setResults([])@@ -87,6 +95,13 @@ function ExplorationColumn({return}+ if (selected) {+ // When a step is already selected (pre-populated by "Suggest a journey"),+ // fetch is unnecessary+ setLoading(false)+ return+ }+setLoading(true)setResults([])@@ -115,10 +130,23 @@ function ExplorationColumn({// eslint-disable-next-line react-hooks/exhaustive-deps}, [dashboardState, stepsFingerprint, filter, direction, site, selected])- const stepMaxVisitors =- direction === EXPLORATION_DIRECTIONS.BACKWARD- ? results[0]?.visitors- : maxVisitors || results[0]?.visitors+ const stepMaxVisitors = maxVisitors || results[0]?.visitors++ // When a step is selected we show only that one item.+ // If the full results list has loaded and contains a matching entry we use it+ // so the bar widths are still relative to the column's data; otherwise we+ // fall back to a synthetic item built from the funnel data+ const selectedResult =+ selected && results.find(({ step }) => isSameStep(step, selected))++ const listItems = selected+ ? [+ selectedResult || {+ step: selected,+ visitors: selectedVisitors ?? 0+ }+ ]+ : results.slice(0, 10)return (<div className="min-w-80 flex-1 border border-gray-200 dark:border-gray-750 rounded-lg overflow-hidden">@@ -152,29 +180,19 @@ function ExplorationColumn({<div></div></div></div>- ) : results.length === 0 ? (+ ) : results.length === 0 && !selected ? (<div className="h-108 flex items-center justify-center text-sm text-gray-400 dark:text-gray-500">{steps === null ? 'Select an event to continue' : 'No data'}</div>) : (<ul className="flex flex-col gap-y-0.5 px-1.5 pb-1.5 h-108 overflow-y-auto">- {(selected- ? results.filter(- ({ step }) =>- step.name === selected.name &&- step.pathname === selected.pathname- )- : results.slice(0, 10)- ).map(({ step, visitors }) => {- const isSelected =- !!selected &&- step.name === selected.name &&- step.pathname === selected.pathname+ {listItems.map(({ step, visitors }) => {+ const isSelected = !!selected && isSameStep(step, selected)const visitorsToShow =isSelected && selectedVisitors !== null? selectedVisitors: visitors- const conversionRateToShow =+ const barWidth =isSelected && selectedConversionRate !== null? selectedConversionRate: Math.round((visitors / stepMaxVisitors) * 100)@@ -210,7 +228,7 @@ function ExplorationColumn({? 'bg-indigo-500 dark:bg-white': 'bg-indigo-300 dark:bg-gray-300 group-hover:bg-indigo-400 dark:group-hover:bg-white'}`}- style={{ width: `${conversionRateToShow}%` }}+ style={{ width: `${barWidth}%` }}/></div></button>@@ -241,8 +259,49 @@ export function FunnelExploration() {const [steps, setSteps] = useState([])const [direction, setDirection] = useState(EXPLORATION_DIRECTIONS.FORWARD)const [funnel, setFunnel] = useState([])+ // track in flight "Suggest a journey" request+ const [isSuggestingJourney, setIsSuggestingJourney] = useState(false)+ // counter to detect and discard stale suggestion responses+ const suggestionRequestIdRef = useRef(0)++ function cancelPendingSuggestion() {+ suggestionRequestIdRef.current += 1+ setIsSuggestingJourney(false)+ }++ function handleSuggestJourney() {+ if (isSuggestingJourney) {+ return+ }++ const requestId = ++suggestionRequestIdRef.current+ setIsSuggestingJourney(true)++ fetchSuggestedJourney(site, dashboardState)+ .then((response) => {+ // newer request (or an explicit cancel)+ if (suggestionRequestIdRef.current !== requestId) {+ return+ }++ if (response && response.length > 0) {+ setSteps(response.map(({ step }) => step))+ setFunnel(response)+ }+ })+ .catch(() => {})+ .finally(() => {+ if (suggestionRequestIdRef.current === requestId) {+ setIsSuggestingJourney(false)+ }+ })+ }function handleSelect(columnIndex, selected) {+ if (isSuggestingJourney) {+ cancelPendingSuggestion()+ }+if (selected === null) {setSteps(steps.slice(0, columnIndex))} else {@@ -252,6 +311,11 @@ export function FunnelExploration() {function handleDirectionSelect(nextDirection) {if (nextDirection === direction) return++ if (isSuggestingJourney) {+ cancelPendingSuggestion()+ }+setDirection(nextDirection)setSteps([])setFunnel([])@@ -299,31 +363,43 @@ export function FunnelExploration() {Explore</h4></div>- <div className="flex shrink-0 gap-1 overflow-hidden">- <button- onClick={() =>- handleDirectionSelect(EXPLORATION_DIRECTIONS.FORWARD)- }- className={`px-2 py-1.5 text-xs font-medium rounded-md ${- direction === EXPLORATION_DIRECTIONS.FORWARD- ? 'bg-gray-150 text-gray-900 dark:bg-gray-750 dark:text-gray-100'- : 'text-gray-500 dark:text-gray-400'- }`}- >- Starting point- </button>- <button- onClick={() =>- handleDirectionSelect(EXPLORATION_DIRECTIONS.BACKWARD)- }- className={`px-2 py-1.5 text-xs font-medium rounded-md ${- direction === EXPLORATION_DIRECTIONS.BACKWARD- ? 'bg-gray-150 text-gray-900 dark:bg-gray-750 dark:text-gray-100'- : 'text-gray-500 dark:text-gray-400'- }`}- >- End point- </button>+ <div className="flex shrink-0 items-center gap-3">+ {steps.length === 0 &&+ direction === EXPLORATION_DIRECTIONS.FORWARD && (+ <button+ onClick={handleSuggestJourney}+ disabled={isSuggestingJourney}+ className="text-xs font-medium text-indigo-500 hover:text-indigo-700 dark:text-indigo-400 dark:hover:text-indigo-200 disabled:opacity-50"+ >+ {isSuggestingJourney ? 'Suggesting...' : 'Suggest a journey'}+ </button>+ )}+ <div className="flex gap-1 overflow-hidden">+ <button+ onClick={() =>+ handleDirectionSelect(EXPLORATION_DIRECTIONS.FORWARD)+ }+ className={`px-2 py-1.5 text-xs font-medium rounded-md ${+ direction === EXPLORATION_DIRECTIONS.FORWARD+ ? 'bg-gray-150 text-gray-900 dark:bg-gray-750 dark:text-gray-100'+ : 'text-gray-500 dark:text-gray-400'+ }`}+ >+ Starting point+ </button>+ <button+ onClick={() =>+ handleDirectionSelect(EXPLORATION_DIRECTIONS.BACKWARD)+ }+ className={`px-2 py-1.5 text-xs font-medium rounded-md ${+ direction === EXPLORATION_DIRECTIONS.BACKWARD+ ? 'bg-gray-150 text-gray-900 dark:bg-gray-750 dark:text-gray-100'+ : 'text-gray-500 dark:text-gray-400'+ }`}+ >+ End point+ </button>+ </div></div></div>diff --git a/lib/plausible/stats/exploration.ex b/lib/plausible/stats/exploration.exindex e4377cbbdb..8c51e55955 100644--- a/lib/plausible/stats/exploration.ex+++ b/lib/plausible/stats/exploration.ex@@ -79,6 +79,59 @@ defmodule Plausible.Stats.Exploration do|> then(&{:ok, &1})end+ @doc """+ Builds a "teaser" funnel by greedily selecting steps.++ We currently don't know what the "interesting" funnel might be,+ but blindly following the most visited cascade, oftentimes results with+ a repetitive back and forth between two pages.++ Therefore we start with the most visited entry and+ iteratively pick the most popular next step, that hasn't appeared+ in the journey yet. Trailing slashes are ignored when+ comparing pathnames (e.g. `/foo` and `/foo/` are treated as+ the same page - we should probably do that when deduplicating step candidates too).+ """+ @spec interesting_funnel(Query.t(), pos_integer()) ::+ {:ok, [funnel_step()]} | {:error, :not_found}+ def interesting_funnel(query, max_steps \\ 6) do+ case build_interesting_journey(query, [], MapSet.new(), max_steps) do+ [] -> {:error, :not_found}+ journey -> journey_funnel(query, journey)+ end+ end++ defp build_interesting_journey(_query, journey, _seen, max_steps)+ when length(journey) >= max_steps do+ journey+ end++ defp build_interesting_journey(query, journey, seen, max_steps) do+ {:ok, candidates} = next_steps(query, journey, "")++ case find_unseen_step(candidates, seen) do+ nil ->+ journey++ step ->+ new_seen = MapSet.put(seen, normalize_step_key(step))+ build_interesting_journey(query, journey ++ [step], new_seen, max_steps)+ end+ end++ defp find_unseen_step(candidates, seen) do+ Enum.find_value(candidates, fn %{step: step} ->+ if not MapSet.member?(seen, normalize_step_key(step)), do: step+ end)+ end++ defp normalize_step_key(%Journey.Step{name: name, pathname: pathname}) do+ {name, normalize_pathname(pathname)}+ end++ defp normalize_pathname("/"), do: "/"+ defp normalize_pathname(pathname), do: String.trim_trailing(pathname, "/")+defp next_steps_query(query, steps, search_term, direction) donext_step_idx = length(steps) + 1q_steps = steps_query(query, next_step_idx, direction)diff --git a/lib/plausible_web/controllers/api/stats_controller.ex b/lib/plausible_web/controllers/api/stats_controller.exindex 07abd203e6..9a41faff14 100644--- a/lib/plausible_web/controllers/api/stats_controller.ex+++ b/lib/plausible_web/controllers/api/stats_controller.ex@@ -25,7 +25,12 @@ defmodule PlausibleWeb.Api.StatsController do@revenue_metrics on_ee(do: Plausible.Stats.Goal.Revenue.revenue_metrics(), else: [])@not_set "(not set)"- plug PlausibleWeb.SuperAdminOnlyPlug when action in [:exploration_next, :exploration_funnel]+ plug PlausibleWeb.SuperAdminOnlyPlug+ when action in [+ :exploration_next,+ :exploration_funnel,+ :exploration_interesting_funnel+ ]plug(:date_validation_plug when action not in [:query])plug(:validate_required_filters_plug when action not in [:current_visitors])@@ -164,6 +169,16 @@ defmodule PlausibleWeb.Api.StatsController doendend+ def exploration_interesting_funnel(conn, params) do+ site = conn.assigns.site+ query = Query.from(site, params, debug_metadata: debug_metadata(conn))++ case Exploration.interesting_funnel(query) do+ {:ok, funnel} -> json(conn, funnel)+ {:error, :not_found} -> json(conn, [])+ end+ end+defp parse_journey(input) when is_binary(input) doinput|> Jason.decode!()diff --git a/lib/plausible_web/router.ex b/lib/plausible_web/router.exindex d6d9775227..d78f0b7dc4 100644--- a/lib/plausible_web/router.ex+++ b/lib/plausible_web/router.ex@@ -285,6 +285,10 @@ defmodule PlausibleWeb.Router dopost "/:domain/exploration/next", StatsController, :exploration_nextpost "/:domain/exploration/funnel", StatsController, :exploration_funnel+ post "/:domain/exploration/interesting-funnel",+ StatsController,+ :exploration_interesting_funnel+scope private: %{allow_consolidated_views: true} dopost "/:domain/query", StatsController, :queryget "/:domain/current-visitors", StatsController, :current_visitorsdiff --git a/test/plausible/stats/exploration_test.exs b/test/plausible/stats/exploration_test.exsindex 01b42f59fe..e2946b0479 100644--- a/test/plausible/stats/exploration_test.exs+++ b/test/plausible/stats/exploration_test.exs@@ -227,6 +227,195 @@ defmodule Plausible.Stats.ExplorationTest doendend+ describe "interesting_funnel" do+ test "builds a funnel starting with the most visited step", %{site: site} do+ query = QueryBuilder.build!(site, input_date_range: :all)++ assert {:ok, [step1, step2, step3, step4]} = Exploration.interesting_funnel(query)++ assert step1.step.pathname == "/home"+ assert step1.visitors == 2++ assert step2.step.pathname == "/login"+ assert step2.visitors == 2++ assert step3.step.pathname == "/docs"+ assert step3.visitors == 1++ assert step4.step.pathname == "/logout"+ assert step4.visitors == 1+ end++ test "limits the funnel to max_steps", %{site: site} do+ query = QueryBuilder.build!(site, input_date_range: :all)++ assert {:ok, [step1, step2]} = Exploration.interesting_funnel(query, 2)++ assert step1.step.pathname == "/home"+ assert step2.step.pathname == "/login"+ end++ test "returns error when no events exist" do+ empty_site = new_site()+ query = QueryBuilder.build!(empty_site, input_date_range: :all)++ assert {:error, :not_found} = Exploration.interesting_funnel(query)+ end++ test "treats trailing slash as the same page" do+ site = new_site()+ now = DateTime.utc_now()++ populate_stats(site, [+ # user 123+ build(:pageview,+ user_id: 123,+ pathname: "/home",+ timestamp: DateTime.shift(now, minute: -30)+ ),+ build(:pageview,+ user_id: 123,+ pathname: "/about/",+ timestamp: DateTime.shift(now, minute: -25)+ ),+ build(:pageview,+ user_id: 123,+ pathname: "/contact",+ timestamp: DateTime.shift(now, minute: -20)+ ),+ # user 124+ build(:pageview,+ user_id: 124,+ pathname: "/home",+ timestamp: DateTime.shift(now, minute: -30)+ ),+ build(:pageview,+ user_id: 124,+ pathname: "/about",+ timestamp: DateTime.shift(now, minute: -25)+ ),+ build(:pageview,+ user_id: 124,+ pathname: "/pricing",+ timestamp: DateTime.shift(now, minute: -20)+ ),+ # user 125+ build(:pageview,+ user_id: 125,+ pathname: "/home",+ timestamp: DateTime.shift(now, minute: -30)+ ),+ build(:pageview,+ user_id: 125,+ pathname: "/about/",+ timestamp: DateTime.shift(now, minute: -25)+ ),+ build(:pageview,+ user_id: 125,+ pathname: "/contact",+ timestamp: DateTime.shift(now, minute: -20)+ )+ ])++ query = QueryBuilder.build!(site, input_date_range: :all)++ assert {:ok, funnel} = Exploration.interesting_funnel(query)++ pathnames = Enum.map(funnel, & &1.step.pathname)+ assert pathnames == ["/home", "/about/", "/contact"]+ end++ test "stops when no more unseen steps are available" do+ site = new_site()+ now = DateTime.utc_now()++ populate_stats(site, [+ build(:pageview,+ user_id: 123,+ pathname: "/only-page",+ timestamp: DateTime.shift(now, minute: -30)+ )+ ])++ query = QueryBuilder.build!(site, input_date_range: :all)++ assert {:ok, [step1]} = Exploration.interesting_funnel(query, 6)++ assert step1.step.pathname == "/only-page"+ assert step1.visitors == 1+ end++ test "respects query filters", %{site: site} do+ query =+ QueryBuilder.build!(site,+ input_date_range: :all,+ filters: [[:is, "visit:browser", ["Firefox"]]]+ )++ assert {:ok, funnel} = Exploration.interesting_funnel(query)++ pathnames = Enum.map(funnel, & &1.step.pathname)+ assert pathnames == ["/docs", "/logout"]+ end++ test "does not revisit already-seen steps in a cycle" do+ site = new_site()+ now = DateTime.utc_now()++ populate_stats(site, [+ # user 123+ build(:pageview,+ user_id: 123,+ pathname: "/a",+ timestamp: DateTime.shift(now, minute: -50)+ ),+ build(:pageview,+ user_id: 123,+ pathname: "/b",+ timestamp: DateTime.shift(now, minute: -40)+ ),+ build(:pageview,+ user_id: 123,+ pathname: "/a",+ timestamp: DateTime.shift(now, minute: -30)+ ),+ build(:pageview,+ user_id: 123,+ pathname: "/b",+ timestamp: DateTime.shift(now, minute: -20)+ ),+ build(:pageview,+ user_id: 123,+ pathname: "/c",+ timestamp: DateTime.shift(now, minute: -10)+ ),+ # user 124+ build(:pageview,+ user_id: 124,+ pathname: "/a",+ timestamp: DateTime.shift(now, minute: -50)+ ),+ build(:pageview,+ user_id: 124,+ pathname: "/b",+ timestamp: DateTime.shift(now, minute: -40)+ ),+ build(:pageview,+ user_id: 124,+ pathname: "/c",+ timestamp: DateTime.shift(now, minute: -30)+ )+ ])++ query = QueryBuilder.build!(site, input_date_range: :all)++ assert {:ok, funnel} = Exploration.interesting_funnel(query)++ pathnames = Enum.map(funnel, & &1.step.pathname)+ assert pathnames == ["/a", "/b", "/c"]+ end+ end+describe "next_steps" dotest "suggests the next step for a 2-step journey", %{site: site} doquery = QueryBuilder.build!(site, input_date_range: :all)