Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.
TypeScript analyticsanalytics-dashboardclickhouseelixirgoogle-analyticsmarketingopen-source-analyticsphoenixplausible-analyticspostgresqlprivacyprivacy-friendlyself-hostedsimple-analyticsstatisticstailwindcssweb-analyticswebsite-analyticswebsite-statswebsite-tracking
Exploration: lazily load 2-step funnel on first render (#6273)
Adam Rutkowski98392378b942
diff --git a/assets/js/dashboard/extra/exploration.js b/assets/js/dashboard/extra/exploration.jsindex b483ea6c2e..87dd6b9a48 100644--- a/assets/js/dashboard/extra/exploration.js+++ b/assets/js/dashboard/extra/exploration.js@@ -52,11 +52,11 @@ function fetchNextWithFunnel()}-function fetchSuggestedJourney(site, dashboardState) {+function fetchInterestingFunnel(site, dashboardState) {return api.post(url.apiPath(site, '/exploration/interesting-funnel'),dashboardState,- {}+ { max_steps: 2, max_candidates: 6 })}@@ -219,55 +219,16 @@ export function FunnelExploration() {// real funnel response arrives. Prevents from flashing "0 visitors"// during the loading window.const [provisionalFunnelEntries, setProvisionalFunnelEntries] = useState({})- // track in flight "Suggest a journey" request- const [isSuggestingJourney, setIsSuggestingJourney] = useState(false)- // counter to detect and discard stale suggestion responses- const suggestionRequestIdRef = useRef(0)// Tracks the steps/direction/dashboardState values from the previous effect// run so we can tell whether the journey changed (needs funnel) or only the// search filter changed (next steps only, no funnel).const prevStepsRef = useRef(steps)const prevDirectionRef = useRef(direction)const prevDashboardStateRef = useRef(dashboardState)-- function cancelPendingSuggestion() {- suggestionRequestIdRef.current += 1- setIsSuggestingJourney(false)- }-- function handleSuggestJourney() {- if (isSuggestingJourney) {- return- }-- const requestId = ++suggestionRequestIdRef.current- setIsSuggestingJourney(true)-- fetchSuggestedJourney(site, dashboardState)- .then((response) => {- // newer request (or an explicit cancel)- if (suggestionRequestIdRef.current !== requestId) {- return- }-- if (response && response.length > 0) {- setSteps(response.map(({ step }) => step))- setFunnel(response)- }- })- .catch(() => {})- .finally(() => {- if (suggestionRequestIdRef.current === requestId) {- setIsSuggestingJourney(false)- }- })- }+ const preloadFiredRef = useRef(false)+ const funnelFromPreloadRef = useRef(false)function handleSelect(columnIndex, selected) {- if (isSuggestingJourney) {- cancelPendingSuggestion()- }-// Reset the active-column filter whenever the journey changessetActiveColumnFilter('')@@ -307,10 +268,6 @@ export function FunnelExploration() {function handleDirectionSelect(nextDirection) {if (nextDirection === direction) return- if (isSuggestingJourney) {- cancelPendingSuggestion()- }-setDirection(nextDirection)setSteps(steps.toReversed())setFunnel([])@@ -319,14 +276,13 @@ export function FunnelExploration() {setProvisionalFunnelEntries({})}- // Fetch next step suggestions (and funnel, if the journey changed) whenever- // the journey, direction, dashboard filters, or search term change.- // Funnel is only re-fetched when steps or direction/dashboardState change,- // search doesn't affect it.+ // On first render fire the interesting-funnel preload and skip the normal+ // next-with-funnel fetch. Once the preload resolves it sets steps+ // and funnel, which re-triggers this effect for the next-step candidates fetch.+ //+ // On subsequent renders (via user interaction) fetch next steps and,+ // if the journey changed, also refetch the funnel.useEffect(() => {- setActiveColumnLoading(true)- setActiveColumnResults([])-const journeyChanged =prevStepsRef.current !== steps ||prevDirectionRef.current !== direction ||@@ -336,14 +292,65 @@ export function FunnelExploration() {prevDirectionRef.current = directionprevDashboardStateRef.current = dashboardState- const includeFunnel = journeyChanged && steps.length > 0+ let cancelled = false++ if (!preloadFiredRef.current) {+ preloadFiredRef.current = true+ setActiveColumnLoading(true)++ fetchInterestingFunnel(site, dashboardState)+ .then((response) => {+ if (cancelled) return+ if (response && response.length > 0) {+ funnelFromPreloadRef.current = true+ setSteps(response.map(({ step }) => step))+ setFunnel(response)+ } else {+ // Nothing to preload, fall back to a plain next-steps fetch+ fetchNextWithFunnel(site, dashboardState, [], '', direction, false)+ .then((r) => {+ if (!cancelled) setActiveColumnResults(r?.next || [])+ })+ .catch(() => {+ if (!cancelled) setActiveColumnResults([])+ })+ .finally(() => {+ if (!cancelled) setActiveColumnLoading(false)+ })+ }+ })+ .catch(() => {+ if (cancelled) return+ fetchNextWithFunnel(site, dashboardState, [], '', direction, false)+ .then((r) => {+ if (!cancelled) setActiveColumnResults(r?.next || [])+ })+ .catch(() => {+ if (!cancelled) setActiveColumnResults([])+ })+ .finally(() => {+ if (!cancelled) setActiveColumnLoading(false)+ })+ })++ return () => {+ cancelled = true+ }+ }++ setActiveColumnLoading(true)+ setActiveColumnResults([])++ const funnelAlreadyLoaded = funnelFromPreloadRef.current+ funnelFromPreloadRef.current = false++ const includeFunnel =+ journeyChanged && steps.length > 0 && !funnelAlreadyLoadedif (journeyChanged && steps.length === 0) {setFunnel([])}- let cancelled = false-fetchNextWithFunnel(site,dashboardState,@@ -393,16 +400,6 @@ export function FunnelExploration() {</h4></div><div className="flex shrink-0 items-center gap-3">- {steps.length === 0 &&- direction === EXPLORATION_DIRECTIONS.FORWARD && (- <button- onClick={handleSuggestJourney}- disabled={isSuggestingJourney}- className="text-xs font-medium text-indigo-500 hover:text-indigo-700 dark:text-indigo-400 dark:hover:text-indigo-200 disabled:opacity-50"- >- {isSuggestingJourney ? 'Suggesting...' : 'Suggest a journey'}- </button>- )}<div className="flex gap-1 overflow-hidden"><buttononClick={() =>diff --git a/extra/lib/plausible/stats/exploration.ex b/extra/lib/plausible/stats/exploration.exindex 0892a73d3c..95debb658c 100644--- a/extra/lib/plausible/stats/exploration.ex+++ b/extra/lib/plausible/stats/exploration.ex@@ -53,13 +53,18 @@ defmodule Plausible.Stats.Exploration doconversion_rate_step: String.t()}- @spec next_steps(Query.t(), journey(), String.t(), direction()) ::- {:ok, [next_step()]}- def next_steps(query, journey, search_term \\ "", direction \\ :forward)- when is_direction(direction) do+ @next_steps_defaults [search_term: "", direction: :forward, max_candidates: 10]++ @spec next_steps(Query.t(), journey(), keyword()) :: {:ok, [next_step()]}+ def next_steps(query, journey, opts \\ []) do+ opts = Keyword.merge(@next_steps_defaults, opts)+ direction = Keyword.fetch!(opts, :direction)+ search_term = Keyword.fetch!(opts, :search_term)+ max_candidates = min(Keyword.fetch!(opts, :max_candidates), 20)+query|> Base.base_event_query()- |> next_steps_query(journey, search_term, direction)+ |> next_steps_query(journey, search_term, direction, max_candidates)# We pass the query struct to record query metadata for# the CH debug console.|> ClickhouseRepo.all(query: query)@@ -95,23 +100,36 @@ defmodule Plausible.Stats.Exploration doin the journey yet. Trailing slashes are ignored whencomparing pathnames (e.g. `/foo` and `/foo/` are treated asthe same page - we should probably do that when deduplicating step candidates too).++ ## Options++ * `:max_steps` - maximum number of funnel steps to build (default: `6`)+ * `:max_candidates` - passed to `next_steps/3`, limiting+ how many candidate next steps are fetched per step (default: `10`)"""- @spec interesting_funnel(Query.t(), pos_integer()) ::+ @spec interesting_funnel(Query.t(), keyword()) ::{:ok, [funnel_step()]} | {:error, :not_found}- def interesting_funnel(query, max_steps \\ 6) do- case build_interesting_journey(query, [], MapSet.new(), max_steps) do+ def interesting_funnel(query, opts \\ []) do+ max_steps = min(Keyword.get(opts, :max_steps, 6), 20)+ max_candidates = min(Keyword.get(opts, :max_candidates, 10), 20)++ case build_interesting_journey(query, max_steps, max_candidates) do[] -> {:error, :not_found}journey -> journey_funnel(query, journey)endend- defp build_interesting_journey(_query, journey, _seen, max_steps)+ defp build_interesting_journey(query, max_steps, max_candidates) do+ do_build_journey(query, [], MapSet.new(), max_steps, max_candidates)+ end++ defp do_build_journey(_query, journey, _seen, max_steps, _max_candidates)when length(journey) >= max_steps dojourneyend- defp build_interesting_journey(query, journey, seen, max_steps) do- {:ok, candidates} = next_steps(query, journey, "")+ defp do_build_journey(query, journey, seen, max_steps, max_candidates) do+ {:ok, candidates} = next_steps(query, journey, max_candidates: max_candidates)case find_unseen_step(candidates, seen) donil ->@@ -119,7 +137,7 @@ defmodule Plausible.Stats.Exploration dostep ->new_seen = MapSet.put(seen, normalize_step_key(step))- build_interesting_journey(query, journey ++ [step], new_seen, max_steps)+ do_build_journey(query, journey ++ [step], new_seen, max_steps, max_candidates)endend@@ -136,7 +154,8 @@ defmodule Plausible.Stats.Exploration dodefp normalize_pathname("/"), do: "/"defp normalize_pathname(pathname), do: String.trim_trailing(pathname, "/")- defp next_steps_query(query, steps, search_term, direction) do+ defp next_steps_query(query, steps, search_term, direction, max_candidates)+ when is_direction(direction) donext_step_idx = length(steps) + 1q_steps = steps_query(query, next_step_idx, direction)@@ -171,7 +190,7 @@ defmodule Plausible.Stats.Exploration doasc: selected_as(:next_pathname),asc: selected_as(:next_name)],- limit: 10+ limit: ^max_candidates)|> maybe_search(search_term)diff --git a/lib/plausible_web/controllers/api/stats_controller.ex b/lib/plausible_web/controllers/api/stats_controller.exindex 36828ee9ee..b281213ccf 100644--- a/lib/plausible_web/controllers/api/stats_controller.ex+++ b/lib/plausible_web/controllers/api/stats_controller.ex@@ -145,7 +145,10 @@ defmodule PlausibleWeb.Api.StatsController do{:ok, direction} <- parse_exploration_direction(params["direction"]),query = Query.from(site, params, debug_metadata: debug_metadata(conn)),{:ok, next_steps} <-- Plausible.Stats.Exploration.next_steps(query, journey, search_term, direction) do+ Plausible.Stats.Exploration.next_steps(query, journey,+ search_term: search_term,+ direction: direction+ ) dojson(conn, next_steps)else_ ->@@ -178,7 +181,10 @@ defmodule PlausibleWeb.Api.StatsController dosite = conn.assigns.sitequery = Query.from(site, params, debug_metadata: debug_metadata(conn))- case Plausible.Stats.Exploration.interesting_funnel(query) do+ case Plausible.Stats.Exploration.interesting_funnel(query,+ max_steps: params["max_steps"],+ max_candidates: params["max_candidates"]+ ) do{:ok, funnel} -> json(conn, funnel){:error, :not_found} -> json(conn, [])end@@ -193,7 +199,10 @@ defmodule PlausibleWeb.Api.StatsController do{:ok, direction} <- parse_exploration_direction(params["direction"]),query = Query.from(site, params, debug_metadata: debug_metadata(conn)),{:ok, next_steps} <-- Plausible.Stats.Exploration.next_steps(query, journey, search_term, direction),+ Plausible.Stats.Exploration.next_steps(query, journey,+ search_term: search_term,+ direction: direction+ ),funnel <- maybe_include_funnel(include_funnel?, query, journey, direction) dojson(conn, %{next: next_steps, funnel: funnel})elsediff --git a/test/plausible/stats/exploration_test.exs b/test/plausible/stats/exploration_test.exsindex f404597889..817f7474ff 100644--- a/test/plausible/stats/exploration_test.exs+++ b/test/plausible/stats/exploration_test.exs@@ -251,7 +251,7 @@ defmodule Plausible.Stats.ExplorationTest dotest "limits the funnel to max_steps", %{site: site} doquery = QueryBuilder.build!(site, input_date_range: :all)- assert {:ok, [step1, step2]} = Exploration.interesting_funnel(query, 2)+ assert {:ok, [step1, step2]} = Exploration.interesting_funnel(query, max_steps: 2)assert step1.step.pathname == "/home"assert step2.step.pathname == "/login"@@ -341,7 +341,7 @@ defmodule Plausible.Stats.ExplorationTest doquery = QueryBuilder.build!(site, input_date_range: :all)- assert {:ok, [step1]} = Exploration.interesting_funnel(query, 6)+ assert {:ok, [step1]} = Exploration.interesting_funnel(query, max_steps: 6)assert step1.step.pathname == "/only-page"assert step1.visitors == 1@@ -441,6 +441,18 @@ defmodule Plausible.Stats.ExplorationTest doassert next_step3.visitors == 1end+ test "respects max_candidates", %{site: site} do+ query = QueryBuilder.build!(site, input_date_range: :all)++ journey = [+ %Exploration.Journey.Step{name: "pageview", pathname: "/home"},+ %Exploration.Journey.Step{name: "pageview", pathname: "/login"}+ ]++ assert {:ok, [%{step: %{pathname: "/docs"}}]} =+ Exploration.next_steps(query, journey, max_candidates: 1)+ end+test "suggests the first step in the journey", %{site: site} doquery = QueryBuilder.build!(site, input_date_range: :all)@@ -485,7 +497,7 @@ defmodule Plausible.Stats.ExplorationTest do%Exploration.Journey.Step{name: "pageview", pathname: "/login"}]- assert {:ok, [next_step]} = Exploration.next_steps(query, journey, "doc")+ assert {:ok, [next_step]} = Exploration.next_steps(query, journey, search_term: "doc")assert next_step.step.pathname == "/docs"assert next_step.visitors == 1@@ -499,7 +511,8 @@ defmodule Plausible.Stats.ExplorationTest do%Exploration.Journey.Step{name: "pageview", pathname: "/login"}]- assert {:ok, [next_step]} = Exploration.next_steps(query, journey, "isit /doc")+ assert {:ok, [next_step]} =+ Exploration.next_steps(query, journey, search_term: "isit /doc")assert next_step.step.pathname == "/docs"assert next_step.visitors == 1@@ -513,9 +526,8 @@ defmodule Plausible.Stats.ExplorationTest do]assert {:ok, [next_step1, next_step2]} =- Exploration.next_steps(query, journey, "", :backward)+ Exploration.next_steps(query, journey, direction: :backward)- assert next_step1.step.pathname == "/docs"assert next_step1.visitors == 1assert next_step2.step.pathname == "/login"assert next_step2.visitors == 1@@ -570,10 +582,10 @@ defmodule Plausible.Stats.ExplorationTest doquery = QueryBuilder.build!(site, input_date_range: :all)assert {:ok, [%{step: %{pathname: "/:dashboard"}}, %{step: %{pathname: "/:dashboard/"}}]} =- Exploration.next_steps(query, journey, "", :forward)+ Exploration.next_steps(query, journey, direction: :forward)assert {:ok, [%{step: %{pathname: "/:dashboard"}}, %{step: %{pathname: "/:dashboard/"}}]} =- Exploration.next_steps(query, journey, "", :backward)+ Exploration.next_steps(query, journey, direction: :backward)endtest "treats identical sequence of events as a single step" do@@ -722,7 +734,7 @@ defmodule Plausible.Stats.ExplorationTest do%Exploration.Journey.Step{name: "pageview", pathname: "/logout"}]- assert {:ok, [next_step]} = Exploration.next_steps(query, journey, "", :backward)+ assert {:ok, [next_step]} = Exploration.next_steps(query, journey, direction: :backward)assert next_step.step.pathname == "/login"assert next_step.visitors == 1