Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.
TypeScript analyticsanalytics-dashboardclickhouseelixirgoogle-analyticsmarketingopen-source-analyticsphoenixplausible-analyticspostgresqlprivacyprivacy-friendlyself-hostedsimple-analyticsstatisticstailwindcssweb-analyticswebsite-analyticswebsite-statswebsite-tracking
use windows in 'time on page' queries (#3446)
ruslandoga13055aafc09e
diff --git a/lib/plausible/stats/aggregate.ex b/lib/plausible/stats/aggregate.exindex 694665daed..1e3c116f33 100644--- a/lib/plausible/stats/aggregate.ex+++ b/lib/plausible/stats/aggregate.ex@@ -58,6 +58,14 @@ defmodule Plausible.Stats.Aggregate doenddefp aggregate_time_on_page(site, query) do+ if FunWithFlags.enabled?(:window_time_on_page) do+ window_aggregate_time_on_page(site, query)+ else+ neighbor_aggregate_time_on_page(site, query)+ end+ end++ defp neighbor_aggregate_time_on_page(site, query) doq =from(e in base_event_query(site, %Query{@@ -136,6 +144,50 @@ defmodule Plausible.Stats.Aggregate do%{time_on_page: time_on_page}end+ defp window_aggregate_time_on_page(site, query) do+ windowed_pages_q =+ from e in base_event_query(site, %Query{+ query+ | filters: Map.delete(query.filters, "event:page")+ }),+ select: %{+ next_timestamp: over(fragment("leadInFrame(?)", e.timestamp), :event_horizon),+ next_pathname: over(fragment("leadInFrame(?)", e.pathname), :event_horizon),+ timestamp: e.timestamp,+ pathname: e.pathname,+ session_id: e.session_id+ },+ windows: [+ event_horizon: [+ partition_by: e.session_id,+ order_by: e.timestamp,+ frame: fragment("ROWS BETWEEN CURRENT ROW AND 1 FOLLOWING")+ ]+ ]++ timed_page_transitions_q =+ from e in Ecto.Query.subquery(windowed_pages_q),+ group_by: [e.pathname, e.next_pathname, e.session_id],+ where: ^Plausible.Stats.Base.dynamic_filter_condition(query, "event:page", :pathname),+ where: e.next_timestamp != 0,+ select: %{+ pathname: e.pathname,+ transition: e.next_pathname != e.pathname,+ duration: sum(e.next_timestamp - e.timestamp)+ }++ avg_time_per_page_transition_q =+ from e in Ecto.Query.subquery(timed_page_transitions_q),+ select: %{avg: fragment("sum(?)/countIf(?)", e.duration, e.transition)},+ group_by: e.pathname++ time_on_page_q =+ from e in Ecto.Query.subquery(avg_time_per_page_transition_q),+ select: fragment("avg(ifNotFinite(?,NULL))", e.avg)++ %{time_on_page: ClickhouseRepo.one(time_on_page_q)}+ end+@metrics_to_round [:bounce_rate, :time_on_page, :visit_duration, :sample_percent]defp maybe_round_value({metric, nil}), do: {metric, 0}diff --git a/lib/plausible/stats/base.ex b/lib/plausible/stats/base.exindex dc04b706ad..56e95ebfc9 100644--- a/lib/plausible/stats/base.ex+++ b/lib/plausible/stats/base.ex@@ -425,7 +425,7 @@ defmodule Plausible.Stats.Base doend# credo:disable-for-next-line Credo.Check.Refactor.CyclomaticComplexity- defp dynamic_filter_condition(query, filter_key, db_field) do+ def dynamic_filter_condition(query, filter_key, db_field) docase query && query.filters && query.filters[filter_key] do{:is, value} ->value = db_field_val(db_field, value)diff --git a/lib/plausible/stats/breakdown.ex b/lib/plausible/stats/breakdown.exindex 8287456bf4..f86898c35a 100644--- a/lib/plausible/stats/breakdown.ex+++ b/lib/plausible/stats/breakdown.ex@@ -211,10 +211,18 @@ defmodule Plausible.Stats.Breakdown doenddefp breakdown_time_on_page(_site, _query, []) do- []+ %{}enddefp breakdown_time_on_page(site, query, pages) do+ if FunWithFlags.enabled?(:window_time_on_page) do+ window_breakdown_time_on_page(site, query, pages)+ else+ neighbor_breakdown_time_on_page(site, query, pages)+ end+ end++ defp neighbor_breakdown_time_on_page(site, query, pages) doq =from(e in base_event_query(site, Query.remove_event_filters(query, [:page, :props])),@@ -294,6 +302,85 @@ defmodule Plausible.Stats.Breakdown doendend+ defp window_breakdown_time_on_page(site, query, pages) do+ import Ecto.Query++ windowed_pages_q =+ from e in base_event_query(site, Query.remove_event_filters(query, [:page, :props])),+ select: %{+ next_timestamp: over(fragment("leadInFrame(?)", e.timestamp), :event_horizon),+ next_pathname: over(fragment("leadInFrame(?)", e.pathname), :event_horizon),+ timestamp: e.timestamp,+ pathname: e.pathname,+ session_id: e.session_id+ },+ windows: [+ event_horizon: [+ partition_by: e.session_id,+ order_by: e.timestamp,+ frame: fragment("ROWS BETWEEN CURRENT ROW AND 1 FOLLOWING")+ ]+ ]++ timed_page_transitions_q =+ from e in subquery(windowed_pages_q),+ group_by: [e.pathname, e.next_pathname, e.session_id],+ where: e.pathname in ^pages,+ where: e.next_timestamp != 0,+ select: %{+ pathname: e.pathname,+ transition: e.next_pathname != e.pathname,+ duration: sum(e.next_timestamp - e.timestamp)+ }++ no_select_timed_pages_q =+ from e in subquery(timed_page_transitions_q),+ group_by: e.pathname++ timed_pages_q =+ if query.include_imported do+ # Imported page views have pre-calculated values+ imported_timed_pages_q =+ from i in "imported_pages",+ group_by: i.page,+ where: i.site_id == ^site.id,+ where: i.date >= ^query.date_range.first and i.date <= ^query.date_range.last,+ where: i.page in ^pages,+ select: %{+ page: i.page,+ time_on_page: sum(i.time_on_page),+ visits: sum(i.pageviews) - sum(i.exits)+ }++ timed_pages_q =+ from e in no_select_timed_pages_q,+ select: %{+ page: e.pathname,+ time_on_page: sum(e.duration),+ visits: fragment("countIf(?)", e.transition)+ }++ "timed_pages"+ |> with_cte("timed_pages", as: ^timed_pages_q)+ |> with_cte("imported_timed_pages", as: ^imported_timed_pages_q)+ |> join(:full, [t], i in "imported_timed_pages", on: t.page == i.page)+ |> select(+ [t, i],+ {+ fragment("if(empty(?),?,?)", t.page, i.page, t.page),+ (t.time_on_page + i.time_on_page) / (t.visits + i.visits)+ }+ )+ else+ from e in no_select_timed_pages_q,+ select: {e.pathname, fragment("sum(?)/countIf(?)", e.duration, e.transition)}+ end++ timed_pages_q+ |> Plausible.ClickhouseRepo.all()+ |> Map.new()+ end+defp joins_table?(ecto_q, table) doEnum.any?(ecto_q.joins,diff --git a/test/plausible_web/controllers/api/stats_controller/pages_test.exs b/test/plausible_web/controllers/api/stats_controller/pages_test.exsindex fc28a29970..61c9c8d6f9 100644--- a/test/plausible_web/controllers/api/stats_controller/pages_test.exs+++ b/test/plausible_web/controllers/api/stats_controller/pages_test.exs@@ -801,6 +801,39 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do|> json_response(200)end+ test "calculates time on page per unique transition within session", %{conn: conn, site: site} do+ # ┌─p──┬─p2─┬─minus(t2, t)─┬──s─┐+ # │ /a │ /b │ 100 │ s1 │+ # │ /a │ /d │ 100 │ s2 │ <- these two get treated+ # │ /a │ /d │ 0 │ s2 │ <- as single page transition+ # └────┴────┴──────────────┴────┘+ # so that time_on_page(a)=(100+100)/uniq(transition)=200/2=100++ s1 = @user_id+ s2 = @user_id + 1++ now = ~N[2021-01-01 00:00:00]+ later = fn seconds -> NaiveDateTime.add(now, seconds) end++ populate_stats(site, [+ build(:pageview, user_id: s1, timestamp: now, pathname: "/a"),+ build(:pageview, user_id: s1, timestamp: later.(100), pathname: "/b"),+ build(:pageview, user_id: s2, timestamp: now, pathname: "/a"),+ build(:pageview, user_id: s2, timestamp: later.(100), pathname: "/d"),+ build(:pageview, user_id: s2, timestamp: later.(100), pathname: "/a"),+ build(:pageview, user_id: s2, timestamp: later.(100), pathname: "/d")+ ])++ assert [+ %{"name" => "/a", "time_on_page" => 100.0},+ %{"name" => "/b", "time_on_page" => nil},+ %{"name" => "/d", "time_on_page" => 0.0}+ ] =+ conn+ |> get("/api/stats/#{site.domain}/pages?period=day&date=2021-01-01&detailed=true")+ |> json_response(200)+ end+test "calculates bounce rate and time on page for pages with imported data", %{conn: conn,site: sitediff --git a/test/plausible_web/controllers/api/stats_controller/top_stats_test.exs b/test/plausible_web/controllers/api/stats_controller/top_stats_test.exsindex 4a911d6446..e9285aef4b 100644--- a/test/plausible_web/controllers/api/stats_controller/top_stats_test.exs+++ b/test/plausible_web/controllers/api/stats_controller/top_stats_test.exs@@ -332,6 +332,43 @@ defmodule PlausibleWeb.Api.StatsController.TopStatsTest do|> Map.fetch!("top_stats")|> Enum.find(&(&1["name"] == "Time on page"))end++ test "averages time on page over unique transitions across sessions", %{+ conn: conn,+ site: site+ } do+ # ┌─p──┬─p2─┬─minus(t2, t)─┬──s─┐+ # │ /a │ /b │ 100 │ s1 │+ # │ /a │ /d │ 100 │ s2 │ <- these two get treated+ # │ /a │ /d │ 0 │ s2 │ <- as single page transition+ # └────┴────┴──────────────┴────┘+ # so that time_on_page(a)=(100+100)/uniq(transition)=200/2=100++ s1 = @user_id+ s2 = @user_id + 1++ now = ~N[2021-01-01 00:00:00]+ later = fn seconds -> NaiveDateTime.add(now, seconds) end++ populate_stats(site, [+ build(:pageview, user_id: s1, timestamp: now, pathname: "/a"),+ build(:pageview, user_id: s1, timestamp: later.(100), pathname: "/b"),+ build(:pageview, user_id: s2, timestamp: now, pathname: "/a"),+ build(:pageview, user_id: s2, timestamp: later.(100), pathname: "/d"),+ build(:pageview, user_id: s2, timestamp: later.(100), pathname: "/a"),+ build(:pageview, user_id: s2, timestamp: later.(100), pathname: "/d")+ ])++ filters = Jason.encode!(%{page: "/a"})+ path = "/api/stats/#{site.domain}/top-stats?period=day&date=2021-01-01&filters=#{filters}"++ assert %{"name" => "Time on page", "value" => 100} ==+ conn+ |> get(path)+ |> json_response(200)+ |> Map.fetch!("top_stats")+ |> Enum.find(&(&1["name"] == "Time on page"))+ endenddescribe "GET /api/stats/top-stats - with imported data" dodiff --git a/test/test_helper.exs b/test/test_helper.exsindex 494caca6d5..822e02d9ef 100644--- a/test/test_helper.exs+++ b/test/test_helper.exs@@ -2,5 +2,6 @@Mox.defmock(Plausible.HTTPClient.Mock, for: Plausible.HTTPClient.Interface)Application.ensure_all_started(:double)FunWithFlags.enable(:business_tier)+FunWithFlags.enable(:window_time_on_page)Ecto.Adapters.SQL.Sandbox.mode(Plausible.Repo, :manual)ExUnit.configure(exclude: [:slow])