Skip to content

Open source, privacy-first web analytics. Lightweight, cookie-free Google Analytics alternative. Self-hosted or cloud.

use windows in 'time on page' queries (#3446)

ruslandoga13055aafc09e
diff --git a/lib/plausible/stats/aggregate.ex b/lib/plausible/stats/aggregate.ex
index 694665daed..1e3c116f33 100644
--- a/lib/plausible/stats/aggregate.ex
+++ b/lib/plausible/stats/aggregate.ex
@@ -58,6 +58,14 @@ defmodule Plausible.Stats.Aggregate do
end
defp aggregate_time_on_page(site, query) do
+ if FunWithFlags.enabled?(:window_time_on_page) do
+ window_aggregate_time_on_page(site, query)
+ else
+ neighbor_aggregate_time_on_page(site, query)
+ end
+ end
+
+ defp neighbor_aggregate_time_on_page(site, query) do
q =
from(
e in base_event_query(site, %Query{
@@ -136,6 +144,50 @@ defmodule Plausible.Stats.Aggregate do
%{time_on_page: time_on_page}
end
+ defp window_aggregate_time_on_page(site, query) do
+ windowed_pages_q =
+ from e in base_event_query(site, %Query{
+ query
+ | filters: Map.delete(query.filters, "event:page")
+ }),
+ select: %{
+ next_timestamp: over(fragment("leadInFrame(?)", e.timestamp), :event_horizon),
+ next_pathname: over(fragment("leadInFrame(?)", e.pathname), :event_horizon),
+ timestamp: e.timestamp,
+ pathname: e.pathname,
+ session_id: e.session_id
+ },
+ windows: [
+ event_horizon: [
+ partition_by: e.session_id,
+ order_by: e.timestamp,
+ frame: fragment("ROWS BETWEEN CURRENT ROW AND 1 FOLLOWING")
+ ]
+ ]
+
+ timed_page_transitions_q =
+ from e in Ecto.Query.subquery(windowed_pages_q),
+ group_by: [e.pathname, e.next_pathname, e.session_id],
+ where: ^Plausible.Stats.Base.dynamic_filter_condition(query, "event:page", :pathname),
+ where: e.next_timestamp != 0,
+ select: %{
+ pathname: e.pathname,
+ transition: e.next_pathname != e.pathname,
+ duration: sum(e.next_timestamp - e.timestamp)
+ }
+
+ avg_time_per_page_transition_q =
+ from e in Ecto.Query.subquery(timed_page_transitions_q),
+ select: %{avg: fragment("sum(?)/countIf(?)", e.duration, e.transition)},
+ group_by: e.pathname
+
+ time_on_page_q =
+ from e in Ecto.Query.subquery(avg_time_per_page_transition_q),
+ select: fragment("avg(ifNotFinite(?,NULL))", e.avg)
+
+ %{time_on_page: ClickhouseRepo.one(time_on_page_q)}
+ end
+
@metrics_to_round [:bounce_rate, :time_on_page, :visit_duration, :sample_percent]
defp maybe_round_value({metric, nil}), do: {metric, 0}
diff --git a/lib/plausible/stats/base.ex b/lib/plausible/stats/base.ex
index dc04b706ad..56e95ebfc9 100644
--- a/lib/plausible/stats/base.ex
+++ b/lib/plausible/stats/base.ex
@@ -425,7 +425,7 @@ defmodule Plausible.Stats.Base do
end
# credo:disable-for-next-line Credo.Check.Refactor.CyclomaticComplexity
- defp dynamic_filter_condition(query, filter_key, db_field) do
+ def dynamic_filter_condition(query, filter_key, db_field) do
case query && query.filters && query.filters[filter_key] do
{:is, value} ->
value = db_field_val(db_field, value)
diff --git a/lib/plausible/stats/breakdown.ex b/lib/plausible/stats/breakdown.ex
index 8287456bf4..f86898c35a 100644
--- a/lib/plausible/stats/breakdown.ex
+++ b/lib/plausible/stats/breakdown.ex
@@ -211,10 +211,18 @@ defmodule Plausible.Stats.Breakdown do
end
defp breakdown_time_on_page(_site, _query, []) do
- []
+ %{}
end
defp breakdown_time_on_page(site, query, pages) do
+ if FunWithFlags.enabled?(:window_time_on_page) do
+ window_breakdown_time_on_page(site, query, pages)
+ else
+ neighbor_breakdown_time_on_page(site, query, pages)
+ end
+ end
+
+ defp neighbor_breakdown_time_on_page(site, query, pages) do
q =
from(
e in base_event_query(site, Query.remove_event_filters(query, [:page, :props])),
@@ -294,6 +302,85 @@ defmodule Plausible.Stats.Breakdown do
end
end
+ defp window_breakdown_time_on_page(site, query, pages) do
+ import Ecto.Query
+
+ windowed_pages_q =
+ from e in base_event_query(site, Query.remove_event_filters(query, [:page, :props])),
+ select: %{
+ next_timestamp: over(fragment("leadInFrame(?)", e.timestamp), :event_horizon),
+ next_pathname: over(fragment("leadInFrame(?)", e.pathname), :event_horizon),
+ timestamp: e.timestamp,
+ pathname: e.pathname,
+ session_id: e.session_id
+ },
+ windows: [
+ event_horizon: [
+ partition_by: e.session_id,
+ order_by: e.timestamp,
+ frame: fragment("ROWS BETWEEN CURRENT ROW AND 1 FOLLOWING")
+ ]
+ ]
+
+ timed_page_transitions_q =
+ from e in subquery(windowed_pages_q),
+ group_by: [e.pathname, e.next_pathname, e.session_id],
+ where: e.pathname in ^pages,
+ where: e.next_timestamp != 0,
+ select: %{
+ pathname: e.pathname,
+ transition: e.next_pathname != e.pathname,
+ duration: sum(e.next_timestamp - e.timestamp)
+ }
+
+ no_select_timed_pages_q =
+ from e in subquery(timed_page_transitions_q),
+ group_by: e.pathname
+
+ timed_pages_q =
+ if query.include_imported do
+ # Imported page views have pre-calculated values
+ imported_timed_pages_q =
+ from i in "imported_pages",
+ group_by: i.page,
+ where: i.site_id == ^site.id,
+ where: i.date >= ^query.date_range.first and i.date <= ^query.date_range.last,
+ where: i.page in ^pages,
+ select: %{
+ page: i.page,
+ time_on_page: sum(i.time_on_page),
+ visits: sum(i.pageviews) - sum(i.exits)
+ }
+
+ timed_pages_q =
+ from e in no_select_timed_pages_q,
+ select: %{
+ page: e.pathname,
+ time_on_page: sum(e.duration),
+ visits: fragment("countIf(?)", e.transition)
+ }
+
+ "timed_pages"
+ |> with_cte("timed_pages", as: ^timed_pages_q)
+ |> with_cte("imported_timed_pages", as: ^imported_timed_pages_q)
+ |> join(:full, [t], i in "imported_timed_pages", on: t.page == i.page)
+ |> select(
+ [t, i],
+ {
+ fragment("if(empty(?),?,?)", t.page, i.page, t.page),
+ (t.time_on_page + i.time_on_page) / (t.visits + i.visits)
+ }
+ )
+ else
+ from e in no_select_timed_pages_q,
+ select: {e.pathname, fragment("sum(?)/countIf(?)", e.duration, e.transition)}
+ end
+
+ timed_pages_q
+ |> Plausible.ClickhouseRepo.all()
+ |> Map.new()
+ end
+
defp joins_table?(ecto_q, table) do
Enum.any?(
ecto_q.joins,
diff --git a/test/plausible_web/controllers/api/stats_controller/pages_test.exs b/test/plausible_web/controllers/api/stats_controller/pages_test.exs
index fc28a29970..61c9c8d6f9 100644
--- a/test/plausible_web/controllers/api/stats_controller/pages_test.exs
+++ b/test/plausible_web/controllers/api/stats_controller/pages_test.exs
@@ -801,6 +801,39 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
|> json_response(200)
end
+ test "calculates time on page per unique transition within session", %{conn: conn, site: site} do
+ # ┌─p──┬─p2─┬─minus(t2, t)─┬──s─┐
+ # │ /a │ /b │ 100 │ s1 │
+ # │ /a │ /d │ 100 │ s2 │ <- these two get treated
+ # │ /a │ /d │ 0 │ s2 │ <- as single page transition
+ # └────┴────┴──────────────┴────┘
+ # so that time_on_page(a)=(100+100)/uniq(transition)=200/2=100
+
+ s1 = @user_id
+ s2 = @user_id + 1
+
+ now = ~N[2021-01-01 00:00:00]
+ later = fn seconds -> NaiveDateTime.add(now, seconds) end
+
+ populate_stats(site, [
+ build(:pageview, user_id: s1, timestamp: now, pathname: "/a"),
+ build(:pageview, user_id: s1, timestamp: later.(100), pathname: "/b"),
+ build(:pageview, user_id: s2, timestamp: now, pathname: "/a"),
+ build(:pageview, user_id: s2, timestamp: later.(100), pathname: "/d"),
+ build(:pageview, user_id: s2, timestamp: later.(100), pathname: "/a"),
+ build(:pageview, user_id: s2, timestamp: later.(100), pathname: "/d")
+ ])
+
+ assert [
+ %{"name" => "/a", "time_on_page" => 100.0},
+ %{"name" => "/b", "time_on_page" => nil},
+ %{"name" => "/d", "time_on_page" => 0.0}
+ ] =
+ conn
+ |> get("/api/stats/#{site.domain}/pages?period=day&date=2021-01-01&detailed=true")
+ |> json_response(200)
+ end
+
test "calculates bounce rate and time on page for pages with imported data", %{
conn: conn,
site: site
diff --git a/test/plausible_web/controllers/api/stats_controller/top_stats_test.exs b/test/plausible_web/controllers/api/stats_controller/top_stats_test.exs
index 4a911d6446..e9285aef4b 100644
--- a/test/plausible_web/controllers/api/stats_controller/top_stats_test.exs
+++ b/test/plausible_web/controllers/api/stats_controller/top_stats_test.exs
@@ -332,6 +332,43 @@ defmodule PlausibleWeb.Api.StatsController.TopStatsTest do
|> Map.fetch!("top_stats")
|> Enum.find(&(&1["name"] == "Time on page"))
end
+
+ test "averages time on page over unique transitions across sessions", %{
+ conn: conn,
+ site: site
+ } do
+ # ┌─p──┬─p2─┬─minus(t2, t)─┬──s─┐
+ # │ /a │ /b │ 100 │ s1 │
+ # │ /a │ /d │ 100 │ s2 │ <- these two get treated
+ # │ /a │ /d │ 0 │ s2 │ <- as single page transition
+ # └────┴────┴──────────────┴────┘
+ # so that time_on_page(a)=(100+100)/uniq(transition)=200/2=100
+
+ s1 = @user_id
+ s2 = @user_id + 1
+
+ now = ~N[2021-01-01 00:00:00]
+ later = fn seconds -> NaiveDateTime.add(now, seconds) end
+
+ populate_stats(site, [
+ build(:pageview, user_id: s1, timestamp: now, pathname: "/a"),
+ build(:pageview, user_id: s1, timestamp: later.(100), pathname: "/b"),
+ build(:pageview, user_id: s2, timestamp: now, pathname: "/a"),
+ build(:pageview, user_id: s2, timestamp: later.(100), pathname: "/d"),
+ build(:pageview, user_id: s2, timestamp: later.(100), pathname: "/a"),
+ build(:pageview, user_id: s2, timestamp: later.(100), pathname: "/d")
+ ])
+
+ filters = Jason.encode!(%{page: "/a"})
+ path = "/api/stats/#{site.domain}/top-stats?period=day&date=2021-01-01&filters=#{filters}"
+
+ assert %{"name" => "Time on page", "value" => 100} ==
+ conn
+ |> get(path)
+ |> json_response(200)
+ |> Map.fetch!("top_stats")
+ |> Enum.find(&(&1["name"] == "Time on page"))
+ end
end
describe "GET /api/stats/top-stats - with imported data" do
diff --git a/test/test_helper.exs b/test/test_helper.exs
index 494caca6d5..822e02d9ef 100644
--- a/test/test_helper.exs
+++ b/test/test_helper.exs
@@ -2,5 +2,6 @@
Mox.defmock(Plausible.HTTPClient.Mock, for: Plausible.HTTPClient.Interface)
Application.ensure_all_started(:double)
FunWithFlags.enable(:business_tier)
+FunWithFlags.enable(:window_time_on_page)
Ecto.Adapters.SQL.Sandbox.mode(Plausible.Repo, :manual)
ExUnit.configure(exclude: [:slow])